@fireproof/core 0.6.1 → 0.6.3-dev

package/dist/src/loader.js

@@ -0,0 +1,131 @@
+import { readFileSync
+// createReadStream
+ } from 'node:fs';
+import { mkdir, writeFile } from 'fs/promises';
+import { openDB } from 'idb';
+import { join, dirname } from 'path';
+// import { parse } from '@jsonlines/core'
+// import cargoQueue from 'async/cargoQueue.js'
+import { homedir } from 'os';
+const defaultConfig = {
+    dataDir: join(homedir(), '.fireproof'),
+    headerKeyPrefix: 'fp.'
+};
+const FORCE_IDB = typeof process !== 'undefined' && !!process.env?.FORCE_IDB;
+/* global localStorage */
+export class Loader {
+    constructor(name, keyId, config = defaultConfig) {
+        this.name = name;
+        this.keyId = keyId;
+        this.config = config;
+        this.isBrowser = false;
+        try {
+            this.isBrowser = window.localStorage && true;
+        }
+        catch (e) { }
+    }
+    withDB = async (dbWorkFun) => {
+        if (!this.idb) {
+            this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 3, {
+                upgrade(db, oldVersion, newVersion, transaction) {
+                    if (oldVersion < 1) {
+                        db.createObjectStore('cars');
+                    }
+                }
+            });
+        }
+        return await dbWorkFun(this.idb);
+    };
+    async writeCars(cars) {
+        // console.log('writeCars', this.config.dataDir, this.name, cars.map(c => c.cid.toString()))
+        // console.log('writeCars', cars.length)
+        if (FORCE_IDB || this.isBrowser) {
+            await this.writeCarsIDB(cars);
+        }
+        else {
+            const writes = [];
+            for (const { cid, bytes } of cars) {
+                const carFilename = join(this.config.dataDir, this.name, `${cid.toString()}.car`);
+                // console.log('writeCars', carFilename)
+                writes.push(writeSync(carFilename, bytes));
+            }
+            await Promise.all(writes);
+        }
+    }
+    async writeCarsIDB(cars) {
+        return await this.withDB(async (db) => {
+            const tx = db.transaction(['cars'], 'readwrite');
+            for (const { cid, bytes, replaces } of cars) {
+                await tx.objectStore('cars').put(bytes, cid.toString());
+                // todo remove old maps
+                if (replaces) {
+                    await tx.objectStore('cars').delete(replaces.toString());
+                }
+            }
+            return await tx.done;
+        });
+    }
+    async readCar(carCid) {
+        if (FORCE_IDB || this.isBrowser) {
+            return await this.readCarIDB(carCid);
+        }
+        else {
+            const carFilename = join(this.config.dataDir, this.name, `${carCid.toString()}.car`);
+            const got = readFileSync(carFilename);
+            // console.log('readCar', carFilename, got.constructor.name)
+            return got;
+        }
+    }
+    async readCarIDB(carCid) {
+        return await this.withDB(async (db) => {
+            const tx = db.transaction(['cars'], 'readonly');
+            // console.log('getCarReader', carCid)
+            return await tx.objectStore('cars').get(carCid);
+        });
+    }
+    getHeader() {
+        if (this.isBrowser) {
+            return localStorage.getItem(this.config.headerKeyPrefix + this.name);
+        }
+        else {
+            return loadSync(this.headerFilename());
+            // return null
+        }
+    }
+    async saveHeader(stringValue) {
+        // console.log('saveHeader', this.isBrowser)
+        if (this.isBrowser) {
+            // console.log('localStorage!', this.config.headerKeyPrefix)
+            return localStorage.setItem(this.config.headerKeyPrefix + this.name, stringValue);
+        }
+        else {
+            // console.log('no localStorage', this.config.dataDir, this.name)
+            // console.log('saving clock to', this.headerFilename(), stringValue)
+            try {
+                await writeSync(this.headerFilename(), stringValue);
+            }
+            catch (error) {
+                console.log('error', error);
+            }
+            // console.log('saved clock to', this.headerFilename())
+        }
+    }
+    headerFilename() {
+        // console.log('headerFilename', this.config.dataDir, this.name)
+        return join(this.config.dataDir, this.name, 'header.json');
+    }
+}
+function loadSync(filename) {
+    try {
+        return readFileSync(filename, 'utf8').toString();
+    }
+    catch (error) {
+        // console.log('error', error)
+        return null;
+    }
+}
+async function writeSync(fullpath, stringValue) {
+    await mkdir(dirname(fullpath), { recursive: true });
+    // writeFileSync(fullpath, stringValue)
+    await writeFile(fullpath, stringValue);
+}

package/dist/src/prolly.js

@@ -8,6 +8,7 @@ import { nocache as cache } from 'prolly-trees/cache';
 import { CIDCounter, bf, simpleCompare as compare } from 'prolly-trees/utils';
 import * as codec from '@ipld/dag-cbor';
 import { sha256 as hasher } from 'multiformats/hashes/sha2';
+// import { blake2b256 as hasher } from '@multiformats/blake2/blake2b'
 import { doTransaction } from './blockstore.js';
 import { create as createBlock } from 'multiformats/block';
 const blockOpts = { cache, chunker: bf(30), codec, hasher, compare };

package/dist/src/valet.js

@@ -6,8 +6,8 @@ import * as CBW from '@ipld/car/buffer-writer';
 import * as raw from 'multiformats/codecs/raw';
 import * as Block from 'multiformats/block';
 import * as dagcbor from '@ipld/dag-cbor';
-import { openDB } from 'idb';
 import cargoQueue from 'async/cargoQueue.js';
+import { Loader } from './loader.js';
 // @ts-ignore
 // @ts-ignore
 import { bf, simpleCompare as compare } from 'prolly-trees/utils';
@@ -44,6 +44,7 @@ export class Valet {
     constructor(name = 'default', keyMaterial) {
         this.name = name;
         this.setKeyMaterial(keyMaterial);
+        this.loader = new Loader(name, this.keyId); // todo send this config.loader, if we ever need it
         this.uploadQueue = cargoQueue(async (tasks, callback) => {
             // console.log(
             //   'queue worker',
@@ -76,6 +77,9 @@ export class Valet {
             //   })
         });
     }
+    saveHeader(header) {
+        return this.loader.saveHeader(header);
+    }
     getKeyMaterial() {
         return this.keyMaterial;
     }
@@ -116,18 +120,6 @@ export class Valet {
             throw new Error('missing lastCid for car header');
         }
     }
-    withDB = async (dbWorkFun) => {
-        if (!this.idb) {
-            this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 3, {
-                upgrade(db, oldVersion, newVersion, transaction) {
-                    if (oldVersion < 1) {
-                        db.createObjectStore('cars');
-                    }
-                }
-            });
-        }
-        return await dbWorkFun(this.idb);
-    };
     /**
      * Iterate over all blocks in the store.
      *
@@ -167,6 +159,7 @@ export class Valet {
                 blockHasher: blockOpts.hasher,
                 blockCodec: blockOpts.codec
             });
+            this.valetRoot = indexNode;
         }
         const got = await indexNode.get(cid);
         // console.log('getCarCIDForCID', cid, got)
@@ -211,13 +204,14 @@ export class Valet {
      * @param {*} value
      */
     async parkCar(carCid, value, cids) {
+        // const callId = Math.random().toString(36).substring(7)
         // console.log('parkCar', this.instanceId, this.name, carCid, cids)
         const combinedReader = await this.getCombinedReader(carCid);
         const mapNode = await addCidsToCarIndex(combinedReader, this.valetRoot, this.valetRootCid, Array.from(cids).map(cid => ({ key: cid.toString(), value: carCid.toString() })));
         this.valetRoot = mapNode;
         this.valetRootCid = mapNode.cid;
         // make a block set with all the cids of the map
-        const saveValetBlocks = new VMemoryBlockstore(); //  todo this blockstore should read from the last valetCid car also
+        const saveValetBlocks = new VMemoryBlockstore();
         for await (const cidx of mapNode.cids()) {
             const bytes = await combinedReader.get(cidx);
             saveValetBlocks.put(cidx, bytes);
@@ -230,7 +224,8 @@ export class Valet {
             newValetCidCar = await blocksToCarBlock(this.valetRootCid, saveValetBlocks);
         }
         // console.log('newValetCidCar', this.name, Math.floor(newValetCidCar.bytes.length / 1024))
-        await this.writeCars([
+        // console.log('writeCars', callId, carCid.toString(), newValetCidCar.cid.toString())
+        await this.loader.writeCars([
             {
                 cid: carCid,
                 bytes: value,
@@ -244,6 +239,7 @@ export class Valet {
             }
         ]);
         this.valetRootCarCid = newValetCidCar.cid; // goes to clock
+        // console.log('wroteCars', callId, carCid.toString(), newValetCidCar.cid.toString())
         // console.log('parked car', carCid, value.length, Array.from(cids))
         // upload to web3.storage if we have credentials
         if (this.uploadFunction) {
@@ -260,27 +256,12 @@ export class Valet {
             // console.log('no upload function', carCid, value.length, this.uploadFunction)
         }
     }
-    async writeCars(cars) {
-        return await this.withDB(async (db) => {
-            const tx = db.transaction(['cars'], 'readwrite');
-            for (const { cid, bytes, replaces } of cars) {
-                await tx.objectStore('cars').put(bytes, cid.toString());
-                // todo remove old maps
-                if (replaces) {
-                    await tx.objectStore('cars').delete(replaces.toString());
-                }
-            }
-            return await tx.done;
-        });
-    }
     remoteBlockFunction = null;
     async getCarReader(carCid) {
         carCid = carCid.toString();
-        const carBytes = await this.withDB(async (db) => {
-            const tx = db.transaction(['cars'], 'readonly');
-            // console.log('getCarReader', carCid)
-            return await tx.objectStore('cars').get(carCid);
-        });
+        const carBytes = await this.loader.readCar(carCid);
+        // const callID = Math.random().toString(36).substring(7)
+        // console.log('getCarReader', callID, carCid)
         const reader = await CarReader.fromBytes(carBytes);
         if (this.keyMaterial) {
             const roots = await reader.getRoots();
@@ -303,6 +284,7 @@ export class Valet {
             const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial);
             // last block is the root ??? todo
             const rootBlock = blocks[blocks.length - 1];
+            // console.log('got reader', callID, carCid)
             return {
                 root: rootBlock,
                 get: async (dataCID) => {
@@ -424,6 +406,8 @@ const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
 };
 const addCidsToCarIndex = async (blockstore, valetRoot, valetRootCid, bulkOperations) => {
     let indexNode;
+    // const callID = Math.random().toString(32).substring(2, 8)
+    // console.log('addCidsToCarIndex', callID, valetRootCid, bulkOperations.length)
     if (valetRootCid) {
         if (valetRoot) {
             indexNode = valetRoot;
@@ -445,6 +429,7 @@ const addCidsToCarIndex = async (blockstore, valetRoot, valetRootCid, bulkOperat
         // console.log('adding', key, value)
         await indexNode.set(key, value);
     }
+    // console.log('newCidsToCarIndex', callID, indexNode.cid, bulkOperations.length)
     return indexNode;
 };
 export class VMemoryBlockstore {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.6.1",
+  "version": "0.6.3-dev",
   "description": "Live data for React, accelerated by proofs, powered by IPFS",
   "main": "dist/src/fireproof.js",
   "module": "dist/src/fireproof.mjs",
@@ -40,6 +40,8 @@
   "dependencies": {
     "@ipld/car": "^5.1.0",
     "@ipld/dag-cbor": "^9.0.0",
+    "@jsonlines/core": "^1.0.2",
+    "@multiformats/blake2": "^1.0.13",
     "@rollup/plugin-commonjs": "^24.0.1",
     "archy": "^1.0.0",
     "async": "^3.2.4",
@@ -74,6 +76,7 @@
     "rollup-plugin-esbuild": "^5.0.0",
     "rollup-plugin-node-builtins": "^2.1.2",
     "rollup-plugin-polyfill-node": "^0.12.0",
+    "rollup-plugin-visualizer": "^5.9.0",
     "standard": "^17.0.0",
     "typescript": "^5.0.2",
     "webpack": "^5.78.0",
@@ -119,8 +122,5 @@
     "dist",
     "hooks",
     "README.md"
-  ],
-  "workspaces": [
-    "examples/todomvc"
   ]
 }

package/src/database.js

@@ -2,7 +2,6 @@
 import { visMerkleClock, visMerkleTree, vis, put, get, getAll, eventsSince } from './prolly.js'
 import { doTransaction, TransactionBlockstore } from './blockstore.js'
 import charwise from 'charwise'
-import { localSet } from './utils.js'
 import { CID } from 'multiformats'
 
 // TypeScript Types
@@ -34,7 +33,7 @@ export class Database {
     this.name = name
     this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`
     this.blocks = new TransactionBlockstore(name, config.key)
-    this.indexBlocks = new TransactionBlockstore(name + '.indexes', config.key)
+    this.indexBlocks = new TransactionBlockstore(name ? name + '.indexes' : null, config.key)
     this.clock = clock
     this.config = config
   }
@@ -78,7 +77,7 @@ export class Database {
 
   maybeSaveClock () {
     if (this.name && this.blocks.valet) {
-      localSet('fp.' + this.name, JSON.stringify(this))
+      this.blocks.valet.saveHeader(JSON.stringify(this))
     }
   }
 
@@ -314,6 +313,7 @@ export class Database {
       console.error('failed', event)
       throw new Error('failed to put at storage layer')
     }
+    // await new Promise(resolve => setTimeout(resolve, 10)) // makes concurrent tests work
     this.applyClock(prevClock, result.head)
     await this.notifyListeners([decodedEvent]) // this type is odd
     return {

package/src/fireproof.js

@@ -1,9 +1,10 @@
 import randomBytes from 'randombytes'
+// import { randomBytes } from 'crypto'
 import { Database, parseCID } from './database.js'
 import { Listener } from './listener.js'
 import { DbIndex as Index } from './db-index.js'
 // import { TransactionBlockstore } from './blockstore.js'
-import { localGet } from './utils.js'
+import { Loader } from './loader.js'
 import { Sync } from './sync.js'
 
 // todo remove Listener in 0.7.0
@@ -21,8 +22,8 @@ export class Fireproof {
   static storage = (name = null, opts = {}) => {
     if (name) {
       opts.name = name
-      // todo this can come from a registry also eg remote database / config, etc
-      const existing = localGet('fp.' + name)
+      const loader = new Loader(name, opts.loader)
+      const existing = loader.getHeader()
       if (existing) {
         const existingConfig = JSON.parse(existing)
         return Fireproof.fromConfig(name, existingConfig, opts)

package/src/loader.js

@@ -0,0 +1,168 @@
+import {
+  readFileSync
+  // createReadStream
+} from 'node:fs'
+import { mkdir, writeFile } from 'fs/promises'
+import { openDB } from 'idb'
+import { join, dirname } from 'path'
+// import { parse } from '@jsonlines/core'
+// import cargoQueue from 'async/cargoQueue.js'
+import { homedir } from 'os'
+
+const defaultConfig = {
+  dataDir: join(homedir(), '.fireproof'),
+  headerKeyPrefix: 'fp.'
+}
+
+const FORCE_IDB = typeof process !== 'undefined' && !!process.env?.FORCE_IDB
+
+/* global localStorage */
+
+export class Loader {
+  constructor (name, keyId, config = defaultConfig) {
+    this.name = name
+    this.keyId = keyId
+    this.config = config
+    this.isBrowser = false
+    try {
+      this.isBrowser = window.localStorage && true
+    } catch (e) {}
+  }
+
+  withDB = async (dbWorkFun) => {
+    if (!this.idb) {
+      this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 3, {
+        upgrade (db, oldVersion, newVersion, transaction) {
+          if (oldVersion < 1) {
+            db.createObjectStore('cars')
+          }
+        }
+      })
+    }
+    return await dbWorkFun(this.idb)
+  }
+
+  async writeCars (cars) {
+    // console.log('writeCars', this.config.dataDir, this.name, cars.map(c => c.cid.toString()))
+    // console.log('writeCars', cars.length)
+
+    if (FORCE_IDB || this.isBrowser) {
+      await this.writeCarsIDB(cars)
+    } else {
+      const writes = []
+      for (const { cid, bytes } of cars) {
+        const carFilename = join(this.config.dataDir, this.name, `${cid.toString()}.car`)
+        // console.log('writeCars', carFilename)
+        writes.push(writeSync(carFilename, bytes))
+      }
+      await Promise.all(writes)
+    }
+  }
+
+  async writeCarsIDB (cars) {
+    return await this.withDB(async db => {
+      const tx = db.transaction(['cars'], 'readwrite')
+      for (const { cid, bytes, replaces } of cars) {
+        await tx.objectStore('cars').put(bytes, cid.toString())
+        // todo remove old maps
+        if (replaces) {
+          await tx.objectStore('cars').delete(replaces.toString())
+        }
+      }
+      return await tx.done
+    })
+  }
+
+  async readCar (carCid) {
+    if (FORCE_IDB || this.isBrowser) {
+      return await this.readCarIDB(carCid)
+    } else {
+      const carFilename = join(this.config.dataDir, this.name, `${carCid.toString()}.car`)
+      const got = readFileSync(carFilename)
+      // console.log('readCar', carFilename, got.constructor.name)
+      return got
+    }
+  }
+
+  async readCarIDB (carCid) {
+    return await this.withDB(async db => {
+      const tx = db.transaction(['cars'], 'readonly')
+      // console.log('getCarReader', carCid)
+      return await tx.objectStore('cars').get(carCid)
+    })
+  }
+
+  getHeader () {
+    if (this.isBrowser) {
+      return localStorage.getItem(this.config.headerKeyPrefix + this.name)
+    } else {
+      return loadSync(this.headerFilename())
+      // return null
+    }
+  }
+
+  async saveHeader (stringValue) {
+    // console.log('saveHeader', this.isBrowser)
+    if (this.isBrowser) {
+      // console.log('localStorage!', this.config.headerKeyPrefix)
+      return localStorage.setItem(this.config.headerKeyPrefix + this.name, stringValue)
+    } else {
+      // console.log('no localStorage', this.config.dataDir, this.name)
+      // console.log('saving clock to', this.headerFilename(), stringValue)
+
+      try {
+        await writeSync(this.headerFilename(), stringValue)
+      } catch (error) {
+        console.log('error', error)
+      }
+
+      // console.log('saved clock to', this.headerFilename())
+    }
+  }
+
+  headerFilename () {
+    // console.log('headerFilename', this.config.dataDir, this.name)
+    return join(this.config.dataDir, this.name, 'header.json')
+  }
+
+  //   async loadData (database, filename) {
+  //     const fullFilePath = join(process.cwd(), filename)
+  //     const readableStream = createReadStream(fullFilePath)
+  //     const parseStream = parse()
+  //     readableStream.pipe(parseStream)
+
+  //     const saveQueue = cargoQueue(async (tasks, callback) => {
+  //       for (const t of tasks) {
+  //         await database.put(t)
+  //       }
+  //       callback()
+  //     })
+
+//     parseStream.on('data', async (data) => {
+//       saveQueue.push(data)
+//     })
+//     let res
+//     const p = new Promise((resolve, reject) => {
+//       res = resolve
+//     })
+//     saveQueue.drain(async (x) => {
+//       res()
+//     })
+//     return p
+//   }
+}
+
+function loadSync (filename) {
+  try {
+    return readFileSync(filename, 'utf8').toString()
+  } catch (error) {
+    // console.log('error', error)
+    return null
+  }
+}
+
+async function writeSync (fullpath, stringValue) {
+  await mkdir(dirname(fullpath), { recursive: true })
+  // writeFileSync(fullpath, stringValue)
+  await writeFile(fullpath, stringValue)
+}

package/src/prolly.js

@@ -15,6 +15,8 @@ import { nocache as cache } from 'prolly-trees/cache'
 import { CIDCounter, bf, simpleCompare as compare } from 'prolly-trees/utils'
 import * as codec from '@ipld/dag-cbor'
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
+// import { blake2b256 as hasher } from '@multiformats/blake2/blake2b'
+
 import { doTransaction } from './blockstore.js'
 import { create as createBlock } from 'multiformats/block'
 const blockOpts = { cache, chunker: bf(30), codec, hasher, compare }