@fireproof/core 0.6.3-dev2 → 0.6.4

package/dist/src/prolly.js

@@ -8,7 +8,6 @@ import { nocache as cache } from 'prolly-trees/cache';
 import { CIDCounter, bf, simpleCompare as compare } from 'prolly-trees/utils';
 import * as codec from '@ipld/dag-cbor';
 import { sha256 as hasher } from 'multiformats/hashes/sha2';
-// import { blake2b256 as hasher } from '@multiformats/blake2/blake2b'
 import { doTransaction } from './blockstore.js';
 import { create as createBlock } from 'multiformats/block';
 const blockOpts = { cache, chunker: bf(30), codec, hasher, compare };

package/dist/src/valet.js

@@ -6,8 +6,8 @@ import * as CBW from '@ipld/car/buffer-writer';
 import * as raw from 'multiformats/codecs/raw';
 import * as Block from 'multiformats/block';
 import * as dagcbor from '@ipld/dag-cbor';
+import { openDB } from 'idb';
 import cargoQueue from 'async/cargoQueue.js';
-import { Loader } from './loader.js';
 // @ts-ignore
 // @ts-ignore
 import { bf, simpleCompare as compare } from 'prolly-trees/utils';
@@ -44,7 +44,6 @@ export class Valet {
     constructor(name = 'default', keyMaterial) {
         this.name = name;
         this.setKeyMaterial(keyMaterial);
-        this.loader = new Loader(name, this.keyId); // todo send this config.loader, if we ever need it
         this.uploadQueue = cargoQueue(async (tasks, callback) => {
             // console.log(
             //   'queue worker',
@@ -77,9 +76,6 @@ export class Valet {
             //   })
         });
     }
-    saveHeader(header) {
-        return this.loader.saveHeader(header);
-    }
     getKeyMaterial() {
         return this.keyMaterial;
     }
@@ -120,6 +116,18 @@ export class Valet {
             throw new Error('missing lastCid for car header');
         }
     }
+    withDB = async (dbWorkFun) => {
+        if (!this.idb) {
+            this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 3, {
+                upgrade(db, oldVersion, newVersion, transaction) {
+                    if (oldVersion < 1) {
+                        db.createObjectStore('cars');
+                    }
+                }
+            });
+        }
+        return await dbWorkFun(this.idb);
+    };
     /**
      * Iterate over all blocks in the store.
      *
@@ -159,7 +167,6 @@ export class Valet {
                 blockHasher: blockOpts.hasher,
                 blockCodec: blockOpts.codec
             });
-            this.valetRoot = indexNode;
         }
         const got = await indexNode.get(cid);
         // console.log('getCarCIDForCID', cid, got)
@@ -204,14 +211,13 @@ export class Valet {
      * @param {*} value
      */
     async parkCar(carCid, value, cids) {
-        // const callId = Math.random().toString(36).substring(7)
         // console.log('parkCar', this.instanceId, this.name, carCid, cids)
         const combinedReader = await this.getCombinedReader(carCid);
         const mapNode = await addCidsToCarIndex(combinedReader, this.valetRoot, this.valetRootCid, Array.from(cids).map(cid => ({ key: cid.toString(), value: carCid.toString() })));
         this.valetRoot = mapNode;
         this.valetRootCid = mapNode.cid;
         // make a block set with all the cids of the map
-        const saveValetBlocks = new VMemoryBlockstore();
+        const saveValetBlocks = new VMemoryBlockstore(); //  todo this blockstore should read from the last valetCid car also
         for await (const cidx of mapNode.cids()) {
             const bytes = await combinedReader.get(cidx);
             saveValetBlocks.put(cidx, bytes);
@@ -224,8 +230,7 @@ export class Valet {
             newValetCidCar = await blocksToCarBlock(this.valetRootCid, saveValetBlocks);
         }
         // console.log('newValetCidCar', this.name, Math.floor(newValetCidCar.bytes.length / 1024))
-        // console.log('writeCars', callId, carCid.toString(), newValetCidCar.cid.toString())
-        await this.loader.writeCars([
+        await this.writeCars([
             {
                 cid: carCid,
                 bytes: value,
@@ -239,7 +244,6 @@ export class Valet {
             }
         ]);
         this.valetRootCarCid = newValetCidCar.cid; // goes to clock
-        // console.log('wroteCars', callId, carCid.toString(), newValetCidCar.cid.toString())
         // console.log('parked car', carCid, value.length, Array.from(cids))
         // upload to web3.storage if we have credentials
         if (this.uploadFunction) {
@@ -256,12 +260,27 @@ export class Valet {
             // console.log('no upload function', carCid, value.length, this.uploadFunction)
         }
     }
+    async writeCars(cars) {
+        return await this.withDB(async (db) => {
+            const tx = db.transaction(['cars'], 'readwrite');
+            for (const { cid, bytes, replaces } of cars) {
+                await tx.objectStore('cars').put(bytes, cid.toString());
+                // todo remove old maps
+                if (replaces) {
+                    await tx.objectStore('cars').delete(replaces.toString());
+                }
+            }
+            return await tx.done;
+        });
+    }
     remoteBlockFunction = null;
     async getCarReader(carCid) {
         carCid = carCid.toString();
-        const carBytes = await this.loader.readCar(carCid);
-        // const callID = Math.random().toString(36).substring(7)
-        // console.log('getCarReader', callID, carCid)
+        const carBytes = await this.withDB(async (db) => {
+            const tx = db.transaction(['cars'], 'readonly');
+            // console.log('getCarReader', carCid)
+            return await tx.objectStore('cars').get(carCid);
+        });
         const reader = await CarReader.fromBytes(carBytes);
         if (this.keyMaterial) {
             const roots = await reader.getRoots();
@@ -284,7 +303,6 @@ export class Valet {
             const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial);
             // last block is the root ??? todo
             const rootBlock = blocks[blocks.length - 1];
-            // console.log('got reader', callID, carCid)
             return {
                 root: rootBlock,
                 get: async (dataCID) => {
@@ -406,8 +424,6 @@ const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
 };
 const addCidsToCarIndex = async (blockstore, valetRoot, valetRootCid, bulkOperations) => {
     let indexNode;
-    // const callID = Math.random().toString(32).substring(2, 8)
-    // console.log('addCidsToCarIndex', callID, valetRootCid, bulkOperations.length)
     if (valetRootCid) {
         if (valetRoot) {
             indexNode = valetRoot;
@@ -429,7 +445,6 @@ const addCidsToCarIndex = async (blockstore, valetRoot, valetRootCid, bulkOperat
         // console.log('adding', key, value)
         await indexNode.set(key, value);
     }
-    // console.log('newCidsToCarIndex', callID, indexNode.cid, bulkOperations.length)
     return indexNode;
 };
 export class VMemoryBlockstore {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.6.3-dev2",
+  "version": "0.6.4",
   "description": "Live data for React, accelerated by proofs, powered by IPFS",
   "main": "dist/src/fireproof.js",
   "module": "dist/src/fireproof.mjs",
@@ -40,8 +40,6 @@
   "dependencies": {
     "@ipld/car": "^5.1.0",
     "@ipld/dag-cbor": "^9.0.0",
-    "@jsonlines/core": "^1.0.2",
-    "@multiformats/blake2": "^1.0.13",
     "@rollup/plugin-commonjs": "^24.0.1",
     "archy": "^1.0.0",
     "async": "^3.2.4",

package/src/database.js

@@ -2,6 +2,7 @@
 import { visMerkleClock, visMerkleTree, vis, put, get, getAll, eventsSince } from './prolly.js'
 import { doTransaction, TransactionBlockstore } from './blockstore.js'
 import charwise from 'charwise'
+import { localSet } from './utils.js'
 import { CID } from 'multiformats'
 
 // TypeScript Types
@@ -33,7 +34,7 @@ export class Database {
     this.name = name
     this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`
     this.blocks = new TransactionBlockstore(name, config.key)
-    this.indexBlocks = new TransactionBlockstore(name ? name + '.indexes' : null, config.key)
+    this.indexBlocks = new TransactionBlockstore(name + '.indexes', config.key)
     this.clock = clock
     this.config = config
   }
@@ -77,7 +78,7 @@ export class Database {
 
   maybeSaveClock () {
     if (this.name && this.blocks.valet) {
-      this.blocks.valet.saveHeader(JSON.stringify(this))
+      localSet('fp.' + this.name, JSON.stringify(this))
     }
   }
 
@@ -313,7 +314,6 @@ export class Database {
       console.error('failed', event)
       throw new Error('failed to put at storage layer')
     }
-    // await new Promise(resolve => setTimeout(resolve, 10)) // makes concurrent tests work
     this.applyClock(prevClock, result.head)
     await this.notifyListeners([decodedEvent]) // this type is odd
     return {

package/src/fireproof.js

@@ -1,10 +1,9 @@
 import randomBytes from 'randombytes'
-// import { randomBytes } from 'crypto'
 import { Database, parseCID } from './database.js'
 import { Listener } from './listener.js'
 import { DbIndex as Index } from './db-index.js'
 // import { TransactionBlockstore } from './blockstore.js'
-import { Loader } from './loader.js'
+import { localGet } from './utils.js'
 import { Sync } from './sync.js'
 
 // todo remove Listener in 0.7.0
@@ -22,8 +21,8 @@ export class Fireproof {
   static storage = (name = null, opts = {}) => {
     if (name) {
       opts.name = name
-      const loader = new Loader(name, opts.loader)
-      const existing = loader.getHeader()
+      // todo this can come from a registry also eg remote database / config, etc
+      const existing = localGet('fp.' + name)
       if (existing) {
         const existingConfig = JSON.parse(existing)
         return Fireproof.fromConfig(name, existingConfig, opts)

package/src/prolly.js

@@ -15,8 +15,6 @@ import { nocache as cache } from 'prolly-trees/cache'
 import { CIDCounter, bf, simpleCompare as compare } from 'prolly-trees/utils'
 import * as codec from '@ipld/dag-cbor'
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
-// import { blake2b256 as hasher } from '@multiformats/blake2/blake2b'
-
 import { doTransaction } from './blockstore.js'
 import { create as createBlock } from 'multiformats/block'
 const blockOpts = { cache, chunker: bf(30), codec, hasher, compare }

package/src/utils.js

@@ -0,0 +1,16 @@
+
+/* global localStorage */
+let storageSupported = false
+try {
+  storageSupported = window.localStorage && true
+} catch (e) {}
+export function localGet (key) {
+  if (storageSupported) {
+    return localStorage && localStorage.getItem(key)
+  }
+}
+export function localSet (key, value) {
+  if (storageSupported) {
+    return localStorage && localStorage.setItem(key, value)
+  }
+}

package/src/valet.js

@@ -6,9 +6,8 @@ import * as CBW from '@ipld/car/buffer-writer'
 import * as raw from 'multiformats/codecs/raw'
 import * as Block from 'multiformats/block'
 import * as dagcbor from '@ipld/dag-cbor'
+import { openDB } from 'idb'
 import cargoQueue from 'async/cargoQueue.js'
-import { Loader } from './loader.js'
-
 // @ts-ignore
 
 // @ts-ignore
@@ -54,7 +53,6 @@ export class Valet {
   constructor (name = 'default', keyMaterial) {
     this.name = name
     this.setKeyMaterial(keyMaterial)
-    this.loader = new Loader(name, this.keyId) // todo send this config.loader, if we ever need it
     this.uploadQueue = cargoQueue(async (tasks, callback) => {
       // console.log(
       //   'queue worker',
@@ -89,10 +87,6 @@ export class Valet {
     })
   }
 
-  saveHeader (header) {
-    return this.loader.saveHeader(header)
-  }
-
   getKeyMaterial () {
     return this.keyMaterial
   }
@@ -133,6 +127,19 @@ export class Valet {
     }
   }
 
+  withDB = async dbWorkFun => {
+    if (!this.idb) {
+      this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 3, {
+        upgrade (db, oldVersion, newVersion, transaction) {
+          if (oldVersion < 1) {
+            db.createObjectStore('cars')
+          }
+        }
+      })
+    }
+    return await dbWorkFun(this.idb)
+  }
+
   /**
    * Iterate over all blocks in the store.
    *
@@ -173,7 +180,6 @@ export class Valet {
         blockHasher: blockOpts.hasher,
         blockCodec: blockOpts.codec
       })
-      this.valetRoot = indexNode
     }
 
     const got = await indexNode.get(cid)
@@ -220,7 +226,6 @@ export class Valet {
    * @param {*} value
    */
   async parkCar (carCid, value, cids) {
-    // const callId = Math.random().toString(36).substring(7)
     // console.log('parkCar', this.instanceId, this.name, carCid, cids)
     const combinedReader = await this.getCombinedReader(carCid)
     const mapNode = await addCidsToCarIndex(
@@ -233,7 +238,7 @@ export class Valet {
     this.valetRoot = mapNode
     this.valetRootCid = mapNode.cid
     // make a block set with all the cids of the map
-    const saveValetBlocks = new VMemoryBlockstore()
+    const saveValetBlocks = new VMemoryBlockstore() //  todo this blockstore should read from the last valetCid car also
 
     for await (const cidx of mapNode.cids()) {
       const bytes = await combinedReader.get(cidx)
@@ -246,8 +251,7 @@ export class Valet {
       newValetCidCar = await blocksToCarBlock(this.valetRootCid, saveValetBlocks)
     }
     // console.log('newValetCidCar', this.name, Math.floor(newValetCidCar.bytes.length / 1024))
-    // console.log('writeCars', callId, carCid.toString(), newValetCidCar.cid.toString())
-    await this.loader.writeCars([
+    await this.writeCars([
       {
         cid: carCid,
         bytes: value,
@@ -263,8 +267,6 @@ export class Valet {
 
     this.valetRootCarCid = newValetCidCar.cid // goes to clock
 
-    // console.log('wroteCars', callId, carCid.toString(), newValetCidCar.cid.toString())
-
     // console.log('parked car', carCid, value.length, Array.from(cids))
     // upload to web3.storage if we have credentials
     if (this.uploadFunction) {
@@ -281,13 +283,29 @@ export class Valet {
     }
   }
 
+  async writeCars (cars) {
+    return await this.withDB(async db => {
+      const tx = db.transaction(['cars'], 'readwrite')
+      for (const { cid, bytes, replaces } of cars) {
+        await tx.objectStore('cars').put(bytes, cid.toString())
+        // todo remove old maps
+        if (replaces) {
+          await tx.objectStore('cars').delete(replaces.toString())
+        }
+      }
+      return await tx.done
+    })
+  }
+
   remoteBlockFunction = null
 
   async getCarReader (carCid) {
     carCid = carCid.toString()
-    const carBytes = await this.loader.readCar(carCid)
-    // const callID = Math.random().toString(36).substring(7)
-    // console.log('getCarReader', callID, carCid)
+    const carBytes = await this.withDB(async db => {
+      const tx = db.transaction(['cars'], 'readonly')
+      // console.log('getCarReader', carCid)
+      return await tx.objectStore('cars').get(carCid)
+    })
     const reader = await CarReader.fromBytes(carBytes)
     if (this.keyMaterial) {
       const roots = await reader.getRoots()
@@ -311,7 +329,7 @@ export class Valet {
 
       // last block is the root ??? todo
       const rootBlock = blocks[blocks.length - 1]
-      // console.log('got reader', callID, carCid)
+
       return {
         root: rootBlock,
         get: async dataCID => {
@@ -438,8 +456,6 @@ const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
 
 const addCidsToCarIndex = async (blockstore, valetRoot, valetRootCid, bulkOperations) => {
   let indexNode
-  // const callID = Math.random().toString(32).substring(2, 8)
-  // console.log('addCidsToCarIndex', callID, valetRootCid, bulkOperations.length)
   if (valetRootCid) {
     if (valetRoot) {
       indexNode = valetRoot
@@ -459,7 +475,6 @@ const addCidsToCarIndex = async (blockstore, valetRoot, valetRootCid, bulkOperat
     // console.log('adding', key, value)
     await indexNode.set(key, value)
   }
-  // console.log('newCidsToCarIndex', callID, indexNode.cid, bulkOperations.length)
   return indexNode
 }
 

package/src/loader.js

@@ -1,168 +0,0 @@
-import {
-  readFileSync
-  // createReadStream
-} from 'node:fs'
-import { mkdir, writeFile } from 'node:fs/promises'
-import { openDB } from 'idb'
-import { join, dirname } from 'path'
-// import { parse } from '@jsonlines/core'
-// import cargoQueue from 'async/cargoQueue.js'
-import { homedir } from 'os'
-
-const defaultConfig = {
-  dataDir: join(homedir(), '.fireproof'),
-  headerKeyPrefix: 'fp.'
-}
-
-const FORCE_IDB = typeof process !== 'undefined' && !!process.env?.FORCE_IDB
-
-/* global localStorage */
-
-export class Loader {
-  constructor (name, keyId, config = defaultConfig) {
-    this.name = name
-    this.keyId = keyId
-    this.config = config
-    this.isBrowser = false
-    try {
-      this.isBrowser = window.localStorage && true
-    } catch (e) {}
-  }
-
-  withDB = async (dbWorkFun) => {
-    if (!this.idb) {
-      this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 3, {
-        upgrade (db, oldVersion, newVersion, transaction) {
-          if (oldVersion < 1) {
-            db.createObjectStore('cars')
-          }
-        }
-      })
-    }
-    return await dbWorkFun(this.idb)
-  }
-
-  async writeCars (cars) {
-    // console.log('writeCars', this.config.dataDir, this.name, cars.map(c => c.cid.toString()))
-    // console.log('writeCars', cars.length)
-
-    if (FORCE_IDB || this.isBrowser) {
-      await this.writeCarsIDB(cars)
-    } else {
-      const writes = []
-      for (const { cid, bytes } of cars) {
-        const carFilename = join(this.config.dataDir, this.name, `${cid.toString()}.car`)
-        // console.log('writeCars', carFilename)
-        writes.push(writeSync(carFilename, bytes))
-      }
-      await Promise.all(writes)
-    }
-  }
-
-  async writeCarsIDB (cars) {
-    return await this.withDB(async db => {
-      const tx = db.transaction(['cars'], 'readwrite')
-      for (const { cid, bytes, replaces } of cars) {
-        await tx.objectStore('cars').put(bytes, cid.toString())
-        // todo remove old maps
-        if (replaces) {
-          await tx.objectStore('cars').delete(replaces.toString())
-        }
-      }
-      return await tx.done
-    })
-  }
-
-  async readCar (carCid) {
-    if (FORCE_IDB || this.isBrowser) {
-      return await this.readCarIDB(carCid)
-    } else {
-      const carFilename = join(this.config.dataDir, this.name, `${carCid.toString()}.car`)
-      const got = readFileSync(carFilename)
-      // console.log('readCar', carFilename, got.constructor.name)
-      return got
-    }
-  }
-
-  async readCarIDB (carCid) {
-    return await this.withDB(async db => {
-      const tx = db.transaction(['cars'], 'readonly')
-      // console.log('getCarReader', carCid)
-      return await tx.objectStore('cars').get(carCid)
-    })
-  }
-
-  getHeader () {
-    if (this.isBrowser) {
-      return localStorage.getItem(this.config.headerKeyPrefix + this.name)
-    } else {
-      return loadSync(this.headerFilename())
-      // return null
-    }
-  }
-
-  async saveHeader (stringValue) {
-    // console.log('saveHeader', this.isBrowser)
-    if (this.isBrowser) {
-      // console.log('localStorage!', this.config.headerKeyPrefix)
-      return localStorage.setItem(this.config.headerKeyPrefix + this.name, stringValue)
-    } else {
-      // console.log('no localStorage', this.config.dataDir, this.name)
-      // console.log('saving clock to', this.headerFilename(), stringValue)
-
-      try {
-        await writeSync(this.headerFilename(), stringValue)
-      } catch (error) {
-        console.log('error', error)
-      }
-
-      // console.log('saved clock to', this.headerFilename())
-    }
-  }
-
-  headerFilename () {
-    // console.log('headerFilename', this.config.dataDir, this.name)
-    return join(this.config.dataDir, this.name, 'header.json')
-  }
-
-  //   async loadData (database, filename) {
-  //     const fullFilePath = join(process.cwd(), filename)
-  //     const readableStream = createReadStream(fullFilePath)
-  //     const parseStream = parse()
-  //     readableStream.pipe(parseStream)
-
-  //     const saveQueue = cargoQueue(async (tasks, callback) => {
-  //       for (const t of tasks) {
-  //         await database.put(t)
-  //       }
-  //       callback()
-  //     })
-
-//     parseStream.on('data', async (data) => {
-//       saveQueue.push(data)
-//     })
-//     let res
-//     const p = new Promise((resolve, reject) => {
-//       res = resolve
-//     })
-//     saveQueue.drain(async (x) => {
-//       res()
-//     })
-//     return p
-//   }
-}
-
-function loadSync (filename) {
-  try {
-    return readFileSync(filename, 'utf8').toString()
-  } catch (error) {
-    // console.log('error', error)
-    return null
-  }
-}
-
-async function writeSync (fullpath, stringValue) {
-  await mkdir(dirname(fullpath), { recursive: true })
-  // writeFileSync(fullpath, stringValue)
-  await writeFile(fullpath, stringValue)
-}