@fireproof/core 0.5.19 → 0.5.21

package/dist/src/prolly.js

@@ -153,12 +153,6 @@ const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
         return root;
     }
 };
-// async function bigMerge (events, head, getBlock) {
-//   const allRoots = await Promise.all(head.map(async h => prollyRootFromAncestor(events, h, getBlock)))
-//   console.log('allRoots', allRoots)
-//   // todo query over all roots and merge them, but how do they not have a common ancestor? they all start with the _sync root
-//   throw new Error('not implemented')
-// }
 const doProllyBulk = async (inBlocks, head, event, doFull = false) => {
     const { getBlock, blocks } = makeGetAndPutBlock(inBlocks); // this is doubled with eventfetcher
     let bulkSorted = [];
@@ -316,8 +310,6 @@ export async function eventsSince(blocks, head, since) {
  *
  */
 export async function getAll(blocks, head, rootCache = null, doFull = false) {
-    // todo use the root node left around from put, etc
-    // move load to a central place
     if (!head.length) {
         return { root: null, clockCIDs: new CIDCounter(), cids: new CIDCounter(), result: [] };
     }

package/dist/src/sync.js

@@ -153,7 +153,7 @@ export class Sync {
         if (!this.peer || !this.isReady)
             return;
         // console.log('send update from', this.database.instanceId)
-        // todo should send updates since last sync
+        // todo should send updates since last sync (currently sends each transaction)
         const newCar = await blocksToCarBlock(blockstore.lastCid, blockstore);
         this.status = 'sending update car';
         this.peer.send(newCar.bytes);

package/dist/src/valet.js

@@ -52,28 +52,28 @@ export class Valet {
             // )
             if (this.uploadFunction) {
                 // todo we can coalesce these into a single car file
-                return await this.withDB(async (db) => {
-                    for (const task of tasks) {
-                        await this.uploadFunction(task.carCid, task.value);
-                        // update the indexedb to mark this car as no longer pending
-                        const carMeta = await db.get('cidToCar', task.carCid);
-                        delete carMeta.pending;
-                        await db.put('cidToCar', carMeta);
-                    }
-                });
+                // todo remove idb usage here
+                for (const task of tasks) {
+                    await this.uploadFunction(task.carCid, task.value);
+                    // todo update syncCidMap to say this has been synced
+                    // const carMeta = await db.get('cidToCar', task.carCid)
+                    // delete carMeta.pending
+                    // await db.put('cidToCar', carMeta)
+                }
             }
             callback();
         });
         this.uploadQueue.drain(async () => {
-            return await this.withDB(async (db) => {
-                const carKeys = (await db.getAllFromIndex('cidToCar', 'pending')).map(c => c.car);
-                for (const carKey of carKeys) {
-                    await this.uploadFunction(carKey, await db.get('cars', carKey));
-                    const carMeta = await db.get('cidToCar', carKey);
-                    delete carMeta.pending;
-                    await db.put('cidToCar', carMeta);
-                }
-            });
+            // todo read syncCidMap and sync any that are still unsynced
+            //   return await this.withDB(async db => {
+            //     const carKeys = (await db.getAllFromIndex('cidToCar', 'pending')).map(c => c.car)
+            //     for (const carKey of carKeys) {
+            //       await this.uploadFunction(carKey, await db.get('cars', carKey))
+            //       const carMeta = await db.get('cidToCar', carKey)
+            //       delete carMeta.pending
+            //       await db.put('cidToCar', carMeta)
+            //     }
+            //   })
         });
     }
     getKeyMaterial() {
@@ -118,16 +118,10 @@ export class Valet {
     }
     withDB = async (dbWorkFun) => {
         if (!this.idb) {
-            this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 2, {
+            this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 3, {
                 upgrade(db, oldVersion, newVersion, transaction) {
                     if (oldVersion < 1) {
-                        db.createObjectStore('cars'); // todo use database name
-                        const cidToCar = db.createObjectStore('cidToCar', { keyPath: 'car' });
-                        cidToCar.createIndex('cids', 'cids', { multiEntry: true });
-                    }
-                    if (oldVersion < 2) {
-                        const cidToCar = transaction.objectStore('cidToCar');
-                        cidToCar.createIndex('pending', 'pending');
+                        db.createObjectStore('cars');
                     }
                 }
             });
@@ -142,23 +136,22 @@ export class Valet {
      */
     async *cids() {
         // console.log('valet cids')
-        const db = await this.withDB(async (db) => db);
-        const tx = db.transaction(['cidToCar'], 'readonly');
-        let cursor = await tx.store.openCursor();
-        while (cursor) {
-            yield { cid: cursor.key, car: cursor.value.car };
-            cursor = await cursor.continue();
-        }
+        // todo use cidMap
+        // while (cursor) {
+        // yield { cid: cursor.key, car: cursor.value.car }
+        // cursor = await cursor.continue()
+        // }
     }
     setRootCarCid(cid) {
         this.valetRootCarCid = cid;
         this.valetRoot = null;
         this.valetRootCid = null;
     }
+    // todo memoize this
     async getCarCIDForCID(cid) {
         // make a car reader for this.valetRootCarCid
         if (!this.valetRootCarCid)
-            return;
+            return { result: null };
         let indexNode;
         if (this.valetRoot) {
             indexNode = this.valetRoot;
@@ -179,14 +172,6 @@ export class Valet {
         // console.log('getCarCIDForCID', cid, got)
         return { result: got };
     }
-    async OLDgetCarCIDForCID(cid) {
-        const carCid = await this.withDB(async (db) => {
-            const tx = db.transaction(['cars', 'cidToCar'], 'readonly');
-            const indexResp = await tx.objectStore('cidToCar').index('cids').get(cid);
-            return indexResp?.car;
-        });
-        return { result: carCid };
-    }
     async getCombinedReader(carCid) {
         let carMapReader;
         if (this.valetRootCarCid) {
@@ -245,17 +230,19 @@ export class Valet {
             newValetCidCar = await blocksToCarBlock(this.valetRootCid, saveValetBlocks);
         }
         // console.log('newValetCidCar', this.name, Math.floor(newValetCidCar.bytes.length / 1024))
-        await this.withDB(async (db) => {
-            const tx = db.transaction(['cars'], 'readwrite');
-            await tx.objectStore('cars').put(value, carCid.toString());
-            if (newValetCidCar) {
-                if (this.valetRootCarCid) {
-                    // await tx.objectStore('cars').delete(this.valetRootCarCid.toString())
-                }
-                await tx.objectStore('cars').put(newValetCidCar.bytes, newValetCidCar.cid.toString());
+        await this.writeCars([
+            {
+                cid: carCid,
+                bytes: value,
+                replaces: null
+            },
+            {
+                cid: newValetCidCar.cid,
+                bytes: newValetCidCar.bytes,
+                replaces: null
+                // replaces: this.valetRootCarCid // todo
             }
-            return await tx.done;
-        });
+        ]);
         this.valetRootCarCid = newValetCidCar.cid; // goes to clock
         // console.log('parked car', carCid, value.length, Array.from(cids))
         // upload to web3.storage if we have credentials
@@ -273,6 +260,19 @@ export class Valet {
             // console.log('no upload function', carCid, value.length, this.uploadFunction)
         }
     }
+    async writeCars(cars) {
+        return await this.withDB(async (db) => {
+            const tx = db.transaction(['cars'], 'readwrite');
+            for (const { cid, bytes, replaces } of cars) {
+                await tx.objectStore('cars').put(bytes, cid.toString());
+                // todo remove old maps
+                if (replaces) {
+                    await tx.objectStore('cars').delete(replaces.toString());
+                }
+            }
+            return await tx.done;
+        });
+    }
     remoteBlockFunction = null;
     async getCarReader(carCid) {
         carCid = carCid.toString();
@@ -301,7 +301,7 @@ export class Valet {
                 return decoded;
             };
             const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial);
-            // last block is the root ???
+            // last block is the root ??? todo
             const rootBlock = blocks[blocks.length - 1];
             return {
                 root: rootBlock,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.5.19",
+  "version": "0.5.21",
   "description": "Live data for your React app, powered by IPFS",
   "main": "dist/src/fireproof.js",
   "module": "dist/src/fireproof.mjs",

package/src/blockstore.js

@@ -91,7 +91,6 @@ export class TransactionBlockstore {
 
   async networkGet (key) {
     if (this.remoteBlockFunction) {
-      // todo why is this on valet?
       const value = await husher(key, async () => await this.remoteBlockFunction(key))
       if (value) {
         // console.log('networkGot: ' + key, value.length)

package/src/database.js

@@ -46,7 +46,6 @@ export class Database {
    * @instance
    */
   toJSON () {
-    // todo this also needs to return the index roots...
     return {
       clock: this.clockToJSON(),
       name: this.name,
@@ -183,8 +182,7 @@ export class Database {
     const cids = await cidsToProof(allResp.cids)
     const clockCids = await cidsToProof(allResp.clockCIDs)
     // console.log('allcids', cids, clockCids)
-    // todo we need to put the clock head as the last block in the encrypted car
-    return [...cids, ...clockCids] // need a single block version of clock head, maybe an encoded block for it
+    return [...cids, ...clockCids] // clock CID last -- need to handle multiple entry clocks
   }
 
   async allStoredCIDs () {
@@ -226,7 +224,7 @@ export class Database {
     const clock = opts.clock || this.clock
     const resp = await get(this.blocks, clock, charwise.encode(key), this.rootCache)
     this.rootCache = { root: resp.root, clockCIDs: resp.clockCIDs }
-    // this tombstone is temporary until we can get the prolly tree to delete
+    // ? this tombstone is temporary until we can get the prolly tree to delete
     if (!resp || resp.result === null) {
       throw new Error('Not found')
     }

package/src/db-index.js

@@ -306,7 +306,7 @@ export class DbIndex {
   async innerUpdateIndex (inBlocks) {
     // const callTag = Math.random().toString(36).substring(4)
     // console.log(`updateIndex ${callTag} >`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
-    // todo remove this hack
+    // todo remove this hack in 0.7.0
     if (ALWAYS_REBUILD) {
       this.indexById = { root: null, cid: null }
       this.indexByKey = { root: null, cid: null }
@@ -353,9 +353,7 @@ export class DbIndex {
       )
       this.indexByKey = await bulkIndex(blocks, this.indexByKey, oldIndexEntries.concat(indexEntries), dbIndexOpts)
       this.dbHead = result.clock
-    }, false /* don't sync transaction -- maybe move this flag to database.indexBlocks? */)
-    // todo index subscriptions
-    // this.database.notifyExternal('dbIndex')
+    }, false /* don't sync transaction -- todo move this flag to database.indexBlocks, and concept of sync channels */)
     // console.timeEnd(callTag + '.doTransactionupdateIndex')
     // console.log(`updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
     return didT

package/src/fireproof.js

@@ -6,6 +6,7 @@ import { DbIndex as Index } from './db-index.js'
 import { localGet } from './utils.js'
 import { Sync } from './sync.js'
 
+// todo remove Listener in 0.7.0
 export { Index, Listener, Database, Sync }
 
 export class Fireproof {
@@ -20,7 +21,7 @@ export class Fireproof {
   static storage = (name = null, opts = {}) => {
     if (name) {
       opts.name = name
-      // todo this can come from a registry also
+      // todo this can come from a registry also eg remote database / config, etc
       const existing = localGet('fp.' + name)
       if (existing) {
         const existingConfig = JSON.parse(existing)

package/src/prolly.js

@@ -172,13 +172,6 @@ const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
   }
 }
 
-// async function bigMerge (events, head, getBlock) {
-//   const allRoots = await Promise.all(head.map(async h => prollyRootFromAncestor(events, h, getBlock)))
-//   console.log('allRoots', allRoots)
-//   // todo query over all roots and merge them, but how do they not have a common ancestor? they all start with the _sync root
-//   throw new Error('not implemented')
-// }
-
 const doProllyBulk = async (inBlocks, head, event, doFull = false) => {
   const { getBlock, blocks } = makeGetAndPutBlock(inBlocks) // this is doubled with eventfetcher
   let bulkSorted = []
@@ -347,8 +340,6 @@ export async function eventsSince (blocks, head, since) {
  *
  */
 export async function getAll (blocks, head, rootCache = null, doFull = false) {
-  // todo use the root node left around from put, etc
-  // move load to a central place
   if (!head.length) {
     return { root: null, clockCIDs: new CIDCounter(), cids: new CIDCounter(), result: [] }
   }

package/src/sync.js

@@ -162,7 +162,7 @@ export class Sync {
   async sendUpdate (blockstore) {
     if (!this.peer || !this.isReady) return
     // console.log('send update from', this.database.instanceId)
-    // todo should send updates since last sync
+    // todo should send updates since last sync (currently sends each transaction)
     const newCar = await blocksToCarBlock(blockstore.lastCid, blockstore)
     this.status = 'sending update car'
     this.peer.send(newCar.bytes)

package/src/valet.js

@@ -61,29 +61,29 @@ export class Valet {
       // )
       if (this.uploadFunction) {
         // todo we can coalesce these into a single car file
-        return await this.withDB(async db => {
-          for (const task of tasks) {
-            await this.uploadFunction(task.carCid, task.value)
-            // update the indexedb to mark this car as no longer pending
-            const carMeta = await db.get('cidToCar', task.carCid)
-            delete carMeta.pending
-            await db.put('cidToCar', carMeta)
-          }
-        })
+        // todo remove idb usage here
+        for (const task of tasks) {
+          await this.uploadFunction(task.carCid, task.value)
+          // todo update syncCidMap to say this has been synced
+          // const carMeta = await db.get('cidToCar', task.carCid)
+          // delete carMeta.pending
+          // await db.put('cidToCar', carMeta)
+        }
       }
       callback()
     })
 
     this.uploadQueue.drain(async () => {
-      return await this.withDB(async db => {
-        const carKeys = (await db.getAllFromIndex('cidToCar', 'pending')).map(c => c.car)
-        for (const carKey of carKeys) {
-          await this.uploadFunction(carKey, await db.get('cars', carKey))
-          const carMeta = await db.get('cidToCar', carKey)
-          delete carMeta.pending
-          await db.put('cidToCar', carMeta)
-        }
-      })
+      // todo read syncCidMap and sync any that are still unsynced
+      //   return await this.withDB(async db => {
+      //     const carKeys = (await db.getAllFromIndex('cidToCar', 'pending')).map(c => c.car)
+      //     for (const carKey of carKeys) {
+      //       await this.uploadFunction(carKey, await db.get('cars', carKey))
+      //       const carMeta = await db.get('cidToCar', carKey)
+      //       delete carMeta.pending
+      //       await db.put('cidToCar', carMeta)
+      //     }
+      //   })
     })
   }
 
@@ -129,16 +129,10 @@ export class Valet {
 
   withDB = async dbWorkFun => {
     if (!this.idb) {
-      this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 2, {
+      this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 3, {
         upgrade (db, oldVersion, newVersion, transaction) {
           if (oldVersion < 1) {
-            db.createObjectStore('cars') // todo use database name
-            const cidToCar = db.createObjectStore('cidToCar', { keyPath: 'car' })
-            cidToCar.createIndex('cids', 'cids', { multiEntry: true })
-          }
-          if (oldVersion < 2) {
-            const cidToCar = transaction.objectStore('cidToCar')
-            cidToCar.createIndex('pending', 'pending')
+            db.createObjectStore('cars')
           }
         }
       })
@@ -154,13 +148,11 @@ export class Valet {
    */
   async * cids () {
     // console.log('valet cids')
-    const db = await this.withDB(async db => db)
-    const tx = db.transaction(['cidToCar'], 'readonly')
-    let cursor = await tx.store.openCursor()
-    while (cursor) {
-      yield { cid: cursor.key, car: cursor.value.car }
-      cursor = await cursor.continue()
-    }
+    // todo use cidMap
+    // while (cursor) {
+    // yield { cid: cursor.key, car: cursor.value.car }
+    // cursor = await cursor.continue()
+    // }
   }
 
   setRootCarCid (cid) {
@@ -169,9 +161,10 @@ export class Valet {
     this.valetRootCid = null
   }
 
+  // todo memoize this
   async getCarCIDForCID (cid) {
     // make a car reader for this.valetRootCarCid
-    if (!this.valetRootCarCid) return
+    if (!this.valetRootCarCid) return { result: null }
 
     let indexNode
     if (this.valetRoot) {
@@ -194,15 +187,6 @@ export class Valet {
     return { result: got }
   }
 
-  async OLDgetCarCIDForCID (cid) {
-    const carCid = await this.withDB(async db => {
-      const tx = db.transaction(['cars', 'cidToCar'], 'readonly')
-      const indexResp = await tx.objectStore('cidToCar').index('cids').get(cid)
-      return indexResp?.car
-    })
-    return { result: carCid }
-  }
-
   async getCombinedReader (carCid) {
     let carMapReader
     if (this.valetRootCarCid) {
@@ -267,17 +251,20 @@ export class Valet {
       newValetCidCar = await blocksToCarBlock(this.valetRootCid, saveValetBlocks)
     }
     // console.log('newValetCidCar', this.name, Math.floor(newValetCidCar.bytes.length / 1024))
-    await this.withDB(async db => {
-      const tx = db.transaction(['cars'], 'readwrite')
-      await tx.objectStore('cars').put(value, carCid.toString())
-      if (newValetCidCar) {
-        if (this.valetRootCarCid) {
-          // await tx.objectStore('cars').delete(this.valetRootCarCid.toString())
-        }
-        await tx.objectStore('cars').put(newValetCidCar.bytes, newValetCidCar.cid.toString())
+    await this.writeCars([
+      {
+        cid: carCid,
+        bytes: value,
+        replaces: null
+      },
+      {
+        cid: newValetCidCar.cid,
+        bytes: newValetCidCar.bytes,
+        replaces: null
+        // replaces: this.valetRootCarCid // todo
       }
-      return await tx.done
-    })
+    ])
+
     this.valetRootCarCid = newValetCidCar.cid // goes to clock
 
     // console.log('parked car', carCid, value.length, Array.from(cids))
@@ -296,6 +283,20 @@ export class Valet {
     }
   }
 
+  async writeCars (cars) {
+    return await this.withDB(async db => {
+      const tx = db.transaction(['cars'], 'readwrite')
+      for (const { cid, bytes, replaces } of cars) {
+        await tx.objectStore('cars').put(bytes, cid.toString())
+        // todo remove old maps
+        if (replaces) {
+          await tx.objectStore('cars').delete(replaces.toString())
+        }
+      }
+      return await tx.done
+    })
+  }
+
   remoteBlockFunction = null
 
   async getCarReader (carCid) {
@@ -326,7 +327,7 @@ export class Valet {
       }
       const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial)
 
-      // last block is the root ???
+      // last block is the root ??? todo
       const rootBlock = blocks[blocks.length - 1]
 
       return {