@fireproof/core 0.5.16 → 0.5.18

package/dist/src/sync.js

@@ -147,7 +147,7 @@ export class Sync {
     destroy() {
         this.database.blocks.syncs.delete(this);
         this.status = 'destroyed';
-        this.peer.destroy();
+        // this.peer.destroy()  todo
     }
     async sendUpdate(blockstore) {
         if (!this.peer || !this.isReady)

package/dist/src/valet.js

@@ -1,6 +1,7 @@
 import { CarReader } from '@ipld/car';
 import { CID } from 'multiformats/cid';
 import { sha256 } from 'multiformats/hashes/sha2';
+import { parse } from 'multiformats/link';
 import * as CBW from '@ipld/car/buffer-writer';
 import * as raw from 'multiformats/codecs/raw';
 import * as Block from 'multiformats/block';
@@ -8,15 +9,19 @@ import * as dagcbor from '@ipld/dag-cbor';
 import { openDB } from 'idb';
 import cargoQueue from 'async/cargoQueue.js';
 // @ts-ignore
-import { bf } from 'prolly-trees/utils';
+// @ts-ignore
+import { bf, simpleCompare as compare } from 'prolly-trees/utils';
 // @ts-ignore
 import { nocache as cache } from 'prolly-trees/cache';
+// import { makeGetBlock } from './prolly.js'
 import { encrypt, decrypt } from './crypto.js';
 import { Buffer } from 'buffer';
 // @ts-ignore
 import * as codec from 'encrypted-block';
+import { create, load } from 'ipld-hashmap';
 import { rawSha1 as sha1sync } from './sha1.js';
 const chunker = bf(30);
+const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare };
 const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT;
 // ? process.env.NO_ENCRYPT : import.meta && import.meta.env.VITE_NO_ENCRYPT
 export class Valet {
@@ -26,6 +31,11 @@ export class Valet {
     alreadyEnqueued = new Set();
     keyMaterial = null;
     keyId = 'null';
+    valetRoot = null;
+    valetRootCid = null; // set by hydrate
+    valetRootCarCid = null; // most recent diff
+    valetCidBlocks = new VMemoryBlockstore();
+    instanceId = Math.random().toString(36).slice(2);
     /**
      * Function installed by the database to upload car files
      * @type {null|function(string, Uint8Array):Promise<void>}
@@ -140,18 +150,113 @@ export class Valet {
             cursor = await cursor.continue();
         }
     }
+    setRootCarCid(cid) {
+        this.valetRootCarCid = cid;
+        this.valetRoot = null;
+        this.valetRootCid = null;
+    }
+    async getCarCIDForCID(cid) {
+        // make a car reader for this.valetRootCarCid
+        if (!this.valetRootCarCid)
+            return;
+        let indexNode;
+        if (this.valetRoot) {
+            indexNode = this.valetRoot;
+        }
+        else {
+            const combinedReader = await this.getCombinedReader(this.valetRootCarCid);
+            if (!this.valetRootCid) {
+                const root = combinedReader.root.cid;
+                // console.log('roots', this.instanceId, this.name, root, this.valetRootCarCid, this.valetRootCid)
+                this.valetRootCid = root;
+            }
+            indexNode = await load(combinedReader, this.valetRootCid, {
+                blockHasher: blockOpts.hasher,
+                blockCodec: blockOpts.codec
+            });
+        }
+        const got = await indexNode.get(cid);
+        // console.log('getCarCIDForCID', cid, got)
+        return { result: got };
+    }
+    async OLDgetCarCIDForCID(cid) {
+        const carCid = await this.withDB(async (db) => {
+            const tx = db.transaction(['cars', 'cidToCar'], 'readonly');
+            const indexResp = await tx.objectStore('cidToCar').index('cids').get(cid);
+            return indexResp?.car;
+        });
+        return { result: carCid };
+    }
+    async getCombinedReader(carCid) {
+        let carMapReader;
+        if (this.valetRootCarCid) {
+            // todo only need this if we are cold starting
+            carMapReader = await this.getCarReader(this.valetRootCarCid);
+        }
+        const theseValetCidBlocks = this.valetCidBlocks;
+        // console.log('theseValetCidBlocks', theseValetCidBlocks)
+        const combinedReader = {
+            root: carMapReader?.root,
+            put: async (cid, bytes) => {
+                // console.log('mapPut', cid, bytes.length)
+                return await theseValetCidBlocks.put(cid, bytes);
+            },
+            get: async (cid) => {
+                // console.log('mapGet', cid)
+                try {
+                    const got = await theseValetCidBlocks.get(cid);
+                    return got.bytes;
+                }
+                catch (e) {
+                    // console.log('get from car', cid, carMapReader)
+                    if (!carMapReader)
+                        throw e;
+                    const bytes = await carMapReader.get(cid);
+                    await theseValetCidBlocks.put(cid, bytes);
+                    // console.log('mapGet', cid, bytes.length, bytes.constructor.name)
+                    return bytes;
+                }
+            }
+        };
+        return combinedReader;
+    }
     /**
      *
      * @param {string} carCid
      * @param {*} value
      */
     async parkCar(carCid, value, cids) {
+        // console.log('parkCar', this.instanceId, this.name, carCid, cids)
+        const combinedReader = await this.getCombinedReader(carCid);
+        const mapNode = await addCidsToCarIndex(combinedReader, this.valetRoot, this.valetRootCid, Array.from(cids).map(cid => ({ key: cid.toString(), value: carCid.toString() })));
+        this.valetRoot = mapNode;
+        this.valetRootCid = mapNode.cid;
+        // make a block set with all the cids of the map
+        const saveValetBlocks = new VMemoryBlockstore(); //  todo this blockstore should read from the last valetCid car also
+        for await (const cidx of mapNode.cids()) {
+            const bytes = await combinedReader.get(cidx);
+            saveValetBlocks.put(cidx, bytes);
+        }
+        let newValetCidCar;
+        if (this.keyMaterial) {
+            newValetCidCar = await blocksToEncryptedCarBlock(this.valetRootCid, saveValetBlocks, this.keyMaterial);
+        }
+        else {
+            newValetCidCar = await blocksToCarBlock(this.valetRootCid, saveValetBlocks);
+        }
+        // console.log('newValetCidCar', this.name, Math.floor(newValetCidCar.bytes.length / 1024))
         await this.withDB(async (db) => {
-            const tx = db.transaction(['cars', 'cidToCar'], 'readwrite');
-            await tx.objectStore('cars').put(value, carCid);
-            await tx.objectStore('cidToCar').put({ pending: 'y', car: carCid, cids: Array.from(cids) });
+            const tx = db.transaction(['cars'], 'readwrite');
+            await tx.objectStore('cars').put(value, carCid.toString());
+            if (newValetCidCar) {
+                if (this.valetRootCarCid) {
+                    // await tx.objectStore('cars').delete(this.valetRootCarCid.toString())
+                }
+                await tx.objectStore('cars').put(newValetCidCar.bytes, newValetCidCar.cid.toString());
+            }
             return await tx.done;
         });
+        this.valetRootCarCid = newValetCidCar.cid; // goes to clock
         // console.log('parked car', carCid, value.length, Array.from(cids))
         // upload to web3.storage if we have credentials
         if (this.uploadFunction) {
@@ -169,49 +274,73 @@ export class Valet {
         }
     }
     remoteBlockFunction = null;
-    async getBlock(dataCID) {
-        return await this.withDB(async (db) => {
-            const tx = db.transaction(['cars', 'cidToCar'], 'readonly');
-            const indexResp = await tx.objectStore('cidToCar').index('cids').get(dataCID);
-            const carCid = indexResp?.car;
-            if (!carCid) {
-                throw new Error('Missing block: ' + dataCID);
-            }
-            const carBytes = await tx.objectStore('cars').get(carCid);
-            const reader = await CarReader.fromBytes(carBytes);
-            if (this.keyMaterial) {
-                const roots = await reader.getRoots();
-                const readerGetWithCodec = async (cid) => {
-                    const got = await reader.get(cid);
-                    // console.log('got.', cid.toString())
-                    let useCodec = codec;
-                    if (cid.toString().indexOf('bafy') === 0) {
-                        useCodec = dagcbor;
+    async getCarReader(carCid) {
+        carCid = carCid.toString();
+        const carBytes = await this.withDB(async (db) => {
+            const tx = db.transaction(['cars'], 'readonly');
+            // console.log('getCarReader', carCid)
+            return await tx.objectStore('cars').get(carCid);
+        });
+        const reader = await CarReader.fromBytes(carBytes);
+        if (this.keyMaterial) {
+            const roots = await reader.getRoots();
+            const readerGetWithCodec = async (cid) => {
+                const got = await reader.get(cid);
+                // console.log('got.', cid.toString())
+                let useCodec = codec;
+                if (cid.toString().indexOf('bafy') === 0) {
+                    // todo cleanup types
+                    useCodec = dagcbor;
+                }
+                const decoded = await Block.decode({
+                    ...got,
+                    codec: useCodec,
+                    hasher: sha256
+                });
+                // console.log('decoded', decoded.value)
+                return decoded;
+            };
+            const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial);
+            // last block is the root ???
+            const rootBlock = blocks[blocks.length - 1];
+            return {
+                root: rootBlock,
+                get: async (dataCID) => {
+                    // console.log('getCarReader dataCID', dataCID)
+                    dataCID = dataCID.toString();
+                    const block = blocks.find(b => b.cid.toString() === dataCID);
+                    // console.log('getCarReader block', block)
+                    if (block) {
+                        return block.bytes;
                     }
-                    const decoded = await Block.decode({
-                        ...got,
-                        codec: useCodec,
-                        hasher: sha256
-                    });
-                    // console.log('decoded', decoded.value)
-                    return decoded;
-                };
-                const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial);
-                const block = blocks.find(b => b.cid.toString() === dataCID);
-                if (block) {
-                    return block.bytes;
                 }
-            }
-            else {
-                const gotBlock = await reader.get(CID.parse(dataCID));
-                if (gotBlock) {
-                    return gotBlock.bytes;
+            };
+        }
+        else {
+            return {
+                root: reader.getRoots()[0],
+                get: async (dataCID) => {
+                    const gotBlock = await reader.get(CID.parse(dataCID));
+                    if (gotBlock) {
+                        return gotBlock.bytes;
+                    }
                 }
-            }
-        });
+            };
+        }
+    }
+    // todo memoize this
+    async getValetBlock(dataCID) {
+        // console.log('get valet block', dataCID)
+        const { result: carCid } = await this.getCarCIDForCID(dataCID);
+        if (!carCid) {
+            throw new Error('Missing block: ' + dataCID);
+        }
+        const reader = await this.getCarReader(carCid);
+        return await reader.get(dataCID);
     }
 }
 export const blocksToCarBlock = async (rootCids, blocks) => {
+    // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
     let size = 0;
     if (!Array.isArray(rootCids)) {
         rootCids = [rootCids];
@@ -293,3 +422,55 @@ const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
         return blocksPromise;
     }
 };
+const addCidsToCarIndex = async (blockstore, valetRoot, valetRootCid, bulkOperations) => {
+    let indexNode;
+    if (valetRootCid) {
+        if (valetRoot) {
+            indexNode = valetRoot;
+        }
+        else {
+            indexNode = await load(blockstore, valetRootCid, { blockHasher: blockOpts.hasher, blockCodec: blockOpts.codec });
+        }
+    }
+    else {
+        indexNode = await create(blockstore, {
+            bitWidth: 4,
+            bucketSize: 2,
+            blockHasher: blockOpts.hasher,
+            blockCodec: blockOpts.codec
+        });
+    }
+    // console.log('adding', bulkOperations.length, 'cids to index')
+    for (const { key, value } of bulkOperations) {
+        // console.log('adding', key, value)
+        await indexNode.set(key, value);
+    }
+    return indexNode;
+};
+export class VMemoryBlockstore {
+    /** @type {Map<string, Uint8Array>} */
+    blocks = new Map();
+    instanceId = Math.random().toString(36).slice(2);
+    async get(cid) {
+        const bytes = this.blocks.get(cid.toString());
+        // console.log('getvm', bytes.constructor.name, this.instanceId, cid, bytes && bytes.length)
+        if (bytes.length === 253) {
+            // console.log('getvm', bytes.())
+        }
+        if (!bytes)
+            throw new Error('block not found ' + cid.toString());
+        return { cid, bytes };
+    }
+    /**
+     * @param {import('../src/link').AnyLink} cid
+     * @param {Uint8Array} bytes
+     */
+    async put(cid, bytes) {
+        this.blocks.set(cid.toString(), bytes);
+    }
+    *entries() {
+        for (const [str, bytes] of this.blocks) {
+            yield { cid: parse(str), bytes };
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.5.16",
+  "version": "0.5.18",
   "description": "Cloudless database for apps, the browser, and IPFS",
   "main": "dist/src/fireproof.js",
   "module": "dist/src/fireproof.mjs",
@@ -49,6 +49,7 @@
     "crypto-browserify": "^3.12.0",
     "encrypted-block": "^0.0.3",
     "idb": "^7.1.1",
+    "ipld-hashmap": "^2.1.18",
     "multiformats": "^11.0.1",
     "node-polyfill-webpack-plugin": "^2.0.1",
     "prolly-trees": "1.0.4",

package/src/blockstore.js

@@ -80,7 +80,7 @@ export class TransactionBlockstore {
     // console.log('committedGet: ' + key + ' ' + this.instanceId, old.length)
     if (old) return old
     if (!this.valet) throw new Error('Missing block: ' + key)
-    const got = await this.valet.getBlock(key)
+    const got = await this.valet.getValetBlock(key)
     this.committedBlocks.set(key, got)
     return got
   }

package/src/clock.js

@@ -237,12 +237,13 @@ export async function findEventsToSync (blocks, head) {
   // console.time(callTag + '.findCommonAncestorWithSortedEvents')
   const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head)
   // console.timeEnd(callTag + '.findCommonAncestorWithSortedEvents')
-  // console.log('sorted', !!ancestor, sorted.length)
+  // console.log('sorted', !!ancestor, sorted)
   // console.time(callTag + '.contains')
 
   const toSync = ancestor ? await asyncFilter(sorted, async uks => !(await contains(events, ancestor, uks.cid))) : sorted
   // console.timeEnd(callTag + '.contains')
-  // console.log('optimize sorted', !!ancestor, sorted.length - toSync.length)
+  const sortDifference = sorted.length - toSync.length
+  if (sortDifference / sorted.length > 0.6) console.log('optimize sorted', !!ancestor, sortDifference)
 
   return { cids: events, events: toSync }
 }

package/src/database.js

@@ -1,6 +1,6 @@
 // @ts-nocheck
 import { visMerkleClock, visMerkleTree, vis, put, get, getAll, eventsSince } from './prolly.js'
-import { doTransaction } from './blockstore.js'
+import { doTransaction, TransactionBlockstore } from './blockstore.js'
 import charwise from 'charwise'
 import { localSet } from './utils.js'
 import { CID } from 'multiformats'
@@ -19,7 +19,6 @@ export const parseCID = cid => (typeof cid === 'string' ? CID.parse(cid) : cid)
  *  This is the main class for saving and loading JSON and other documents with the database. You can find additional examples and
  *  usage guides in the repository README.
  *
- * @param {import('./blockstore.js').TransactionBlockstore} blocks - The block storage instance to use documents and indexes
  * @param {CID[]} clock - The Merkle clock head to use for the Fireproof instance.
  * @param {object} [config] - Optional configuration options for the Fireproof instance.
  * @param {object} [authCtx] - Optional authorization context object to use for any authentication checks.
@@ -31,10 +30,11 @@ export class Database {
   rootCache = null
   eventsCache = new Map()
 
-  constructor (blocks, clock, config = {}) {
-    this.name = config.name
+  constructor (name, clock, config = {}) {
+    this.name = name
     this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`
-    this.blocks = blocks
+    this.blocks = new TransactionBlockstore(name, config.key)
+    this.indexBlocks = new TransactionBlockstore(name + '.indexes', config.key)
     this.clock = clock
     this.config = config
   }
@@ -51,6 +51,8 @@ export class Database {
       clock: this.clockToJSON(),
       name: this.name,
       key: this.blocks.valet?.getKeyMaterial(),
+      car: this.blocks.valet?.valetRootCarCid.toString(),
+      indexCar: this.indexBlocks.valet?.valetRootCarCid?.toString(),
       indexes: [...this.indexes.values()].map(index => index.toJSON())
     }
   }
@@ -65,11 +67,14 @@ export class Database {
     return (clock || this.clock).map(cid => cid.toString())
   }
 
-  hydrate ({ clock, name, key }) {
+  hydrate ({ clock, name, key, car, indexCar }) {
     this.name = name
     this.clock = clock
     this.blocks.valet?.setKeyMaterial(key)
-    this.indexBlocks = null
+    this.blocks.valet?.setRootCarCid(car) // maybe
+    this.indexBlocks.valet?.setKeyMaterial(key)
+    this.indexBlocks.valet?.setRootCarCid(indexCar) // maybe
+    // this.indexBlocks = null
   }
 
   maybeSaveClock () {
@@ -108,7 +113,7 @@ export class Database {
     let rows, dataCIDs, clockCIDs
     // if (!aClock) aClock = []
     if (aClock && aClock.length > 0) {
-      aClock = aClock.map((cid) => cid.toString())
+      aClock = aClock.map(cid => cid.toString())
       const eventKey = JSON.stringify([...this.clockToJSON(aClock), ...this.clockToJSON()])
 
       let resp
@@ -276,6 +281,7 @@ export class Database {
    * @returns {Promise<{ proof:{}, id: string, clock: CID[] }>} - The result of adding the event to storage
    */
   async putToProllyTree (decodedEvent, clock = null) {
+    // console.log('putToProllyTree', decodedEvent)
     const event = encodeEvent(decodedEvent)
     if (clock && JSON.stringify(this.clockToJSON(clock)) !== JSON.stringify(this.clockToJSON())) {
       // console.log('this.clock', this.clockToJSON())
@@ -393,7 +399,9 @@ export class Database {
 
 export async function cidsToProof (cids) {
   if (!cids) return []
-  if (!cids.all) { return [...cids] }
+  if (!cids.all) {
+    return [...cids]
+  }
 
   const all = await cids.all()
   return [...all].map(cid => cid.toString())

package/src/db-index.js

@@ -13,7 +13,7 @@ import { Database, cidsToProof } from './database.js'
 
 import * as codec from '@ipld/dag-cbor'
 // import { create as createBlock } from 'multiformats/block'
-import { TransactionBlockstore, doTransaction } from './blockstore.js'
+import { doTransaction } from './blockstore.js'
 // @ts-ignore
 import charwise from 'charwise'
 
@@ -69,21 +69,21 @@ const makeDoc = ({ key, value }) => ({ _id: key, ...value })
  */
 const indexEntriesForChanges = (changes, mapFn) => {
   const indexEntries = []
-  changes.forEach(({ key, value, del }) => {
+  changes.forEach(({ key: _id, value, del }) => {
     // key is _id, value is the document
     if (del || !value) return
     let mapCalled = false
-    const mapReturn = mapFn(makeDoc({ key, value }), (k, v) => {
+    const mapReturn = mapFn(makeDoc({ key: _id, value }), (k, v) => {
       mapCalled = true
       if (typeof k === 'undefined') return
       indexEntries.push({
-        key: [charwise.encode(k), key],
+        key: [charwise.encode(k), _id],
         value: v || null
       })
     })
     if (!mapCalled && mapReturn) {
       indexEntries.push({
-        key: [charwise.encode(mapReturn), key],
+        key: [charwise.encode(mapReturn), _id],
         value: null
       })
     }
@@ -107,12 +107,6 @@ export class DbIndex {
    */
   constructor (database, name, mapFn, clock = null, opts = {}) {
     this.database = database
-    if (!database.indexBlocks) {
-      database.indexBlocks = new TransactionBlockstore(
-        database?.name + '.indexes',
-        database.blocks.valet?.getKeyMaterial()
-      )
-    }
     if (typeof name === 'function') {
       // app is using deprecated API, remove in 0.7
       opts = clock || {}
@@ -265,7 +259,14 @@ export class DbIndex {
     await loadIndex(this.database.indexBlocks, this.indexByKey, dbIndexOpts)
     if (!this.indexByKey.root) return { result: [] }
     if (query.includeDocs === undefined) query.includeDocs = this.includeDocsDefault
-    if (query.range) {
+    if (query.prefix) {
+      // ensure prefix is an array
+      if (!Array.isArray(query.prefix)) query.prefix = [query.prefix]
+      const start = [...query.prefix, NaN]
+      const end = [...query.prefix, Infinity]
+      const prefixRange = [start, end].map(key => charwise.encode(key))
+      return await this.applyQuery(await this.indexByKey.root.range(...prefixRange), query)
+    } else if (query.range) {
       const encodedRange = query.range.map(key => charwise.encode(key))
       return await this.applyQuery(await this.indexByKey.root.range(...encodedRange), query)
     } else if (query.key) {

package/src/fireproof.js

@@ -2,7 +2,7 @@ import randomBytes from 'randombytes'
 import { Database, parseCID } from './database.js'
 import { Listener } from './listener.js'
 import { DbIndex as Index } from './db-index.js'
-import { TransactionBlockstore } from './blockstore.js'
+// import { TransactionBlockstore } from './blockstore.js'
 import { localGet } from './utils.js'
 import { Sync } from './sync.js'
 
@@ -20,22 +20,29 @@ export class Fireproof {
   static storage = (name = null, opts = {}) => {
     if (name) {
       opts.name = name
+      // todo this can come from a registry also
       const existing = localGet('fp.' + name)
       if (existing) {
         const existingConfig = JSON.parse(existing)
-        const fp = new Database(new TransactionBlockstore(name, existingConfig.key), [], opts)
-        return Fireproof.fromJSON(existingConfig, fp)
+        return Fireproof.fromConfig(name, existingConfig, opts)
       } else {
         const instanceKey = randomBytes(32).toString('hex') // pass null to disable encryption
-        return new Database(new TransactionBlockstore(name, instanceKey), [], opts)
+        opts.key = instanceKey
+        return new Database(name, [], opts)
       }
     } else {
-      return new Database(new TransactionBlockstore(), [], opts)
+      return new Database(null, [], opts)
     }
   }
 
+  static fromConfig (name, existingConfig, opts = {}) {
+    opts.key = existingConfig.key
+    const fp = new Database(name, [], opts)
+    return Fireproof.fromJSON(existingConfig, fp)
+  }
+
   static fromJSON (json, database) {
-    database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name, key: json.key })
+    database.hydrate({ car: json.car, indexCar: json.indexCar, clock: json.clock.map(c => parseCID(c)), name: json.name, key: json.key })
     if (json.indexes) {
       for (const {
         name,
@@ -58,7 +65,8 @@ export class Fireproof {
 
   static snapshot (database, clock) {
     const definition = database.toJSON()
-    const withBlocks = new Database(database.blocks)
+    const withBlocks = new Database(database.name)
+    withBlocks.blocks = database.blocks
     if (clock) {
       definition.clock = clock.map(c => parseCID(c))
       definition.indexes.forEach(index => {

package/src/sync.js

@@ -156,7 +156,7 @@ export class Sync {
   destroy () {
     this.database.blocks.syncs.delete(this)
     this.status = 'destroyed'
-    this.peer.destroy()
+    // this.peer.destroy()  todo
   }
 
   async sendUpdate (blockstore) {