@fireproof/core 0.5.6 → 0.5.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.5.6",
+  "version": "0.5.7",
   "description": "Cloudless database for apps, the browser, and IPFS",
   "main": "dist/src/fireproof.js",
   "module": "dist/src/fireproof.mjs",
@@ -36,6 +36,7 @@
     "Mikeal Rogers"
   ],
   "license": "Apache-2.0 OR MIT",
+  "gptdoc": "Fireproof/API/Usage: import { Fireproof, Listener, Index } from '@fireproof/core'; const db = fireproof.storage('app-db-name'); const ok = await db.put({ any: 'json' }); const doc = await db.get(ok.id); await db.del(doc._id); const all = await db.allDocuments(); all.rows.map(({key, value}) => value); const listener = new Listener(db); listener.on('*', updateReactStateFn); const index = new Index(db, (doc, map) => map(doc.any, {custom: Object.keys(doc)})); const result = await index.query({range : ['a', 'z']}); result.rows.map(({ key }) => key);",
   "dependencies": {
     "@ipld/car": "^5.1.0",
     "@ipld/dag-cbor": "^9.0.0",
@@ -53,7 +54,8 @@
     "prolly-trees": "1.0.4",
     "randombytes": "^2.1.0",
     "rollup-plugin-commonjs": "^10.1.0",
-    "sade": "^1.8.1"
+    "sade": "^1.8.1",
+    "simple-peer": "^9.11.1"
   },
   "devDependencies": {
     "@rollup/plugin-alias": "^5.0.0",

package/src/blockstore.js

@@ -34,10 +34,12 @@ export class TransactionBlockstore {
   /** @type {Map<string, Uint8Array>} */
   committedBlocks = new Map()
 
+  /** @type {Valet} */
   valet = null
 
   instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4)
   inflightTransactions = new Set()
+  syncs = new Set()
 
   constructor (name, encryptionKey) {
     if (name) {
@@ -75,10 +77,10 @@ export class TransactionBlockstore {
 
   async committedGet (key) {
     const old = this.committedBlocks.get(key)
+    // console.log('committedGet: ' + key + ' ' + this.instanceId, old.length)
     if (old) return old
     if (!this.valet) throw new Error('Missing block: ' + key)
     const got = await this.valet.getBlock(key)
-    // console.log('committedGet: ' + key)
     this.committedBlocks.set(key, got)
     return got
   }
@@ -120,18 +122,24 @@ export class TransactionBlockstore {
   /**
    * Iterate over all blocks in the store.
    *
-   * @yields {AnyBlock}
-   * @returns {AsyncGenerator<AnyBlock>}
+   * @yields {{cid: string, bytes: Uint8Array}}
+   * @returns {AsyncGenerator<any, any, any>}
    */
-  // * entries () {
-  //   // needs transaction blocks?
-  //   // for (const [str, bytes] of this.blocks) {
-  //   //   yield { cid: parse(str), bytes }
-  //   // }
-  //   for (const [str, bytes] of this.committedBlocks) {
-  //     yield { cid: parse(str), bytes }
-  //   }
-  // }
+  async * entries () {
+    for (const transaction of this.inflightTransactions) {
+      for (const [str, bytes] of transaction) {
+        yield { cid: str, bytes }
+      }
+    }
+    for (const [str, bytes] of this.committedBlocks) {
+      yield { cid: str, bytes }
+    }
+    if (this.valet) {
+      for await (const { cid } of this.valet.cids()) {
+        yield { cid }
+      }
+    }
+  }
 
   /**
    * Begin a transaction. Ensures the uncommited blocks are empty at the begining.
@@ -150,8 +158,13 @@ export class TransactionBlockstore {
    * @returns {Promise<void>}
    * @memberof TransactionBlockstore
    */
-  async commit (innerBlockstore) {
+  async commit (innerBlockstore, doSync = true) {
+    // console.log('commit', doSync, innerBlockstore.label)
     await this.doCommit(innerBlockstore)
+    if (doSync) {
+      // const all =
+      await Promise.all([...this.syncs].map(async sync => sync.sendUpdate(innerBlockstore)))
+    }
   }
 
   // first get the transaction blockstore from the map of transaction blockstores
@@ -170,8 +183,8 @@ export class TransactionBlockstore {
         cids.add(stringCid)
       }
     }
+    // console.log(innerBlockstore.label, 'committing', cids.size, 'blocks', [...cids].map(cid => cid.toString()), this.valet)
     if (cids.size > 0 && this.valet) {
-      // console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
       await this.valet.writeTransaction(innerBlockstore, cids)
     }
   }
@@ -195,7 +208,7 @@ export class TransactionBlockstore {
  * @returns {Promise<any>}
  * @memberof TransactionBlockstore
  */
-export const doTransaction = async (label, blockstore, doFun) => {
+export const doTransaction = async (label, blockstore, doFun, doSync = true) => {
   // @ts-ignore
   if (!blockstore.commit) return await doFun(blockstore)
   // @ts-ignore
@@ -203,7 +216,7 @@ export const doTransaction = async (label, blockstore, doFun) => {
   try {
     const result = await doFun(innerBlockstore)
     // @ts-ignore
-    await blockstore.commit(innerBlockstore)
+    await blockstore.commit(innerBlockstore, doSync)
     return result
   } catch (e) {
     console.error(`Transaction ${label} failed`, e, e.stack)

package/src/clock.js

@@ -232,7 +232,7 @@ export async function findEventsToSync (blocks, head) {
   const toSync = await asyncFilter(sorted, async (uks) => !(await contains(events, ancestor, uks.cid)))
   // console.timeEnd(callTag + '.contains')
 
-  return { cids: events.all(), events: toSync }
+  return { cids: events, events: toSync }
 }
 
 const asyncFilter = async (arr, predicate) =>

package/src/database.js

@@ -144,6 +144,7 @@ export class Database {
 
   async allCIDs () {
     const allResp = await getAll(this.blocks, this.clock)
+    // console.log('allcids', allResp.cids, allResp.clockCIDs)
     const cids = await cidsToProof(allResp.cids)
     const clockCids = await cidsToProof(allResp.clockCIDs)
     // console.log('allcids', cids, clockCids)
@@ -151,6 +152,14 @@ export class Database {
     return [...cids, ...clockCids] // need a single block version of clock head, maybe an encoded block for it
   }
 
+  async allStoredCIDs () {
+    const allCIDs = []
+    for await (const { cid } of this.blocks.entries()) {
+      allCIDs.push(cid)
+    }
+    return allCIDs
+  }
+
   /**
    * Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
    *
@@ -281,8 +290,15 @@ export class Database {
 
   applyClock (prevClock, newClock) {
     // console.log('applyClock', prevClock, newClock, this.clock)
-    const removedprevCIDs = this.clock.filter(cid => prevClock.indexOf(cid) === -1)
-    this.clock = removedprevCIDs.concat(newClock)
+    const stPrev = prevClock.map(cid => cid.toString())
+    const keptPrevClock = this.clock.filter(cid => stPrev.indexOf(cid.toString()) === -1)
+    const merged = keptPrevClock.concat(newClock)
+    const uniquebyCid = new Map()
+    for (const cid of merged) {
+      uniquebyCid.set(cid.toString(), cid)
+    }
+    this.clock = Array.from(uniquebyCid.values())
+    // console.log('afterClock', this.clock)
   }
 
   //   /**
@@ -341,7 +357,9 @@ export class Database {
 }
 
 export async function cidsToProof (cids) {
-  if (!cids || !cids.all) return []
+  if (!cids) return []
+  if (!cids.all) { return [...cids] }
+
   const all = await cids.all()
   return [...all].map(cid => cid.toString())
 }

package/src/fireproof.js

@@ -1,13 +1,12 @@
 import randomBytes from 'randombytes'
-
 import { Database, parseCID } from './database.js'
 import { Listener } from './listener.js'
 import { DbIndex as Index } from './db-index.js'
 import { TransactionBlockstore } from './blockstore.js'
 import { localGet } from './utils.js'
-import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js'
+import { Sync } from './sync.js'
 
-export { Index, Listener, Database }
+export { Index, Listener, Database, Sync }
 
 export class Fireproof {
   /**
@@ -85,41 +84,4 @@ export class Fireproof {
     await database.notifyReset() // hmm... indexes should listen to this? might be more complex than worth it. so far this is the only caller
     return database
   }
-
-  // get all the cids
-  // tell valet to make a file
-  static async makeCar (database, key) {
-    const allCIDs = await database.allCIDs()
-    const blocks = database.blocks
-
-    const rootCid = parseCID(allCIDs[allCIDs.length - 1])
-    if (typeof key === 'undefined') {
-      key = blocks.valet?.getKeyMaterial()
-    }
-    if (key) {
-      return blocksToEncryptedCarBlock(
-        rootCid,
-        {
-          entries: () => allCIDs.map(cid => ({ cid })),
-          get: async cid => await blocks.get(cid)
-        },
-        key
-      )
-    } else {
-      const carBlocks = await Promise.all(
-        allCIDs.map(async c => {
-          const b = await blocks.get(c)
-          // console.log('block', b)
-          if (typeof b.cid === 'string') {
-            b.cid = parseCID(b.cid)
-          }
-          // if (b.bytes.constructor.name === 'Buffer') console.log('conver vbuff')
-          return b
-        })
-      )
-      return blocksToCarBlock(rootCid, {
-        entries: () => carBlocks
-      })
-    }
-  }
 }

package/src/listener.js

@@ -41,7 +41,7 @@ export class Listener {
    * @returns {Function} A function to unsubscribe from the topic.
    * @memberof Listener
    * @instance
-   * @param {any} [since] - clock to flush from on launch
+   * @param {any} [since] - clock to flush from on launch, pass null for all
    */
   on (topic, subscriber, since = undefined) {
     const listOfTopicSubscribers = getTopicList(this.subcribers, topic)

package/src/prolly.js

@@ -157,9 +157,9 @@ const doProllyBulk = async (inBlocks, head, event) => {
   const { getBlock, blocks } = makeGetAndPutBlock(inBlocks)
   let bulkSorted = []
   let prollyRootNode = null
+  const events = new EventFetcher(blocks)
   if (head.length) {
   // Otherwise, we find the common ancestor and update the root and other blocks
-    const events = new EventFetcher(blocks)
     // todo this is returning more events than necessary, lets define the desired semantics from the top down
     // good semantics mean we can cache the results of this call
     const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head)
@@ -177,15 +177,17 @@ const doProllyBulk = async (inBlocks, head, event) => {
     const newBlocks = []
     // if all operations are deletes, we can just return an empty root
     if (bulkOperations.every((op) => op.del)) {
-      return { root: null, blocks: [] }
+      return { root: null, blocks: [], clockCIDs: await events.all() }
     }
     for await (const node of create({ get: getBlock, list: bulkOperations, ...blockOpts })) {
       root = await node.block
       newBlocks.push(root)
     }
-    return { root, blocks: newBlocks }
+    return { root, blocks: newBlocks, clockCIDs: await events.all() }
   } else {
-    return await prollyRootNode.bulk(bulkOperations) // { root: newProllyRootNode, blocks: newBlocks }
+    const writeResp = await prollyRootNode.bulk(bulkOperations) // { root: newProllyRootNode, blocks: newBlocks }
+    writeResp.clockCIDs = await events.all()
+    return writeResp
   }
 }
 
@@ -250,15 +252,15 @@ export async function root (inBlocks, head) {
   if (!head.length) {
     throw new Error('no head')
   }
-  const { root: newProllyRootNode, blocks: newBlocks, cids } = await doProllyBulk(inBlocks, head)
+  const { root: newProllyRootNode, blocks: newBlocks, clockCIDs } = await doProllyBulk(inBlocks, head)
   // todo maybe these should go to a temp blockstore?
   await doTransaction('root', inBlocks, async (transactionBlocks) => {
     const { bigPut } = makeGetAndPutBlock(transactionBlocks)
     for (const nb of newBlocks) {
       bigPut(nb)
     }
-  })
-  return { cids, node: newProllyRootNode }
+  }, false)
+  return { clockCIDs, node: newProllyRootNode }
 }
 
 /**
@@ -293,7 +295,8 @@ export async function getAll (blocks, head) {
   if (!head.length) {
     return { clockCIDs: new CIDCounter(), cids: new CIDCounter(), result: [] }
   }
-  const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head)
+  const { node: prollyRootNode, clockCIDs } = await root(blocks, head)
+
   if (!prollyRootNode) {
     return { clockCIDs, cids: new CIDCounter(), result: [] }
   }

package/src/sync.js

@@ -0,0 +1,194 @@
+import SimplePeer from 'simple-peer'
+import { parseCID } from './database.js'
+import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js'
+import { CarReader } from '@ipld/car'
+
+/**
+ * @typedef {import('./database.js').Database} Database
+ */
+export class Sync {
+  /**
+   * @param {Database} database
+   * @param {typeof SimplePeer} [PeerClass]
+   * @memberof Sync
+   * @static
+   */
+  constructor (database, PeerClass = SimplePeer) {
+    this.database = database
+    this.database.blocks.syncs.add(this)
+    this.PeerClass = PeerClass
+    this.pushBacklog = new Promise((resolve, reject) => {
+      this.pushBacklogResolve = resolve
+      this.pushBacklogReject = reject
+    })
+    // this.pushBacklog.then(() => {
+    //   // console.log('sync backlog resolved')
+    //   this.database.notifyReset()
+    // })
+  }
+
+  async offer () {
+    return this.setupPeer(true)
+  }
+
+  async accept (base64offer) {
+    const offer = JSON.parse(atob(base64offer))
+    const p = this.setupPeer(false)
+    this.peer.signal(offer)
+    return p
+  }
+
+  connect (base64accept) {
+    const accept = JSON.parse(atob(base64accept))
+    this.peer.signal(accept)
+  }
+
+  async setupPeer (initiator = false) {
+    this.peer = new this.PeerClass({
+      initiator,
+      trickle: false
+    })
+    this.peer.on('connect', () => this.startSync())
+    this.peer.on('data', data => this.gotData(data))
+    const p = new Promise((resolve, reject) => {
+      this.peer.on('signal', resolve)
+      this.peer.on('error', reject)
+    })
+    return p.then(signal => btoa(JSON.stringify(signal)))
+  }
+
+  async backlog () {
+    return this.pushBacklog
+  }
+
+  async gotData (data) {
+    // console.log('got data', data.toString())
+    try {
+      const reader = await CarReader.fromBytes(data)
+      const blz = new Set()
+      for await (const block of reader.blocks()) {
+        blz.add(block)
+      }
+      const roots = await reader.getRoots()
+      // console.log(
+      //   'got car',
+      //   roots.map(c => c.toString()),
+      //   this.database.clock.map(c => c.toString())
+      // )
+      // console.log(
+      //   'got blocks',
+      //   [...blz].map(({ cid }) => cid.toString())
+      // )
+      // @ts-ignore
+      reader.entries = reader.blocks
+      await this.database.blocks.commit({
+        label: 'sync',
+        entries: () => [...blz],
+        get: async cid => await reader.get(cid),
+        lastCid: [...blz][0].cid // doesn't matter
+      }, false)
+      this.database.applyClock([], roots)
+      this.database.notifyReset()
+      // console.log('after', this.database.clockToJSON())
+      this.pushBacklogResolve({ ok: true })
+    } catch (e) {
+      // console.error(e)
+      // if e.message matche 'CBOR' we can ignore it
+      if (!e.message.match(/CBOR|fromBytes/)) {
+        throw e
+      }
+
+      // data is a json string, parse it
+      const message = JSON.parse(data.toString())
+      // console.log('got message', message)
+      if (message.ok) {
+        this.pushBacklogResolve({ ok: true })
+      } else if (message.clock) {
+        const reqCidDiff = message
+        // this might be a CID diff
+        // console.log('got diff', reqCidDiff)
+        const carBlock = await Sync.makeCar(this.database, null, reqCidDiff.cids)
+        if (!carBlock) {
+        // we are full synced
+        // console.log('we are full synced')
+          this.peer.send(JSON.stringify({ ok: true }))
+          // this.pushBacklogResolve({ ok: true })
+        } else {
+        // console.log('do send', carBlock.bytes.length)
+          this.peer.send(carBlock.bytes)
+          // this.pushBacklogResolve({ ok: true })
+        }
+      }
+    }
+  }
+
+  async sendUpdate (blockstore) {
+    console.log('send update from', this.database.instanceId)
+    // todo should send updates since last sync
+    const newCar = await blocksToCarBlock(blockstore.lastCid, blockstore)
+    this.peer.send(newCar.bytes)
+  }
+
+  async startSync () {
+    // console.log('start sync', this.peer.initiator)
+    const allCIDs = await this.database.allStoredCIDs()
+    // console.log('allCIDs', allCIDs)
+    const reqCidDiff = {
+      clock: this.database.clockToJSON(),
+      cids: allCIDs.map(cid => cid.toString())
+    }
+    // console.log('send diff', reqCidDiff)
+    this.peer.send(JSON.stringify(reqCidDiff))
+  }
+
+  // get all the cids
+  // tell valet to make a file
+  /**
+   * @param {import("./database.js").Database} database
+   * @param {string} key
+   */
+  static async makeCar (database, key, skip = []) {
+    const allCIDs = await database.allCIDs()
+    const blocks = database.blocks
+    const rootCIDs = database.clock
+
+    const syncCIDs = [...new Set([...rootCIDs, ...allCIDs])].filter(cid => !skip.includes(cid.toString()))
+    // console.log(
+    //   'makeCar',
+    //   rootCIDs.map(c => c.toString()),
+    //   syncCIDs.map(c => c.toString()),
+    //   allCIDs.map(c => c.toString())
+    // )
+    if (syncCIDs.length === 0) {
+      return null
+    }
+
+    if (typeof key === 'undefined') {
+      key = blocks.valet?.getKeyMaterial()
+    }
+    if (key) {
+      return blocksToEncryptedCarBlock(
+        rootCIDs,
+        {
+          entries: () => syncCIDs.map(cid => ({ cid })),
+          get: async cid => await blocks.get(cid)
+        },
+        key
+      )
+    } else {
+      const carBlocks = await Promise.all(
+        syncCIDs.map(async c => {
+          const b = await blocks.get(c)
+          if (typeof b.cid === 'string') {
+            b.cid = parseCID(b.cid)
+          }
+          return b
+        })
+      )
+      // console.log('carblock')
+      return blocksToCarBlock(rootCIDs, {
+        entries: () => carBlocks
+      })
+    }
+  }
+}

package/src/valet.js

@@ -100,12 +100,15 @@ export class Valet {
     if (innerBlockstore.lastCid) {
       if (this.keyMaterial) {
         // console.log('encrypting car', innerBlockstore.label)
+        // should we pass cids in instead of iterating frin innerBlockstore?
         const newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial)
         await this.parkCar(newCar.cid.toString(), newCar.bytes, cids)
       } else {
         const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
         await this.parkCar(newCar.cid.toString(), newCar.bytes, cids)
       }
+    } else {
+      throw new Error('missing lastCid for car header')
     }
   }
 
@@ -128,6 +131,23 @@ export class Valet {
     return await dbWorkFun(this.idb)
   }
 
+  /**
+   * Iterate over all blocks in the store.
+   *
+   * @yields {{cid: string, value: Uint8Array}}
+   * @returns {AsyncGenerator<any, any, any>}
+   */
+  async * cids () {
+    // console.log('valet cids')
+    const db = await this.withDB(async db => db)
+    const tx = db.transaction(['cidToCar'], 'readonly')
+    let cursor = await tx.store.openCursor()
+    while (cursor) {
+      yield { cid: cursor.key, car: cursor.value.car }
+      cursor = await cursor.continue()
+    }
+  }
+
   /**
    *
    * @param {string} carCid
@@ -140,7 +160,7 @@ export class Valet {
       await tx.objectStore('cidToCar').put({ pending: 'y', car: carCid, cids: Array.from(cids) })
       return await tx.done
     })
-
+    // console.log('parked car', carCid, value.length, Array.from(cids))
     // upload to web3.storage if we have credentials
     if (this.uploadFunction) {
       if (this.alreadyEnqueued.has(carCid)) {
@@ -200,9 +220,12 @@ export class Valet {
   }
 }
 
-export const blocksToCarBlock = async (lastCid, blocks) => {
+export const blocksToCarBlock = async (rootCids, blocks) => {
   let size = 0
-  const headerSize = CBW.headerLength({ roots: [lastCid] })
+  if (!Array.isArray(rootCids)) {
+    rootCids = [rootCids]
+  }
+  const headerSize = CBW.headerLength({ roots: rootCids })
   size += headerSize
   if (!Array.isArray(blocks)) {
     blocks = Array.from(blocks.entries())
@@ -214,7 +237,9 @@ export const blocksToCarBlock = async (lastCid, blocks) => {
   const buffer = new Uint8Array(size)
   const writer = await CBW.createWriter(buffer, { headerSize })
 
-  writer.addRoot(lastCid)
+  for (const cid of rootCids) {
+    writer.addRoot(cid)
+  }
 
   for (const { cid, bytes } of blocks) {
     writer.write({ cid, bytes })
@@ -230,7 +255,8 @@ export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blo
   for (const { cid } of blocks.entries()) {
     theCids.push(cid.toString())
   }
-
+  // console.log('encrypting', theCids.length, 'blocks', theCids.includes(innerBlockStoreClockRootCid.toString()))
+  // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
   let last
   for await (const block of encrypt({
     cids: theCids,