@fireproof/core 0.5.6 → 0.5.8

package/src/prolly.js

@@ -17,23 +17,24 @@ import * as codec from '@ipld/dag-cbor'
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
 import { doTransaction } from './blockstore.js'
 import { create as createBlock } from 'multiformats/block'
-const blockOpts = { cache, chunker: bf(3), codec, hasher, compare }
+const blockOpts = { cache, chunker: bf(30), codec, hasher, compare }
 
 /**
  * @typedef {import('./blockstore.js').TransactionBlockstore} TransactionBlockstore
  */
 
-const withLog = async (label, fn) => {
-  const resp = await fn()
-  // console.log('withLog', label, !!resp)
-  return resp
-}
+// const withLog = async (label, fn) => {
+//   const resp = await fn()
+//   // console.log('withLog', label, !!resp)
+//   return resp
+// }
 
 // should also return a CIDCounter
-export const makeGetBlock = (blocks) => {
+export const makeGetBlock = blocks => {
   // const cids = new CIDCounter() // this could be used for proofs of mutations
-  const getBlockFn = async (address) => {
-    const { cid, bytes } = await withLog(address, () => blocks.get(address))
+  const getBlockFn = async address => {
+    // const { cid, bytes } = await withLog(address, () => blocks.get(address))
+    const { cid, bytes } = await blocks.get(address)
     // cids.add({ address: cid })
     return createBlock({ cid, bytes, hasher, codec })
   }
@@ -48,25 +49,17 @@ export const makeGetBlock = (blocks) => {
  * @param {*} param0
  * @returns
  */
-async function createAndSaveNewEvent ({
-  inBlocks,
-  bigPut,
-  root,
-  event: inEvent,
-  head,
-  additions,
-  removals = []
-}) {
+async function createAndSaveNewEvent ({ inBlocks, bigPut, root, event: inEvent, head, additions, removals = [] }) {
   let cids
   const { key, value, del } = inEvent
   const data = {
-    root: (root
+    root: root
       ? {
           cid: root.cid,
           bytes: root.bytes, // can we remove this?
           value: root.value // can we remove this?
         }
-      : null),
+      : null,
     key
   }
   // import('./clock').EventLink<import('./clock').EventData>
@@ -94,7 +87,7 @@ async function createAndSaveNewEvent ({
   }
 }
 
-const makeGetAndPutBlock = (inBlocks) => {
+const makeGetAndPutBlock = inBlocks => {
   // const mblocks = new MemoryBlockstore()
   // const blocks = new MultiBlockFetcher(mblocks, inBlocks)
   const { getBlock, cids } = makeGetBlock(inBlocks)
@@ -153,20 +146,24 @@ const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
   }
 }
 
-const doProllyBulk = async (inBlocks, head, event) => {
+const doProllyBulk = async (inBlocks, head, event, doFull = false) => {
   const { getBlock, blocks } = makeGetAndPutBlock(inBlocks)
   let bulkSorted = []
   let prollyRootNode = null
+  const events = new EventFetcher(blocks)
   if (head.length) {
-  // Otherwise, we find the common ancestor and update the root and other blocks
-    const events = new EventFetcher(blocks)
+    if (!doFull && head.length === 1) {
+      prollyRootNode = await prollyRootFromAncestor(events, head[0], getBlock)
+    } else {
+    // Otherwise, we find the common ancestor and update the root and other blocks
     // todo this is returning more events than necessary, lets define the desired semantics from the top down
     // good semantics mean we can cache the results of this call
-    const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head)
-    bulkSorted = sorted
-    // console.log('sorted', JSON.stringify(sorted.map(({ value: { data: { key, value } } }) => ({ key, value }))))
-    prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock)
+      const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head, doFull)
+      bulkSorted = sorted
+      // console.log('sorted', JSON.stringify(sorted.map(({ value: { data: { key, value } } }) => ({ key, value }))))
+      prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock)
     // console.log('event', event)
+    }
   }
 
   const bulkOperations = bulkFromEvents(bulkSorted, event)
@@ -176,16 +173,18 @@ const doProllyBulk = async (inBlocks, head, event) => {
     let root
     const newBlocks = []
     // if all operations are deletes, we can just return an empty root
-    if (bulkOperations.every((op) => op.del)) {
-      return { root: null, blocks: [] }
+    if (bulkOperations.every(op => op.del)) {
+      return { root: null, blocks: [], clockCIDs: await events.all() }
     }
     for await (const node of create({ get: getBlock, list: bulkOperations, ...blockOpts })) {
       root = await node.block
       newBlocks.push(root)
     }
-    return { root, blocks: newBlocks }
+    return { root, blocks: newBlocks, clockCIDs: await events.all() }
   } else {
-    return await prollyRootNode.bulk(bulkOperations) // { root: newProllyRootNode, blocks: newBlocks }
+    const writeResp = await prollyRootNode.bulk(bulkOperations) // { root: newProllyRootNode, blocks: newBlocks }
+    writeResp.clockCIDs = await events.all()
+    return writeResp
   }
 }
 
@@ -194,7 +193,7 @@ const doProllyBulk = async (inBlocks, head, event) => {
  *
  * @param {import('./blockstore.js').Blockstore} inBlocks Bucket block storage.
  * @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
-* @param {{key: string, value: import('./clock').EventLink<import('./clock').EventData>}} event The key of the value to put.
+ * @param {{key: string, value: import('./clock').EventLink<import('./clock').EventData>}} event The key of the value to put.
  * @param {object} [options]
  * @returns {Promise<any>}
  */
@@ -246,19 +245,25 @@ export async function put (inBlocks, head, event, options) {
  * @param {TransactionBlockstore} inBlocks Bucket block storage.
  * @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
  */
-export async function root (inBlocks, head) {
+export async function root (inBlocks, head, doFull = false) {
   if (!head.length) {
     throw new Error('no head')
   }
-  const { root: newProllyRootNode, blocks: newBlocks, cids } = await doProllyBulk(inBlocks, head)
+  // console.log('root', head.map(h => h.toString()))
+  const { root: newProllyRootNode, blocks: newBlocks, clockCIDs } = await doProllyBulk(inBlocks, head, null, doFull)
   // todo maybe these should go to a temp blockstore?
-  await doTransaction('root', inBlocks, async (transactionBlocks) => {
-    const { bigPut } = makeGetAndPutBlock(transactionBlocks)
-    for (const nb of newBlocks) {
-      bigPut(nb)
-    }
-  })
-  return { cids, node: newProllyRootNode }
+  await doTransaction(
+    'root',
+    inBlocks,
+    async transactionBlocks => {
+      const { bigPut } = makeGetAndPutBlock(transactionBlocks)
+      for (const nb of newBlocks) {
+        bigPut(nb)
+      }
+    },
+    false
+  )
+  return { clockCIDs, node: newProllyRootNode }
 }
 
 /**
@@ -284,21 +289,40 @@ export async function eventsSince (blocks, head, since) {
  * @param {TransactionBlockstore} blocks Bucket block storage.
  * @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
  *
- * @returns {Promise<{cids: CIDCounter, clockCIDs: CIDCounter, result: import('./clock').EventData[]}>}
+ * @returns {Promise<{root: any, cids: CIDCounter, clockCIDs: CIDCounter, result: import('./clock').EventData[]}>}
  *
  */
-export async function getAll (blocks, head) {
+export async function getAll (blocks, head, rootCache = null, doFull = false) {
   // todo use the root node left around from put, etc
   // move load to a central place
   if (!head.length) {
-    return { clockCIDs: new CIDCounter(), cids: new CIDCounter(), result: [] }
+    return { root: null, clockCIDs: new CIDCounter(), cids: new CIDCounter(), result: [] }
   }
-  const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head)
+  const { node: prollyRootNode, clockCIDs } = await rootOrCache(blocks, head, rootCache, doFull)
+
   if (!prollyRootNode) {
-    return { clockCIDs, cids: new CIDCounter(), result: [] }
+    return { root: null, clockCIDs, cids: new CIDCounter(), result: [] }
   }
   const { result, cids } = await prollyRootNode.getAllEntries() // todo params
-  return { clockCIDs, cids, result: result.map(({ key, value }) => ({ key, value })) }
+  return { root: prollyRootNode, clockCIDs, cids, result: result.map(({ key, value }) => ({ key, value })) }
+}
+
+async function rootOrCache (blocks, head, rootCache, doFull = false) {
+  let node
+  let clockCIDs
+  if (!doFull && rootCache && rootCache.root) {
+    // console.log('get root from cache', rootCache)
+    node = rootCache.root
+    clockCIDs = rootCache.clockCIDs
+  } else {
+    // console.log('finding root')
+    // const callTag = Math.random().toString(36).substring(7)
+    // console.time(callTag + '.root')
+    ;({ node, clockCIDs } = await root(blocks, head, doFull))
+    // console.timeEnd(callTag + '.root')
+    // console.log('found root')
+  }
+  return { node, clockCIDs }
 }
 
 /**
@@ -306,17 +330,19 @@ export async function getAll (blocks, head) {
  * @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
  * @param {string} key The key of the value to retrieve.
  */
-export async function get (blocks, head, key) {
+export async function get (blocks, head, key, rootCache = null) {
   // instead pass root from db? and always update on change
   if (!head.length) {
     return { cids: new CIDCounter(), result: null }
   }
-  const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head)
+
+  const { node: prollyRootNode, clockCIDs } = await rootOrCache(blocks, head, rootCache)
+
   if (!prollyRootNode) {
     return { clockCIDs, cids: new CIDCounter(), result: null }
   }
   const { result, cids } = await prollyRootNode.get(key)
-  return { result, cids, clockCIDs }
+  return { result, cids, clockCIDs, root: prollyRootNode }
 }
 
 export async function * vis (blocks, head) {
@@ -333,8 +359,19 @@ export async function * vis (blocks, head) {
 }
 
 export async function visMerkleTree (blocks, head) {
-  if (!head.length) {
-    return { cids: new CIDCounter(), result: null }
+  // if (!head) return
+  if (head && !Array.isArray(head)) {
+    const getBl = makeGetBlock(blocks)
+    const prollyRootNode = await load({
+      cid: head,
+      get: getBl.getBlock,
+      ...blockOpts
+    })
+    const lines = []
+    for await (const line of prollyRootNode.vis()) {
+      lines.push(line)
+    }
+    return { vis: lines.join('\n'), cids: new CIDCounter() }
   }
   const { node: prollyRootNode, cids } = await root(blocks, head)
   const lines = []

package/src/sync.js

@@ -0,0 +1,201 @@
+import SimplePeer from 'simple-peer'
+import { parseCID } from './database.js'
+import { decodeEventBlock } from './clock.js'
+import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js'
+import { CarReader } from '@ipld/car'
+
+/**
+ * @typedef {import('./database.js').Database} Database
+ */
+export class Sync {
+  /**
+   * @param {Database} database
+   * @param {typeof SimplePeer} [PeerClass]
+   * @memberof Sync
+   * @static
+   */
+  constructor (database, PeerClass = SimplePeer) {
+    this.database = database
+    this.database.blocks.syncs.add(this) // should this happen during setup?
+    this.PeerClass = PeerClass
+    this.pushBacklog = new Promise((resolve, reject) => {
+      this.pushBacklogResolve = resolve
+      this.pushBacklogReject = reject
+    })
+    // this.pushBacklog.then(() => {
+    //   // console.log('sync backlog resolved')
+    //   this.database.notifyReset()
+    // })
+  }
+
+  async offer () {
+    return this.setupPeer(true)
+  }
+
+  async accept (base64offer) {
+    const offer = JSON.parse(atob(base64offer))
+    const p = this.setupPeer(false)
+    this.peer.signal(offer)
+    return p
+  }
+
+  connect (base64accept) {
+    const accept = JSON.parse(atob(base64accept))
+    this.peer.signal(accept)
+  }
+
+  async setupPeer (initiator = false) {
+    this.peer = new this.PeerClass({
+      initiator,
+      trickle: false
+    })
+    this.peer.on('connect', () => this.startSync())
+    this.peer.on('data', data => this.gotData(data))
+    const p = new Promise((resolve, reject) => {
+      this.peer.on('signal', resolve)
+      this.peer.on('error', reject)
+    })
+    return p.then(signal => btoa(JSON.stringify(signal)))
+  }
+
+  async backlog () {
+    return this.pushBacklog
+  }
+
+  async gotData (data) {
+    // console.log('got data', data.toString())
+    let reader = null
+    try {
+      reader = await CarReader.fromBytes(data)
+    } catch (e) {
+      // console.log('not a car', data.toString())
+    }
+    if (reader) {
+      const blz = new Set()
+      for await (const block of reader.blocks()) {
+        blz.add(block)
+      }
+      const roots = await reader.getRoots()
+      // console.log(
+      //   'got car',
+      //   roots.map(c => c.toString()),
+      //   this.database.clock.map(c => c.toString())
+      // )
+      // console.log(
+      //   'got blocks',
+      //   [...blz].map(({ cid }) => cid.toString())
+      // )
+      // @ts-ignore
+      reader.entries = reader.blocks
+      await this.database.blocks.commit({
+        label: 'sync',
+        entries: () => [...blz],
+        get: async cid => await reader.get(cid),
+        lastCid: [...blz][0].cid // doesn't matter
+      }, false)
+      // first arg could be the roots parents?
+      // get the roots parents
+      const parents = await Promise.all(roots.map(async (cid) => {
+        const rbl = await reader.get(cid)
+        const block = await decodeEventBlock(rbl.bytes)
+        return block.value.parents
+      }))
+      this.database.applyClock(parents.flat(), roots)
+      this.database.notifyReset()
+      // console.log('after', this.database.clockToJSON())
+      this.pushBacklogResolve({ ok: true })
+    } else {
+      // data is a json string, parse it
+      const message = JSON.parse(data.toString())
+      // console.log('got message', message)
+      if (message.ok) {
+        this.pushBacklogResolve({ ok: true })
+      } else if (message.clock) {
+        const reqCidDiff = message
+        // this might be a CID diff
+        // console.log('got diff', reqCidDiff)
+        const carBlock = await Sync.makeCar(this.database, null, reqCidDiff.cids)
+        if (!carBlock) {
+        // we are full synced
+        // console.log('we are full synced')
+          this.peer.send(JSON.stringify({ ok: true }))
+          // this.pushBacklogResolve({ ok: true })
+        } else {
+        // console.log('do send', carBlock.bytes.length)
+          this.peer.send(carBlock.bytes)
+          // this.pushBacklogResolve({ ok: true })
+        }
+      }
+    }
+  }
+
+  async sendUpdate (blockstore) {
+    // console.log('send update from', this.database.instanceId)
+    // todo should send updates since last sync
+    const newCar = await blocksToCarBlock(blockstore.lastCid, blockstore)
+    this.peer.send(newCar.bytes)
+  }
+
+  async startSync () {
+    // console.log('start sync', this.peer.initiator)
+    const allCIDs = await this.database.allStoredCIDs()
+    // console.log('allCIDs', allCIDs)
+    const reqCidDiff = {
+      clock: this.database.clockToJSON(),
+      cids: allCIDs.map(cid => cid.toString())
+    }
+    // console.log('send diff', reqCidDiff)
+    this.peer.send(JSON.stringify(reqCidDiff))
+  }
+
+  // get all the cids
+  // tell valet to make a file
+  /**
+   * @param {import("./database.js").Database} database
+   * @param {string} key
+   */
+  static async makeCar (database, key, skip = []) {
+    const allCIDs = await database.allCIDs()
+    const blocks = database.blocks
+    const rootCIDs = database.clock
+
+    const syncCIDs = [...new Set([...rootCIDs, ...allCIDs])].filter(cid => !skip.includes(cid.toString()))
+    // console.log(
+    //   'makeCar',
+    //   rootCIDs.map(c => c.toString()),
+    //   syncCIDs.map(c => c.toString()),
+    //   allCIDs.map(c => c.toString())
+    // )
+    if (syncCIDs.length === 0) {
+      return null
+    }
+
+    if (typeof key === 'undefined') {
+      key = blocks.valet?.getKeyMaterial()
+    }
+    if (key) {
+      return blocksToEncryptedCarBlock(
+        rootCIDs,
+        {
+          entries: () => syncCIDs.map(cid => ({ cid })),
+          get: async cid => await blocks.get(cid)
+        },
+        key
+      )
+    } else {
+      const carBlocks = await Promise.all(
+        syncCIDs.map(async c => {
+          const b = await blocks.get(c)
+          if (typeof b.cid === 'string') {
+            b.cid = parseCID(b.cid)
+          }
+          return b
+        })
+      )
+      // console.log('carblock')
+      return blocksToCarBlock(rootCIDs, {
+        entries: () => carBlocks
+      })
+    }
+  }
+}

package/src/valet.js

@@ -16,7 +16,7 @@ import { Buffer } from 'buffer'
 // @ts-ignore
 import * as codec from 'encrypted-block'
 import { rawSha1 as sha1sync } from './sha1.js'
-const chunker = bf(3)
+const chunker = bf(30)
 
 const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT
 // ? process.env.NO_ENCRYPT : import.meta && import.meta.env.VITE_NO_ENCRYPT
@@ -100,12 +100,15 @@ export class Valet {
     if (innerBlockstore.lastCid) {
       if (this.keyMaterial) {
         // console.log('encrypting car', innerBlockstore.label)
+        // should we pass cids in instead of iterating frin innerBlockstore?
         const newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial)
         await this.parkCar(newCar.cid.toString(), newCar.bytes, cids)
       } else {
         const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
         await this.parkCar(newCar.cid.toString(), newCar.bytes, cids)
       }
+    } else {
+      throw new Error('missing lastCid for car header')
     }
   }
 
@@ -128,6 +131,23 @@ export class Valet {
     return await dbWorkFun(this.idb)
   }
 
+  /**
+   * Iterate over all blocks in the store.
+   *
+   * @yields {{cid: string, value: Uint8Array}}
+   * @returns {AsyncGenerator<any, any, any>}
+   */
+  async * cids () {
+    // console.log('valet cids')
+    const db = await this.withDB(async db => db)
+    const tx = db.transaction(['cidToCar'], 'readonly')
+    let cursor = await tx.store.openCursor()
+    while (cursor) {
+      yield { cid: cursor.key, car: cursor.value.car }
+      cursor = await cursor.continue()
+    }
+  }
+
   /**
    *
    * @param {string} carCid
@@ -140,7 +160,7 @@ export class Valet {
       await tx.objectStore('cidToCar').put({ pending: 'y', car: carCid, cids: Array.from(cids) })
       return await tx.done
     })
-
+    // console.log('parked car', carCid, value.length, Array.from(cids))
     // upload to web3.storage if we have credentials
     if (this.uploadFunction) {
       if (this.alreadyEnqueued.has(carCid)) {
@@ -200,9 +220,12 @@ export class Valet {
   }
 }
 
-export const blocksToCarBlock = async (lastCid, blocks) => {
+export const blocksToCarBlock = async (rootCids, blocks) => {
   let size = 0
-  const headerSize = CBW.headerLength({ roots: [lastCid] })
+  if (!Array.isArray(rootCids)) {
+    rootCids = [rootCids]
+  }
+  const headerSize = CBW.headerLength({ roots: rootCids })
   size += headerSize
   if (!Array.isArray(blocks)) {
     blocks = Array.from(blocks.entries())
@@ -214,7 +237,9 @@ export const blocksToCarBlock = async (lastCid, blocks) => {
   const buffer = new Uint8Array(size)
   const writer = await CBW.createWriter(buffer, { headerSize })
 
-  writer.addRoot(lastCid)
+  for (const cid of rootCids) {
+    writer.addRoot(cid)
+  }
 
   for (const { cid, bytes } of blocks) {
     writer.write({ cid, bytes })
@@ -230,7 +255,8 @@ export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blo
   for (const { cid } of blocks.entries()) {
     theCids.push(cid.toString())
   }
-
+  // console.log('encrypting', theCids.length, 'blocks', theCids.includes(innerBlockStoreClockRootCid.toString()))
+  // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
   let last
   for await (const block of encrypt({
     cids: theCids,