@fireproof/core 0.5.6 → 0.5.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.5.6",
+  "version": "0.5.8",
   "description": "Cloudless database for apps, the browser, and IPFS",
   "main": "dist/src/fireproof.js",
   "module": "dist/src/fireproof.mjs",
@@ -36,6 +36,7 @@
     "Mikeal Rogers"
   ],
   "license": "Apache-2.0 OR MIT",
+  "gptdoc": "Fireproof/API/Usage: import { Fireproof, Listener, Index } from '@fireproof/core'; const db = fireproof.storage('app-db-name'); const ok = await db.put({ any: 'json' }); const doc = await db.get(ok.id); await db.del(doc._id); const all = await db.allDocuments(); all.rows.map(({key, value}) => value); const listener = new Listener(db); listener.on('*', updateReactStateFn); const index = new Index(db, (doc, map) => map(doc.any, {custom: Object.keys(doc)})); const result = await index.query({range : ['a', 'z']}); result.rows.map(({ key }) => key);",
   "dependencies": {
     "@ipld/car": "^5.1.0",
     "@ipld/dag-cbor": "^9.0.0",
@@ -53,7 +54,8 @@
     "prolly-trees": "1.0.4",
     "randombytes": "^2.1.0",
     "rollup-plugin-commonjs": "^10.1.0",
-    "sade": "^1.8.1"
+    "sade": "^1.8.1",
+    "simple-peer": "^9.11.1"
   },
   "devDependencies": {
     "@rollup/plugin-alias": "^5.0.0",

package/src/blockstore.js

@@ -34,10 +34,12 @@ export class TransactionBlockstore {
   /** @type {Map<string, Uint8Array>} */
   committedBlocks = new Map()
 
+  /** @type {Valet} */
   valet = null
 
   instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4)
   inflightTransactions = new Set()
+  syncs = new Set()
 
   constructor (name, encryptionKey) {
     if (name) {
@@ -75,10 +77,10 @@ export class TransactionBlockstore {
 
   async committedGet (key) {
     const old = this.committedBlocks.get(key)
+    // console.log('committedGet: ' + key + ' ' + this.instanceId, old.length)
     if (old) return old
     if (!this.valet) throw new Error('Missing block: ' + key)
     const got = await this.valet.getBlock(key)
-    // console.log('committedGet: ' + key)
     this.committedBlocks.set(key, got)
     return got
   }
@@ -120,18 +122,24 @@ export class TransactionBlockstore {
   /**
    * Iterate over all blocks in the store.
    *
-   * @yields {AnyBlock}
-   * @returns {AsyncGenerator<AnyBlock>}
+   * @yields {{cid: string, bytes: Uint8Array}}
+   * @returns {AsyncGenerator<any, any, any>}
    */
-  // * entries () {
-  //   // needs transaction blocks?
-  //   // for (const [str, bytes] of this.blocks) {
-  //   //   yield { cid: parse(str), bytes }
-  //   // }
-  //   for (const [str, bytes] of this.committedBlocks) {
-  //     yield { cid: parse(str), bytes }
-  //   }
-  // }
+  async * entries () {
+    for (const transaction of this.inflightTransactions) {
+      for (const [str, bytes] of transaction) {
+        yield { cid: str, bytes }
+      }
+    }
+    for (const [str, bytes] of this.committedBlocks) {
+      yield { cid: str, bytes }
+    }
+    if (this.valet) {
+      for await (const { cid } of this.valet.cids()) {
+        yield { cid }
+      }
+    }
+  }
 
   /**
    * Begin a transaction. Ensures the uncommited blocks are empty at the begining.
@@ -150,8 +158,16 @@ export class TransactionBlockstore {
    * @returns {Promise<void>}
    * @memberof TransactionBlockstore
    */
-  async commit (innerBlockstore) {
+  async commit (innerBlockstore, doSync = true) {
+    // console.log('commit', doSync, innerBlockstore.label)
     await this.doCommit(innerBlockstore)
+    if (doSync) {
+      // const all =
+      await Promise.all([...this.syncs].map(async sync => sync.sendUpdate(innerBlockstore).catch(e => {
+        console.error('sync error', e)
+        this.syncs.delete(sync)
+      })))
+    }
   }
 
   // first get the transaction blockstore from the map of transaction blockstores
@@ -170,8 +186,8 @@ export class TransactionBlockstore {
         cids.add(stringCid)
       }
     }
+    // console.log(innerBlockstore.label, 'committing', cids.size, 'blocks', [...cids].map(cid => cid.toString()), this.valet)
     if (cids.size > 0 && this.valet) {
-      // console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
       await this.valet.writeTransaction(innerBlockstore, cids)
     }
   }
@@ -195,7 +211,7 @@ export class TransactionBlockstore {
  * @returns {Promise<any>}
  * @memberof TransactionBlockstore
  */
-export const doTransaction = async (label, blockstore, doFun) => {
+export const doTransaction = async (label, blockstore, doFun, doSync = true) => {
   // @ts-ignore
   if (!blockstore.commit) return await doFun(blockstore)
   // @ts-ignore
@@ -203,7 +219,7 @@ export const doTransaction = async (label, blockstore, doFun) => {
   try {
     const result = await doFun(innerBlockstore)
     // @ts-ignore
-    await blockstore.commit(innerBlockstore)
+    await blockstore.commit(innerBlockstore, doSync)
     return result
   } catch (e) {
     console.error(`Transaction ${label} failed`, e, e.stack)

package/src/clock.js

@@ -190,7 +190,11 @@ async function contains (events, a, b) {
  */
 export async function * vis (blocks, head, options = {}) {
   // @ts-ignore
-  const renderNodeLabel = options.renderNodeLabel ?? ((b) => b.value.data.value)
+  const renderNodeLabel = options.renderNodeLabel ?? ((b) => {
+    // @ts-ignore
+    const { key, root, type } = b.value.data
+    return b.cid.toString() + '\n' + JSON.stringify({ key, root: root.cid.toString(), type }, null, 2).replace(/"/g, '\'')
+  })
   const events = new EventFetcher(blocks)
   yield 'digraph clock {'
   yield '  node [shape=point fontname="Courier"]; head;'
@@ -231,24 +235,29 @@ export async function findEventsToSync (blocks, head) {
   // console.time(callTag + '.contains')
   const toSync = await asyncFilter(sorted, async (uks) => !(await contains(events, ancestor, uks.cid)))
   // console.timeEnd(callTag + '.contains')
+  // console.log('toSync.contains', toSync.length)
 
-  return { cids: events.all(), events: toSync }
+  return { cids: events, events: toSync }
 }
 
 const asyncFilter = async (arr, predicate) =>
   Promise.all(arr.map(predicate)).then((results) => arr.filter((_v, index) => results[index]))
 
-export async function findCommonAncestorWithSortedEvents (events, children) {
+export async function findCommonAncestorWithSortedEvents (events, children, doFull = false) {
+  // console.trace('findCommonAncestorWithSortedEvents')
   // const callTag = Math.random().toString(36).substring(7)
+  // console.log(callTag + '.children', children.map((c) => c.toString()))
   // console.time(callTag + '.findCommonAncestor')
   const ancestor = await findCommonAncestor(events, children)
   // console.timeEnd(callTag + '.findCommonAncestor')
+  // console.log('ancestor', ancestor.toString())
   if (!ancestor) {
     throw new Error('failed to find common ancestor event')
   }
   // console.time(callTag + '.findSortedEvents')
-  const sorted = await findSortedEvents(events, children, ancestor)
+  const sorted = await findSortedEvents(events, children, ancestor, doFull)
   // console.timeEnd(callTag + '.findSortedEvents')
+  // console.log('sorted', sorted.length)
   return { ancestor, sorted }
 }
 
@@ -261,6 +270,7 @@ export async function findCommonAncestorWithSortedEvents (events, children) {
  */
 async function findCommonAncestor (events, children) {
   if (!children.length) return
+  if (children.length === 1) return children[0]
   const candidates = children.map((c) => [c])
   while (true) {
     let changed = false
@@ -281,7 +291,7 @@ async function findCommonAncestor (events, children) {
  * @param {import('./clock').EventLink<EventData>} root
  */
 async function findAncestorCandidate (events, root) {
-  const { value: event } = await events.get(root)
+  const { value: event } = await events.get(root)// .catch(() => ({ value: { parents: [] } }))
   if (!event.parents.length) return root
   return event.parents.length === 1 ? event.parents[0] : findCommonAncestor(events, event.parents)
 }
@@ -291,6 +301,7 @@ async function findAncestorCandidate (events, root) {
  * @param  {Array<T[]>} arrays
  */
 function findCommonString (arrays) {
+  // console.log('findCommonString', arrays.map((a) => a.map((i) => String(i))))
   arrays = arrays.map((a) => [...a])
   for (const arr of arrays) {
     for (const item of arr) {
@@ -308,15 +319,33 @@ function findCommonString (arrays) {
 /**
  * Find and sort events between the head(s) and the tail.
  * @param {import('./clock').EventFetcher} events
- * @param {import('./clock').EventLink<EventData>[]} head
+ * @param {any[]} head
  * @param {import('./clock').EventLink<EventData>} tail
  */
-async function findSortedEvents (events, head, tail) {
+async function findSortedEvents (events, head, tail, doFull) {
   // const callTag = Math.random().toString(36).substring(7)
   // get weighted events - heavier events happened first
+  // const callTag = Math.random().toString(36).substring(7)
+
   /** @type {Map<string, { event: import('./clock').EventBlockView<EventData>, weight: number }>} */
   const weights = new Map()
+  head = [...new Set([...head.map((h) => h.toString())])]
+  // console.log(callTag + '.head', head.length)
+
+  const allEvents = new Set([tail.toString(), ...head])
+  if (!doFull && allEvents.size === 1) {
+    // console.log('head contains tail', tail.toString())
+    return []
+    // const event = await events.get(tail)
+    // return [event]
+  }
+
+  // console.log('finding events')
+  // console.log(callTag + '.head', head.length, [...head.map((h) => h.toString())], tail.toString())
+
+  // console.time(callTag + '.findEvents')
   const all = await Promise.all(head.map((h) => findEvents(events, h, tail)))
+  // console.timeEnd(callTag + '.findEvents')
   for (const arr of all) {
     for (const { event, depth } of arr) {
       // console.log('event value', event.value.data.value)
@@ -345,7 +374,7 @@ async function findSortedEvents (events, head, tail) {
   const sorted = Array.from(buckets)
     .sort((a, b) => b[0] - a[0])
     .flatMap(([, es]) => es.sort((a, b) => (String(a.cid) < String(b.cid) ? -1 : 1)))
-  // console.log('sorted', sorted.map(s => s.value.data.value))
+  // console.log('sorted', sorted.map(s => s.cid))
 
   return sorted
 }
@@ -357,11 +386,14 @@ async function findSortedEvents (events, head, tail) {
  * @returns {Promise<Array<{ event: EventBlockView<EventData>, depth: number }>>}
  */
 async function findEvents (events, start, end, depth = 0) {
-  // console.log('findEvents', start)
+  // console.log('findEvents', start.toString(), end.toString(), depth)
   const event = await events.get(start)
+  const send = String(end)
   const acc = [{ event, depth }]
   const { parents } = event.value
-  if (parents.length === 1 && String(parents[0]) === String(end)) return acc
+  // if (parents.length === 1 && String(parents[0]) === send) return acc
+  if (parents.findIndex((p) => String(p) === send) !== -1) return acc
+  // if (parents.length === 1) return acc
   const rest = await Promise.all(parents.map((p) => findEvents(events, p, end, depth + 1)))
   return acc.concat(...rest)
 }

package/src/database.js

@@ -26,15 +26,16 @@ export const parseCID = cid => (typeof cid === 'string' ? CID.parse(cid) : cid)
  */
 export class Database {
   listeners = new Set()
+  indexes = new Map()
+  rootCache = null
+  eventsCache = new Map()
 
-  // todo refactor this for the next version
   constructor (blocks, clock, config = {}) {
     this.name = config.name
     this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`
     this.blocks = blocks
     this.clock = clock
     this.config = config
-    this.indexes = new Map()
   }
 
   /**
@@ -101,11 +102,22 @@ export class Database {
    * @instance
    */
   async changesSince (event) {
+    // console.log('events for', this.instanceId, event.constructor.name)
     // console.log('changesSince', this.instanceId, event, this.clock)
     let rows, dataCIDs, clockCIDs
     // if (!event) event = []
     if (event) {
-      const resp = await eventsSince(this.blocks, this.clock, event)
+      event = event.map((cid) => cid.toString())
+      const eventKey = JSON.stringify([...event, ...this.clockToJSON()])
+
+      let resp
+      if (this.eventsCache.has(eventKey)) {
+        console.log('events from cache')
+        resp = this.eventsCache.get(eventKey)
+      } else {
+        resp = await eventsSince(this.blocks, this.clock, event)
+        this.eventsCache.set(eventKey, resp)
+      }
       const docsMap = new Map()
       for (const { key, type, value } of resp.result.map(decodeEvent)) {
         if (type === 'del') {
@@ -118,7 +130,9 @@ export class Database {
       clockCIDs = resp.clockCIDs
       // console.log('change rows', this.instanceId, rows)
     } else {
-      const allResp = await getAll(this.blocks, this.clock)
+      const allResp = await getAll(this.blocks, this.clock, this.rootCache)
+      this.rootCache = { root: allResp.root, clockCIDs: allResp.clockCIDs }
+
       rows = allResp.result.map(({ key, value }) => decodeEvent({ key, value }))
       dataCIDs = allResp.cids
       // console.log('dbdoc rows', this.instanceId, rows)
@@ -131,7 +145,9 @@ export class Database {
   }
 
   async allDocuments () {
-    const allResp = await getAll(this.blocks, this.clock)
+    const allResp = await getAll(this.blocks, this.clock, this.rootCache)
+    this.rootCache = { root: allResp.root, clockCIDs: allResp.clockCIDs }
+
     const rows = allResp.result
       .map(({ key, value }) => decodeEvent({ key, value }))
       .map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
@@ -143,7 +159,9 @@ export class Database {
   }
 
   async allCIDs () {
-    const allResp = await getAll(this.blocks, this.clock)
+    const allResp = await getAll(this.blocks, this.clock, this.rootCache, true)
+    this.rootCache = { root: allResp.root, clockCIDs: allResp.clockCIDs }
+    // console.log('allcids', allResp.cids, allResp.clockCIDs)
     const cids = await cidsToProof(allResp.cids)
     const clockCids = await cidsToProof(allResp.clockCIDs)
     // console.log('allcids', cids, clockCids)
@@ -151,6 +169,14 @@ export class Database {
     return [...cids, ...clockCids] // need a single block version of clock head, maybe an encoded block for it
   }
 
+  async allStoredCIDs () {
+    const allCIDs = []
+    for await (const { cid } of this.blocks.entries()) {
+      allCIDs.push(cid)
+    }
+    return allCIDs
+  }
+
   /**
    * Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
    *
@@ -180,13 +206,13 @@ export class Database {
    */
   async get (key, opts = {}) {
     const clock = opts.clock || this.clock
-    const resp = await get(this.blocks, clock, charwise.encode(key))
-
+    const resp = await get(this.blocks, clock, charwise.encode(key), this.rootCache)
+    this.rootCache = { root: resp.root, clockCIDs: resp.clockCIDs }
     // this tombstone is temporary until we can get the prolly tree to delete
     if (!resp || resp.result === null) {
       throw new Error('Not found')
     }
-    const doc = resp.result
+    const doc = { ...resp.result }
     if (opts.mvcc === true) {
       doc._clock = this.clockToJSON()
     }
@@ -280,9 +306,20 @@ export class Database {
   }
 
   applyClock (prevClock, newClock) {
-    // console.log('applyClock', prevClock, newClock, this.clock)
-    const removedprevCIDs = this.clock.filter(cid => prevClock.indexOf(cid) === -1)
-    this.clock = removedprevCIDs.concat(newClock)
+    // console.log('prevClock', prevClock.length, prevClock.map((cid) => cid.toString()))
+    // console.log('newClock', newClock.length, newClock.map((cid) => cid.toString()))
+    // console.log('this.clock', this.clock.length, this.clockToJSON())
+    const stPrev = prevClock.map(cid => cid.toString())
+    const keptPrevClock = this.clock.filter(cid => stPrev.indexOf(cid.toString()) === -1)
+    const merged = keptPrevClock.concat(newClock)
+    const uniquebyCid = new Map()
+    for (const cid of merged) {
+      uniquebyCid.set(cid.toString(), cid)
+    }
+    this.clock = Array.from(uniquebyCid.values()).sort((a, b) => a.toString().localeCompare(b.toString()))
+    this.rootCache = null
+    this.eventsCache.clear()
+    // console.log('afterClock', this.clock.length, this.clockToJSON())
   }
 
   //   /**
@@ -341,7 +378,9 @@ export class Database {
 }
 
 export async function cidsToProof (cids) {
-  if (!cids || !cids.all) return []
+  if (!cids) return []
+  if (!cids.all) { return [...cids] }
+
   const all = await cids.all()
   return [...all].map(cid => cid.toString())
 }

package/src/db-index.js

@@ -7,7 +7,7 @@ import { sha256 as hasher } from 'multiformats/hashes/sha2'
 import { nocache as cache } from 'prolly-trees/cache'
 // @ts-ignore
 import { bf, simpleCompare } from 'prolly-trees/utils'
-import { makeGetBlock } from './prolly.js'
+import { makeGetBlock, visMerkleTree } from './prolly.js'
 // eslint-disable-next-line no-unused-vars
 import { Database, cidsToProof } from './database.js'
 
@@ -35,8 +35,8 @@ const refCompare = (aRef, bRef) => {
   return simpleCompare(aRef, bRef)
 }
 
-const dbIndexOpts = { cache, chunker: bf(3), codec, hasher, compare }
-const idIndexOpts = { cache, chunker: bf(3), codec, hasher, compare: simpleCompare }
+const dbIndexOpts = { cache, chunker: bf(30), codec, hasher, compare }
+const idIndexOpts = { cache, chunker: bf(30), codec, hasher, compare: simpleCompare }
 
 const makeDoc = ({ key, value }) => ({ _id: key, ...value })
 
@@ -93,6 +93,9 @@ const indexEntriesForChanges = (changes, mapFn) => {
  *
  */
 export class DbIndex {
+  /**
+   * @param {Database} database
+   */
   constructor (database, name, mapFn, clock = null, opts = {}) {
     this.database = database
     if (!database.indexBlocks) {
@@ -164,6 +167,14 @@ export class DbIndex {
     return new DbIndex(database, name, code, clock)
   }
 
+  async visKeyTree () {
+    return await visMerkleTree(this.database.indexBlocks, this.indexById.cid)
+  }
+
+  async visIdTree () {
+    return await visMerkleTree(this.database.indexBlocks, this.indexByKey.cid)
+  }
+
   /**
    * JSDoc for Query type.
    * @typedef {Object} DbQuery

package/src/fireproof.js

@@ -1,13 +1,12 @@
 import randomBytes from 'randombytes'
-
 import { Database, parseCID } from './database.js'
 import { Listener } from './listener.js'
 import { DbIndex as Index } from './db-index.js'
 import { TransactionBlockstore } from './blockstore.js'
 import { localGet } from './utils.js'
-import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js'
+import { Sync } from './sync.js'
 
-export { Index, Listener, Database }
+export { Index, Listener, Database, Sync }
 
 export class Fireproof {
   /**
@@ -85,41 +84,4 @@ export class Fireproof {
     await database.notifyReset() // hmm... indexes should listen to this? might be more complex than worth it. so far this is the only caller
     return database
   }
-
-  // get all the cids
-  // tell valet to make a file
-  static async makeCar (database, key) {
-    const allCIDs = await database.allCIDs()
-    const blocks = database.blocks
-
-    const rootCid = parseCID(allCIDs[allCIDs.length - 1])
-    if (typeof key === 'undefined') {
-      key = blocks.valet?.getKeyMaterial()
-    }
-    if (key) {
-      return blocksToEncryptedCarBlock(
-        rootCid,
-        {
-          entries: () => allCIDs.map(cid => ({ cid })),
-          get: async cid => await blocks.get(cid)
-        },
-        key
-      )
-    } else {
-      const carBlocks = await Promise.all(
-        allCIDs.map(async c => {
-          const b = await blocks.get(c)
-          // console.log('block', b)
-          if (typeof b.cid === 'string') {
-            b.cid = parseCID(b.cid)
-          }
-          // if (b.bytes.constructor.name === 'Buffer') console.log('conver vbuff')
-          return b
-        })
-      )
-      return blocksToCarBlock(rootCid, {
-        entries: () => carBlocks
-      })
-    }
-  }
 }

package/src/listener.js

@@ -41,7 +41,7 @@ export class Listener {
    * @returns {Function} A function to unsubscribe from the topic.
    * @memberof Listener
    * @instance
-   * @param {any} [since] - clock to flush from on launch
+   * @param {any} [since] - clock to flush from on launch, pass null for all
    */
   on (topic, subscriber, since = undefined) {
     const listOfTopicSubscribers = getTopicList(this.subcribers, topic)