@fireproof/core 0.1.1 → 0.3.0

package/src/fireproof.js

@@ -1,4 +1,5 @@
-import { vis, put, get, getAll, eventsSince } from './prolly.js'
+import { randomBytes } from 'crypto'
+import { visMerkleClock, visMerkleTree, vis, put, get, getAll, eventsSince } from './prolly.js'
 import TransactionBlockstore, { doTransaction } from './blockstore.js'
 import charwise from 'charwise'
 
@@ -28,7 +29,10 @@ export default class Fireproof {
    * @returns {Fireproof} - a new Fireproof instance
    */
   static storage = (name = 'global') => {
-    return new Fireproof(new TransactionBlockstore(name), [], { name })
+    const instanceKey = randomBytes(32).toString('hex') // pass null to disable encryption
+    // pick a random key from const validatedKeys
+    // const instanceKey = validatedKeys[Math.floor(Math.random() * validatedKeys.length)]
+    return new Fireproof(new TransactionBlockstore(name, instanceKey), [], { name })
   }
 
   constructor (blocks, clock, config, authCtx = {}) {
@@ -50,15 +54,22 @@ export default class Fireproof {
   toJSON () {
     // todo this also needs to return the index roots...
     return {
-      clock: this.clock.map(cid => cid.toString()),
+      clock: this.clockToJSON(),
       name: this.name,
+      key: this.blocks.valet.getKeyMaterial(),
       indexes: [...this.indexes.values()].map(index => index.toJSON())
     }
   }
 
-  hydrate ({ clock, name }) {
+  clockToJSON () {
+    return this.clock.map(cid => cid.toString())
+  }
+
+  hydrate ({ clock, name, key }) {
     this.name = name
     this.clock = clock
+    this.blocks.valet.setKeyMaterial(key)
+    this.indexBlocks = null
   }
 
   /**
@@ -71,7 +82,12 @@ export default class Fireproof {
    * @instance
    */
   async notifyReset () {
-    await this.#notifyListeners({ reset: true, clock: this.clock })
+    await this.#notifyListeners({ _reset: true, _clock: this.clockToJSON() })
+  }
+
+  // used be indexes etc to notify database listeners of new availability
+  async notifyExternal (source = 'unknown') {
+    await this.#notifyListeners({ _external: source, _clock: this.clockToJSON() })
   }
 
   /**
@@ -85,6 +101,7 @@ export default class Fireproof {
   async changesSince (event) {
     // console.log('changesSince', this.instanceId, event, this.clock)
     let rows, dataCIDs, clockCIDs
+    // if (!event) event = []
     if (event) {
       const resp = await eventsSince(this.blocks, this.clock, event)
       const docsMap = new Map()
@@ -106,29 +123,18 @@ export default class Fireproof {
     }
     return {
       rows,
-      clock: this.clock,
+      clock: this.clockToJSON(),
       proof: { data: await cidsToProof(dataCIDs), clock: await cidsToProof(clockCIDs) }
     }
   }
 
-  /**
-   * Registers a Listener to be called when the Fireproof instance's clock is updated.
-   * Recieves live changes from the database after they are committed.
-   * @param {Function} listener - The listener to be called when the clock is updated.
-   * @returns {Function} - A function that can be called to unregister the listener.
-   * @memberof Fireproof
-   */
-  registerListener (listener) {
-    this.#listeners.add(listener)
-    return () => {
-      this.#listeners.delete(listener)
-    }
-  }
-
-  async #notifyListeners (changes) {
-    // await sleep(10)
-    for (const listener of this.#listeners) {
-      await listener(changes)
+  async allDocuments () {
+    const allResp = await getAll(this.blocks, this.clock)
+    const rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value }))).map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
+    return {
+      rows,
+      clock: this.clockToJSON(),
+      proof: await cidsToProof(allResp.cids)
     }
   }
 
@@ -150,6 +156,35 @@ export default class Fireproof {
     }
   }
 
+  /**
+   * Retrieves the document with the specified ID from the database
+   *
+   * @param {string} key - the ID of the document to retrieve
+   * @param {Object} [opts] - options
+   * @returns {Object<{_id: string, ...doc: Object}>} - the document with the specified ID
+   * @memberof Fireproof
+   * @instance
+   */
+  async get (key, opts = {}) {
+    const clock = opts.clock || this.clock
+    const resp = await get(this.blocks, clock, charwise.encode(key))
+
+    // this tombstone is temporary until we can get the prolly tree to delete
+    if (!resp || resp.result === null) {
+      throw new Error('Not found')
+    }
+    const doc = resp.result
+    if (opts.mvcc === true) {
+      doc._clock = this.clockToJSON()
+    }
+    doc._proof = {
+      data: await cidsToProof(resp.cids),
+      clock: this.clockToJSON()
+    }
+    doc._id = key
+    return doc
+  }
+
   /**
    * Adds a new document to the database, or updates an existing document. Returns the ID of the document and the new clock head.
    *
@@ -183,9 +218,9 @@ export default class Fireproof {
       id = docOrId
     }
     await this.#runValidation({ _id: id, _deleted: true })
-    // return await this.#putToProllyTree({ key: id, del: true }) // not working at prolly tree layer?
+    return await this.#putToProllyTree({ key: id, del: true }, clock) // not working at prolly tree layer?
     // this tombstone is temporary until we can get the prolly tree to delete
-    return await this.#putToProllyTree({ key: id, value: null }, clock)
+    // return await this.#putToProllyTree({ key: id, value: null }, clock)
   }
 
   /**
@@ -196,7 +231,7 @@ export default class Fireproof {
    */
   async #putToProllyTree (decodedEvent, clock = null) {
     const event = encodeEvent(decodedEvent)
-    if (clock && JSON.stringify(clock) !== JSON.stringify(this.clock)) {
+    if (clock && JSON.stringify(clock) !== JSON.stringify(this.clockToJSON())) {
       // we need to check and see what version of the document exists at the clock specified
       // if it is the same as the one we are trying to put, then we can proceed
       const resp = await eventsSince(this.blocks, this.clock, event.value._clock)
@@ -219,7 +254,7 @@ export default class Fireproof {
     await this.#notifyListeners([decodedEvent]) // this type is odd
     return {
       id: decodedEvent.key,
-      clock: this.clock,
+      clock: this.clockToJSON(),
       proof: { data: await cidsToProof(result.cids), clock: await cidsToProof(result.clockCIDs) }
     }
     // todo should include additions (or split clock)
@@ -235,51 +270,37 @@ export default class Fireproof {
   //       return this.clock
   //     }
 
-  /**
-   * Displays a visualization of the current clock in the console
-   */
-  //   async visClock () {
-  //     const shortLink = (l) => `${String(l).slice(0, 4)}..${String(l).slice(-4)}`
-  //     const renderNodeLabel = (event) => {
-  //       return event.value.data.type === 'put'
-  //         ? `${shortLink(event.cid)}\\nput(${shortLink(event.value.data.key)},
-  //         {${Object.values(event.value.data.value)}})`
-  //         : `${shortLink(event.cid)}\\ndel(${event.value.data.key})`
-  //     }
-  //     for await (const line of vis(this.blocks, this.clock, { renderNodeLabel })) console.log(line)
-  //   }
+  async * vis () {
+    return yield * vis(this.blocks, this.clock)
+  }
+
+  async visTree () {
+    return await visMerkleTree(this.blocks, this.clock)
+  }
+
+  async visClock () {
+    return await visMerkleClock(this.blocks, this.clock)
+  }
 
   /**
-   * Retrieves the document with the specified ID from the database
-   *
-   * @param {string} key - the ID of the document to retrieve
-   * @param {Object} [opts] - options
-   * @returns {Object<{_id: string, ...doc: Object}>} - the document with the specified ID
+   * Registers a Listener to be called when the Fireproof instance's clock is updated.
+   * Recieves live changes from the database after they are committed.
+   * @param {Function} listener - The listener to be called when the clock is updated.
+   * @returns {Function} - A function that can be called to unregister the listener.
    * @memberof Fireproof
-   * @instance
    */
-  async get (key, opts = {}) {
-    const clock = opts.clock || this.clock
-    const resp = await get(this.blocks, clock, charwise.encode(key))
-
-    // this tombstone is temporary until we can get the prolly tree to delete
-    if (!resp || resp.result === null) {
-      throw new Error('Not found')
-    }
-    const doc = resp.result
-    if (opts.mvcc === true) {
-      doc._clock = this.clock
-    }
-    doc._proof = {
-      data: await cidsToProof(resp.cids),
-      clock: this.clock
+  registerListener (listener) {
+    this.#listeners.add(listener)
+    return () => {
+      this.#listeners.delete(listener)
     }
-    doc._id = key
-    return doc
   }
 
-  async * vis () {
-    return yield * vis(this.blocks, this.clock)
+  async #notifyListeners (changes) {
+    // await sleep(10)
+    for (const listener of this.#listeners) {
+      await listener(changes)
+    }
   }
 
   setCarUploader (carUploaderFn) {

package/src/hydrator.js

@@ -6,16 +6,19 @@ const parseCID = cid => typeof cid === 'string' ? CID.parse(cid) : cid
 
 export default class Hydrator {
   static fromJSON (json, database) {
-    database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name })
-    for (const { code, clock: { byId, byKey, db } } of json.indexes) {
-      DbIndex.fromJSON(database, {
-        clock: {
-          byId: byId ? parseCID(byId) : null,
-          byKey: byKey ? parseCID(byKey) : null,
-          db: db ? db.map(c => parseCID(c)) : null
-        },
-        code
-      })
+    database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name, key: json.key })
+    if (json.indexes) {
+      for (const { name, code, clock: { byId, byKey, db } } of json.indexes) {
+        DbIndex.fromJSON(database, {
+          clock: {
+            byId: byId ? parseCID(byId) : null,
+            byKey: byKey ? parseCID(byKey) : null,
+            db: db ? db.map(c => parseCID(c)) : null
+          },
+          code,
+          name
+        })
+      }
     }
     return database
   }
@@ -45,7 +48,7 @@ export default class Hydrator {
       index.dbHead = null
     })
     database.clock = clock.map(c => parseCID(c))
-    await database.notifyReset()
+    await database.notifyReset() // hmm... indexes should listen to this? might be more complex than worth it. so far this is the only caller
     return database
   }
 }

package/src/listener.js

@@ -19,7 +19,7 @@ export default class Listener {
      * @type {Fireproof}
      */
     this.database = database
-    this.#doStopListening = database.registerListener((changes) => this.#onChanges(changes))
+    this.#doStopListening = database.registerListener(changes => this.#onChanges(changes))
     /**
      * The map function to apply to each entry in the database.
      * @type {Function}
@@ -46,7 +46,7 @@ export default class Listener {
     if (typeof since !== 'undefined') {
       this.database.changesSince(since).then(({ rows: changes }) => {
         const keys = topicsForChanges(changes, this.routingFn).get(topic)
-        if (keys) keys.forEach((key) => subscriber(key))
+        if (keys) keys.forEach(key => subscriber(key))
       })
     }
     return () => {
@@ -60,18 +60,14 @@ export default class Listener {
       const seenTopics = topicsForChanges(changes, this.routingFn)
       for (const [topic, keys] of seenTopics) {
         const listOfTopicSubscribers = getTopicList(this.#subcribers, topic)
-        listOfTopicSubscribers.forEach((subscriber) => keys.forEach((key) => subscriber(key)))
+        listOfTopicSubscribers.forEach(subscriber => keys.forEach(key => subscriber(key)))
       }
     } else {
-      // reset event
-      if (changes.reset) {
-        for (const [, listOfTopicSubscribers] of this.#subcribers) {
-          listOfTopicSubscribers.forEach((subscriber) => subscriber(changes))
-        }
+      // non-arrays go to all subscribers
+      for (const [, listOfTopicSubscribers] of this.#subcribers) {
+        listOfTopicSubscribers.forEach(subscriber => subscriber(changes))
       }
     }
-    // if changes is special, notify all listeners?
-    // first make the example app use listeners
   }
 }
 
@@ -84,9 +80,6 @@ function getTopicList (subscribersMap, name) {
   return topicList
 }
 
-// copied from src/db-index.js
-const makeDoc = ({ key, value }) => ({ _id: key, ...value })
-
 /**
  * Transforms a set of changes to events using an emitter function.
  *
@@ -99,7 +92,7 @@ const topicsForChanges = (changes, routingFn) => {
   const seenTopics = new Map()
   changes.forEach(({ key, value, del }) => {
     if (del || !value) value = { _deleted: true }
-    routingFn(makeDoc({ key, value }), (t) => {
+    routingFn(({ _id: key, ...value }), t => {
       const topicList = getTopicList(seenTopics, t)
       topicList.push(key)
     })

package/src/prolly.js

@@ -3,7 +3,8 @@ import {
   EventFetcher,
   EventBlock,
   findCommonAncestorWithSortedEvents,
-  findEventsToSync
+  findEventsToSync,
+  vis as visClock
 } from './clock.js'
 import { create, load } from 'prolly-trees/map'
 // import { create, load } from '../../../../prolly-trees/src/map.js'
@@ -13,7 +14,7 @@ import * as codec from '@ipld/dag-cbor'
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
 import { doTransaction } from './blockstore.js'
 import { create as createBlock } from 'multiformats/block'
-const opts = { cache, chunker: bf(3), codec, hasher, compare }
+const blockOpts = { cache, chunker: bf(3), codec, hasher, compare }
 
 const withLog = async (label, fn) => {
   const resp = await fn()
@@ -66,12 +67,13 @@ async function createAndSaveNewEvent ({
   let cids
   const { key, value, del } = inEvent
   const data = {
-    type: 'put',
-    root: {
-      cid: root.cid,
-      bytes: root.bytes,
-      value: root.value
-    },
+    root: (root
+      ? {
+          cid: root.cid,
+          bytes: root.bytes, // can we remove this?
+          value: root.value // can we remove this?
+        }
+      : null),
     key
   }
 
@@ -80,6 +82,7 @@ async function createAndSaveNewEvent ({
     data.type = 'del'
   } else {
     data.value = value
+    data.type = 'put'
   }
   /** @type {EventData} */
 
@@ -114,13 +117,27 @@ const makeGetAndPutBlock = (inBlocks) => {
   return { getBlock, bigPut, blocks: inBlocks, cids }
 }
 
-const bulkFromEvents = (sorted) =>
-  sorted.map(({ value: event }) => {
+const bulkFromEvents = (sorted, event) => {
+  if (event) {
+    const update = { value: { data: { key: event.key } } }
+    if (event.del) {
+      update.value.data.type = 'del'
+    } else {
+      update.value.data.type = 'put'
+      update.value.data.value = event.value
+    }
+    sorted.push(update)
+  }
+  const bulk = new Map()
+  for (const { value: event } of sorted) {
     const {
       data: { type, value, key }
     } = event
-    return type === 'put' ? { key, value } : { key, del: true }
-  })
+    const bulkEvent = type === 'put' ? { key, value } : { key, del: true }
+    bulk.set(bulkEvent.key, bulkEvent) // last wins
+  }
+  return Array.from(bulk.values())
+}
 
 // Get the value of the root from the ancestor event
 /**
@@ -135,7 +152,47 @@ const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
   const event = await events.get(ancestor)
   const { root } = event.value.data
   // console.log('prollyRootFromAncestor', root.cid, JSON.stringify(root.value))
-  return load({ cid: root.cid, get: getBlock, ...opts })
+  if (root) {
+    return load({ cid: root.cid, get: getBlock, ...blockOpts })
+  } else {
+    return null
+  }
+}
+
+const doProllyBulk = async (inBlocks, head, event) => {
+  const { getBlock, blocks } = makeGetAndPutBlock(inBlocks)
+  let bulkSorted = []
+  let prollyRootNode = null
+  if (head.length) {
+  // Otherwise, we find the common ancestor and update the root and other blocks
+    const events = new EventFetcher(blocks)
+    // todo this is returning more events than necessary, lets define the desired semantics from the top down
+    // good semantics mean we can cache the results of this call
+    const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head)
+    bulkSorted = sorted
+    // console.log('sorted', JSON.stringify(sorted.map(({ value: { data: { key, value } } }) => ({ key, value }))))
+    prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock)
+    // console.log('event', event)
+  }
+
+  const bulkOperations = bulkFromEvents(bulkSorted, event)
+
+  // if prolly root node is null, we need to create a new one
+  if (!prollyRootNode) {
+    let root
+    const newBlocks = []
+    // if all operations are deletes, we can just return an empty root
+    if (bulkOperations.every((op) => op.del)) {
+      return { root: null, blocks: [] }
+    }
+    for await (const node of create({ get: getBlock, list: bulkOperations, ...blockOpts })) {
+      root = await node.block
+      newBlocks.push(root)
+    }
+    return { root, blocks: newBlocks }
+  } else {
+    return await prollyRootNode.bulk(bulkOperations) // { root: newProllyRootNode, blocks: newBlocks }
+  }
 }
 
 /**
@@ -149,44 +206,45 @@ const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
  * @returns {Promise<Result>}
  */
 export async function put (inBlocks, head, event, options) {
-  const { getBlock, bigPut, blocks } = makeGetAndPutBlock(inBlocks)
+  const { bigPut } = makeGetAndPutBlock(inBlocks)
 
   // If the head is empty, we create a new event and return the root and addition blocks
   if (!head.length) {
     const additions = new Map()
-    let root
-    for await (const node of create({ get: getBlock, list: [event], ...opts })) {
-      root = await node.block
-      bigPut(root, additions)
+    const { root, blocks } = await doProllyBulk(inBlocks, head, event)
+    for (const b of blocks) {
+      bigPut(b, additions)
     }
     return createAndSaveNewEvent({ inBlocks, bigPut, root, event, head, additions: Array.from(additions.values()) })
   }
+  const { root: newProllyRootNode, blocks: newBlocks } = await doProllyBulk(inBlocks, head, event)
 
-  // Otherwise, we find the common ancestor and update the root and other blocks
-  const events = new EventFetcher(blocks)
-  // todo this is returning more events than necessary, lets define the desired semantics from the top down
-  // good semantics mean we can cache the results of this call
-  const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head)
-  // console.log('sorted', JSON.stringify(sorted.map(({ value: { data: { key, value } } }) => ({ key, value }))))
-  const prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock)
-
-  const bulkOperations = bulkFromEvents(sorted)
-  const { root: newProllyRootNode, blocks: newBlocks } = await prollyRootNode.bulk([...bulkOperations, event]) // ading delete support here
-  const prollyRootBlock = await newProllyRootNode.block
-  const additions = new Map() // ; const removals = new Map()
-  bigPut(prollyRootBlock, additions)
-  for (const nb of newBlocks) {
-    bigPut(nb, additions)
+  if (!newProllyRootNode) {
+    return createAndSaveNewEvent({
+      inBlocks,
+      bigPut,
+      root: null,
+      event,
+      head,
+      additions: []
+    })
+  } else {
+    const prollyRootBlock = await newProllyRootNode.block
+    const additions = new Map() // ; const removals = new Map()
+    bigPut(prollyRootBlock, additions)
+    for (const nb of newBlocks) {
+      bigPut(nb, additions)
+    }
+    // additions are new blocks
+    return createAndSaveNewEvent({
+      inBlocks,
+      bigPut,
+      root: prollyRootBlock,
+      event,
+      head,
+      additions: Array.from(additions.values()) /*, todo? Array.from(removals.values()) */
+    })
   }
-  // additions are new blocks
-  return createAndSaveNewEvent({
-    inBlocks,
-    bigPut,
-    root: prollyRootBlock,
-    event,
-    head,
-    additions: Array.from(additions.values()) /*, todo? Array.from(removals.values()) */
-  })
 }
 
 /**
@@ -199,25 +257,15 @@ export async function root (inBlocks, head) {
   if (!head.length) {
     throw new Error('no head')
   }
-  const { getBlock, blocks } = makeGetAndPutBlock(inBlocks)
-  const events = new EventFetcher(blocks)
-  const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head)
-  const prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock)
-
-  // Perform bulk operations (put or delete) for each event in the sorted array
-  const bulkOperations = bulkFromEvents(sorted)
-  const { root: newProllyRootNode, blocks: newBlocks } = await prollyRootNode.bulk(bulkOperations)
-  // const prollyRootBlock = await newProllyRootNode.block
-  // console.log('newBlocks', newBlocks.map((nb) => nb.cid.toString()))
+  const { root: newProllyRootNode, blocks: newBlocks, cids } = await doProllyBulk(inBlocks, head)
   // todo maybe these should go to a temp blockstore?
   await doTransaction('root', inBlocks, async (transactionBlockstore) => {
     const { bigPut } = makeGetAndPutBlock(transactionBlockstore)
     for (const nb of newBlocks) {
       bigPut(nb)
     }
-    // bigPut(prollyRootBlock)
   })
-  return { cids: events.cids, node: newProllyRootNode }
+  return { cids, node: newProllyRootNode }
 }
 
 /**
@@ -251,6 +299,9 @@ export async function getAll (blocks, head) {
     return { clockCIDs: new CIDCounter(), cids: new CIDCounter(), result: [] }
   }
   const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head)
+  if (!prollyRootNode) {
+    return { clockCIDs, cids: new CIDCounter(), result: [] }
+  }
   const { result, cids } = await prollyRootNode.getAllEntries() // todo params
   return { clockCIDs, cids, result: result.map(({ key, value }) => ({ key, value })) }
 }
@@ -266,6 +317,9 @@ export async function get (blocks, head, key) {
     return { cids: new CIDCounter(), result: null }
   }
   const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head)
+  if (!prollyRootNode) {
+    return { clockCIDs, cids: new CIDCounter(), result: null }
+  }
   const { result, cids } = await prollyRootNode.get(key)
   return { result, cids, clockCIDs }
 }
@@ -278,6 +332,28 @@ export async function * vis (blocks, head) {
   const lines = []
   for await (const line of prollyRootNode.vis()) {
     yield line
+    lines.push(line)
   }
   return { vis: lines.join('\n'), cids }
 }
+
+export async function visMerkleTree (blocks, head) {
+  if (!head.length) {
+    return { cids: new CIDCounter(), result: null }
+  }
+  const { node: prollyRootNode, cids } = await root(blocks, head)
+  const lines = []
+  for await (const line of prollyRootNode.vis()) {
+    lines.push(line)
+  }
+  return { vis: lines.join('\n'), cids }
+}
+
+export async function visMerkleClock (blocks, head) {
+  const lines = []
+  for await (const line of visClock(blocks, head)) {
+    // yield line
+    lines.push(line)
+  }
+  return { vis: lines.join('\n') }
+}