@fireproof/core 0.0.5 → 0.0.7

package/src/fireproof.js

@@ -1,5 +1,6 @@
-import { put, get, getAll, eventsSince } from './prolly.js'
-import Blockstore, { doTransaction } from './blockstore.js'
+import { vis, put, get, getAll, eventsSince } from './prolly.js'
+import TransactionBlockstore, { doTransaction } from './blockstore.js'
+import charwise from 'charwise'
 
 // const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
 
@@ -26,16 +27,17 @@ export default class Fireproof {
    * @static
    * @returns {Fireproof} - a new Fireproof instance
    */
-  static storage = () => {
-    return new Fireproof(new Blockstore(), [])
+  static storage = (name) => {
+    return new Fireproof(new TransactionBlockstore(name), [], { name })
   }
 
-  constructor (blocks, clock, config = {}, authCtx = {}) {
+  constructor (blocks, clock, config, authCtx = {}) {
+    this.name = config?.name || 'global'
+    this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`
     this.blocks = blocks
     this.clock = clock
     this.config = config
     this.authCtx = authCtx
-    this.instanceId = 'fp.' + Math.random().toString(36).substring(2, 7)
   }
 
   /**
@@ -49,7 +51,7 @@ export default class Fireproof {
    */
   snapshot (clock) {
     // how to handle listeners, views, and config?
-    // todo needs a test for that
+    // todo needs a test for listeners, views, and config
     return new Fireproof(this.blocks, clock || this.clock)
   }
 
@@ -89,11 +91,11 @@ export default class Fireproof {
    */
   async changesSince (event) {
     // console.log('changesSince', this.instanceId, event, this.clock)
-    let rows
+    let rows, dataCIDs, clockCIDs
     if (event) {
       const resp = await eventsSince(this.blocks, this.clock, event)
       const docsMap = new Map()
-      for (const { key, type, value } of resp) {
+      for (const { key, type, value } of resp.result.map(decodeEvent)) {
         if (type === 'del') {
           docsMap.set(key, { key, del: true })
         } else {
@@ -101,12 +103,19 @@ export default class Fireproof {
         }
       }
       rows = Array.from(docsMap.values())
+      clockCIDs = resp.cids
       // console.log('change rows', this.instanceId, rows)
     } else {
-      rows = (await getAll(this.blocks, this.clock)).map(({ key, value }) => ({ key, value }))
+      const allResp = await getAll(this.blocks, this.clock)
+      rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value })))
+      dataCIDs = allResp.cids
       // console.log('dbdoc rows', this.instanceId, rows)
     }
-    return { rows, clock: this.clock }
+    return {
+      rows,
+      clock: this.clock,
+      proof: { data: await cidsToProof(dataCIDs), clock: await cidsToProof(clockCIDs) }
+    }
   }
 
   /**
@@ -158,7 +167,7 @@ export default class Fireproof {
    * @memberof Fireproof
    * @instance
    */
-  async put ({ _id, ...doc }) {
+  async put ({ _id, _proof, ...doc }) {
     const id = _id || 'f' + Math.random().toString(36).slice(2)
     await this.#runValidation({ _id: id, ...doc })
     return await this.#putToProllyTree({ key: id, value: doc }, doc._clock)
@@ -192,12 +201,13 @@ export default class Fireproof {
    * @param {Object<{key : string, value: any}>} event - the event to add
    * @returns {Object<{ id: string, clock: CID[] }>} - The result of adding the event to storage
    */
-  async #putToProllyTree (event, clock = null) {
+  async #putToProllyTree (decodedEvent, clock = null) {
+    const event = encodeEvent(decodedEvent)
     if (clock && JSON.stringify(clock) !== JSON.stringify(this.clock)) {
       // we need to check and see what version of the document exists at the clock specified
       // if it is the same as the one we are trying to put, then we can proceed
       const resp = await eventsSince(this.blocks, this.clock, event.value._clock)
-      const missedChange = resp.find(({ key }) => key === event.key)
+      const missedChange = resp.result.find(({ key }) => key === event.key)
       if (missedChange) {
         throw new Error('MVCC conflict, document is changed, please reload the document and try again.')
       }
@@ -212,9 +222,13 @@ export default class Fireproof {
       throw new Error('failed to put at storage layer')
     }
     this.clock = result.head // do we want to do this as a finally block
-    result.id = event.key
-    await this.#notifyListeners([event])
-    return { id: result.id, clock: this.clock }
+    await this.#notifyListeners([decodedEvent]) // this type is odd
+    return {
+      id: decodedEvent.key,
+      clock: this.clock,
+      proof: { data: await cidsToProof(result.cids), clock: await cidsToProof(result.clockCIDs) }
+    }
+    // todo should include additions (or split clock)
   }
 
   //   /**
@@ -251,21 +265,27 @@ export default class Fireproof {
    * @instance
    */
   async get (key, opts = {}) {
-    let got
-    if (opts.clock) {
-      got = await get(this.blocks, opts.clock, key)
-    } else {
-      got = await get(this.blocks, this.clock, key)
-    }
+    const clock = opts.clock || this.clock
+    const resp = await get(this.blocks, clock, charwise.encode(key))
+
     // this tombstone is temporary until we can get the prolly tree to delete
-    if (got === null) {
+    if (!resp || resp.result === null) {
       throw new Error('Not found')
     }
+    const doc = resp.result
     if (opts.mvcc === true) {
-      got._clock = this.clock
+      doc._clock = this.clock
+    }
+    doc._proof = {
+      data: await cidsToProof(resp.cids),
+      clock: this.clock
     }
-    got._id = key
-    return got
+    doc._id = key
+    return doc
+  }
+
+  async * vis () {
+    return yield * vis(this.blocks, this.clock)
   }
 
   setCarUploader (carUploaderFn) {
@@ -279,3 +299,20 @@ export default class Fireproof {
     this.blocks.valet.remoteBlockFunction = remoteBlockReaderFn
   }
 }
+
+export async function cidsToProof (cids) {
+  if (!cids || !cids.all) return []
+  const all = await cids.all()
+  return [...all].map((cid) => cid.toString())
+}
+
+function decodeEvent (event) {
+  const decodedKey = charwise.decode(event.key)
+  return { ...event, key: decodedKey }
+}
+
+function encodeEvent (event) {
+  if (!(event && event.key)) return
+  const encodedKey = charwise.encode(event.key)
+  return { ...event, key: encodedKey }
+}

package/src/listener.js

@@ -11,12 +11,6 @@
 
 export default class Listener {
   #subcribers = new Map()
-
-  // todo code review if there is a better way that doesn't create a circular reference
-  // because otherwise we need to document that the user must call stopListening
-  // or else the listener will never be garbage collected
-  // maybe we can use WeakRef on the db side
-  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WeakRef
   #doStopListening = null
 
   constructor (database, routingFn) {

package/src/prolly.js

@@ -3,16 +3,15 @@ import {
   EventFetcher,
   EventBlock,
   findCommonAncestorWithSortedEvents,
-  findUnknownSortedEvents
+  findEventsToSync
 } from './clock.js'
 import { create, load } from 'prolly-trees/map'
+// import { create, load } from '../../../../prolly-trees/src/map.js'
+import { nocache as cache } from 'prolly-trees/cache'
+import { CIDCounter, bf, simpleCompare as compare } from 'prolly-trees/utils'
 import * as codec from '@ipld/dag-cbor'
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
-import { MemoryBlockstore, MultiBlockFetcher } from './block.js'
 import { doTransaction } from './blockstore.js'
-
-import { nocache as cache } from 'prolly-trees/cache'
-import { bf, simpleCompare as compare } from 'prolly-trees/utils'
 import { create as createBlock } from 'multiformats/block'
 const opts = { cache, chunker: bf(3), codec, hasher, compare }
 
@@ -22,9 +21,18 @@ const withLog = async (label, fn) => {
   return resp
 }
 
-const makeGetBlock = (blocks) => async (address) => {
-  const { cid, bytes } = await withLog(address, () => blocks.get(address))
-  return createBlock({ cid, bytes, hasher, codec })
+// should also return a CIDCounter
+export const makeGetBlock = (blocks) => {
+  // const cids = new CIDCounter() // this could be used for proofs of mutations
+  const getBlockFn = async (address) => {
+    const { cid, bytes } = await withLog(address, () => blocks.get(address))
+    // cids.add({ address: cid })
+    return createBlock({ cid, bytes, hasher, codec })
+  }
+  return {
+    // cids,
+    getBlock: getBlockFn
+  }
 }
 
 /**
@@ -46,17 +54,17 @@ const makeGetBlock = (blocks) => async (address) => {
  *   event: CID[]
  * }>}
  */
-async function createAndSaveNewEvent (
+async function createAndSaveNewEvent ({
   inBlocks,
-  mblocks,
-  getBlock,
   bigPut,
   root,
-  { key, value, del },
+  event: inEvent,
   head,
   additions,
   removals = []
-) {
+}) {
+  let cids
+  const { key, value, del } = inEvent
   const data = {
     type: 'put',
     root: {
@@ -77,32 +85,33 @@ async function createAndSaveNewEvent (
 
   const event = await EventBlock.create(data, head)
   bigPut(event)
-  head = await advance(inBlocks, head, event.cid)
+  ;({ head, cids } = await advance(inBlocks, head, event.cid))
 
   return {
     root,
     additions,
     removals,
     head,
+    clockCIDs: cids,
     event
   }
 }
 
 const makeGetAndPutBlock = (inBlocks) => {
-  const mblocks = new MemoryBlockstore()
-  const blocks = new MultiBlockFetcher(mblocks, inBlocks)
-  const getBlock = makeGetBlock(blocks)
+  // const mblocks = new MemoryBlockstore()
+  // const blocks = new MultiBlockFetcher(mblocks, inBlocks)
+  const { getBlock, cids } = makeGetBlock(inBlocks)
   const put = inBlocks.put.bind(inBlocks)
   const bigPut = async (block, additions) => {
     // console.log('bigPut', block.cid.toString())
     const { cid, bytes } = block
     put(cid, bytes)
-    mblocks.putSync(cid, bytes)
+    // mblocks.putSync(cid, bytes)
     if (additions) {
       additions.set(cid.toString(), block)
     }
   }
-  return { getBlock, bigPut, mblocks, blocks }
+  return { getBlock, bigPut, blocks: inBlocks, cids }
 }
 
 const bulkFromEvents = (sorted) =>
@@ -132,7 +141,7 @@ const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
 /**
  * Put a value (a CID) for the given key. If the key exists it's value is overwritten.
  *
- * @param {import('./block').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  * @param {string} key The key of the value to put.
  * @param {CID} value The value to put.
@@ -140,7 +149,7 @@ const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
  * @returns {Promise<Result>}
  */
 export async function put (inBlocks, head, event, options) {
-  const { getBlock, bigPut, mblocks, blocks } = makeGetAndPutBlock(inBlocks)
+  const { getBlock, bigPut, blocks } = makeGetAndPutBlock(inBlocks)
 
   // If the head is empty, we create a new event and return the root and addition blocks
   if (!head.length) {
@@ -150,12 +159,14 @@ export async function put (inBlocks, head, event, options) {
       root = await node.block
       bigPut(root, additions)
     }
-    return createAndSaveNewEvent(inBlocks, mblocks, getBlock, bigPut, root, event, head, Array.from(additions.values()))
+    return createAndSaveNewEvent({ inBlocks, bigPut, root, event, head, additions: Array.from(additions.values()) })
   }
 
   // Otherwise, we find the common ancestor and update the root and other blocks
   const events = new EventFetcher(blocks)
+  // this is returning more events than necessary
   const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head)
+  // console.log('sorted', JSON.stringify(sorted.map(({ value: { data: { key, value } } }) => ({ key, value }))))
   const prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock)
 
   const bulkOperations = bulkFromEvents(sorted)
@@ -166,23 +177,21 @@ export async function put (inBlocks, head, event, options) {
   for (const nb of newBlocks) {
     bigPut(nb, additions)
   }
-
-  return createAndSaveNewEvent(
+  // additions are new blocks
+  return createAndSaveNewEvent({
     inBlocks,
-    mblocks,
-    getBlock,
     bigPut,
-    prollyRootBlock,
+    root: prollyRootBlock,
     event,
     head,
-    Array.from(additions.values()) /*, Array.from(removals.values()) */
-  )
+    additions: Array.from(additions.values()) /*, todo? Array.from(removals.values()) */
+  })
 }
 
 /**
  * Determine the effective prolly root given the current merkle clock head.
  *
- * @param {import('./block').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  */
 export async function root (inBlocks, head) {
@@ -197,23 +206,22 @@ export async function root (inBlocks, head) {
   // Perform bulk operations (put or delete) for each event in the sorted array
   const bulkOperations = bulkFromEvents(sorted)
   const { root: newProllyRootNode, blocks: newBlocks } = await prollyRootNode.bulk(bulkOperations)
-  const prollyRootBlock = await newProllyRootNode.block
-  // console.log('emphemeral blocks', newBlocks.map((nb) => nb.cid.toString()))
+  // const prollyRootBlock = await newProllyRootNode.block
+  // console.log('newBlocks', newBlocks.map((nb) => nb.cid.toString()))
   // todo maybe these should go to a temp blockstore?
   await doTransaction('root', inBlocks, async (transactionBlockstore) => {
     const { bigPut } = makeGetAndPutBlock(transactionBlockstore)
     for (const nb of newBlocks) {
       bigPut(nb)
     }
-    bigPut(prollyRootBlock)
+    // bigPut(prollyRootBlock)
   })
-
-  return newProllyRootNode // .block).cid // todo return live object not cid
+  return { cids: events.cids, node: newProllyRootNode }
 }
 
 /**
  * Get the list of events not known by the `since` event
- * @param {import('./block').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  * @param {import('./clock').EventLink<EventData>} since Event to compare against.
  * @returns {Promise<import('./clock').EventLink<EventData>[]>}
@@ -223,17 +231,13 @@ export async function eventsSince (blocks, head, since) {
     throw new Error('no head')
   }
   const sinceHead = [...since, ...head]
-  const unknownSorted3 = await findUnknownSortedEvents(
-    blocks,
-    sinceHead,
-    await findCommonAncestorWithSortedEvents(blocks, sinceHead)
-  )
-  return unknownSorted3.map(({ value: { data } }) => data)
+  const { cids, events: unknownSorted3 } = await findEventsToSync(blocks, sinceHead)
+  return { clockCIDs: cids, result: unknownSorted3.map(({ value: { data } }) => data) }
 }
 
 /**
  *
- * @param {import('./block').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  *
  * @returns {Promise<import('./prolly').Entry[]>}
@@ -243,24 +247,36 @@ export async function getAll (blocks, head) {
   // todo use the root node left around from put, etc
   // move load to a central place
   if (!head.length) {
-    return []
+    return { clockCIDs: new CIDCounter(), cids: new CIDCounter(), result: [] }
   }
-  const prollyRootNode = await root(blocks, head)
-  const { result } = await prollyRootNode.getAllEntries()
-  return result.map(({ key, value }) => ({ key, value }))
+  const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head)
+  const { result, cids } = await prollyRootNode.getAllEntries() // todo params
+  return { clockCIDs, cids, result: result.map(({ key, value }) => ({ key, value })) }
 }
 
 /**
- * @param {import('./block').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  * @param {string} key The key of the value to retrieve.
  */
 export async function get (blocks, head, key) {
   // instead pass root from db? and always update on change
   if (!head.length) {
-    return null
+    return { cids: new CIDCounter(), result: null }
+  }
+  const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head)
+  const { result, cids } = await prollyRootNode.get(key)
+  return { result, cids, clockCIDs }
+}
+
+export async function * vis (blocks, head) {
+  if (!head.length) {
+    return { cids: new CIDCounter(), result: null }
+  }
+  const { node: prollyRootNode, cids } = await root(blocks, head)
+  const lines = []
+  for await (const line of prollyRootNode.vis()) {
+    yield line
   }
-  const prollyRootNode = await root(blocks, head)
-  const { result } = await prollyRootNode.get(key)
-  return result
+  return { vis: lines.join('\n'), cids }
 }

package/src/valet.js

@@ -3,17 +3,8 @@ import { CID } from 'multiformats/cid'
 import { openDB } from 'idb'
 import cargoQueue from 'async/cargoQueue.js'
 
-// const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
-// let storageSupported = false
-// try {
-//   storageSupported = window.localStorage && true
-// } catch (e) {}
-// const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
-
 export default class Valet {
-  #cars = new Map() // cars by cid
-  #cidToCar = new Map() // cid to car
-  #db = null
+  idb = null
   #uploadQueue = null
   #alreadyEnqueued = new Set()
 
@@ -23,7 +14,8 @@ export default class Valet {
    */
   uploadFunction = null
 
-  constructor () {
+  constructor (name = 'default') {
+    this.name = name
     this.#uploadQueue = cargoQueue(async (tasks, callback) => {
       console.log(
         'queue worker',
@@ -59,9 +51,8 @@ export default class Valet {
   }
 
   withDB = async (dbWorkFun) => {
-    // if (!storageSupported) return
-    if (!this.#db) {
-      this.#db = await openDB('valet', 2, {
+    if (!this.idb) {
+      this.idb = await openDB(`fp.${this.name}.valet`, 2, {
         upgrade (db, oldVersion, newVersion, transaction) {
           if (oldVersion < 1) {
             db.createObjectStore('cars') // todo use database name
@@ -75,7 +66,7 @@ export default class Valet {
         }
       })
     }
-    return await dbWorkFun(this.#db)
+    return await dbWorkFun(this.idb)
   }
 
   /**
@@ -84,11 +75,6 @@ export default class Valet {
    * @param {*} value
    */
   async parkCar (carCid, value, cids) {
-    // this.#cars.set(carCid, value)
-    // for (const cid of cids) {
-    //   this.#cidToCar.set(cid, carCid)
-    // }
-
     await this.withDB(async (db) => {
       const tx = db.transaction(['cars', 'cidToCar'], 'readwrite')
       await tx.objectStore('cars').put(value, carCid)
@@ -130,45 +116,3 @@ export default class Valet {
     })
   }
 }
-
-// export class MemoryValet {
-//   #cars = new Map() // cars by cid
-//   #cidToCar = new Map() // cid to car
-
-//   /**
-//    *
-//    * @param {string} carCid
-//    * @param {*} value
-//    */
-//   async parkCar (carCid, value, cids) {
-//     this.#cars.set(carCid, value)
-//     for (const cid of cids) {
-//       this.#cidToCar.set(cid, carCid)
-//     }
-//   }
-
-//   async getBlock (dataCID) {
-//     return await this.#valetGet(dataCID)
-//   }
-
-//   /**
-//    * Internal function to load blocks from persistent storage.
-//    * Currently it just searches all the cars for the block, but in the future
-//    * we need to index the block CIDs to the cars, and reference that to find the block.
-//    * This index will also allow us to use accelerator links for the gateway when needed.
-//    * It can itself be a prolly tree...
-//    * @param {string} cid
-//    * @returns {Promise<Uint8Array|undefined>}
-//    */
-//   #valetGet = async (cid) => {
-//     const carCid = this.#cidToCar.get(cid)
-//     if (carCid) {
-//       const carBytes = this.#cars.get(carCid)
-//       const reader = await CarReader.fromBytes(carBytes)
-//       const gotBlock = await reader.get(CID.parse(cid))
-//       if (gotBlock) {
-//         return gotBlock.bytes
-//       }
-//     }
-//   }
-// }

package/{src → test}/block.js

@@ -1,8 +1,8 @@
 import { parse } from 'multiformats/link'
 
 /**
- * @typedef {{ cid: import('./link').AnyLink, bytes: Uint8Array }} AnyBlock
- * @typedef {{ get: (link: import('./link').AnyLink) => Promise<AnyBlock | undefined> }} BlockFetcher
+ * @typedef {{ cid: import('../src/link').AnyLink, bytes: Uint8Array }} AnyBlock
+ * @typedef {{ get: (link: import('../src/link').AnyLink) => Promise<AnyBlock | undefined> }} BlockFetcher
  */
 
 /** @implements {BlockFetcher} */
@@ -11,7 +11,7 @@ export class MemoryBlockstore {
   #blocks = new Map()
 
   /**
-   * @param {import('./link').AnyLink} cid
+   * @param {import('../src/link').AnyLink} cid
    * @returns {Promise<AnyBlock | undefined>}
    */
   async get (cid) {
@@ -21,7 +21,7 @@ export class MemoryBlockstore {
   }
 
   /**
-   * @param {import('./link').AnyLink} cid
+   * @param {import('../src/link').AnyLink} cid
    * @param {Uint8Array} bytes
    */
   async put (cid, bytes) {
@@ -30,7 +30,7 @@ export class MemoryBlockstore {
   }
 
   /**
-   * @param {import('./link').AnyLink} cid
+   * @param {import('../src/link').AnyLink} cid
    * @param {Uint8Array} bytes
    */
   putSync (cid, bytes) {
@@ -53,7 +53,7 @@ export class MultiBlockFetcher {
     this.#fetchers = fetchers
   }
 
-  /** @param {import('./link').AnyLink} link */
+  /** @param {import('../src/link').AnyLink} link */
   async get (link) {
     for (const f of this.#fetchers) {
       const v = await f.get(link)