@fireproof/core 0.1.0 → 0.2.0

package/README.md

@@ -96,7 +96,7 @@ Documents changes are persisted to [Filecoin](https://filecoin.io) via [web3.sto
 
 ### Cryptographic Proofs
 
-The [UCAN protocol](https://ucan.xyz) verifably links Fireproof updates to authorized agents via cryptographic proof chains. These proofs are portable like bearer tokens, but because invocations are signed by end-user device keys, UCAN proofs don't need to be hidden to be secure, allowing for delegation of service capabilities across devices and parties. Additionally, Fireproof's Merkle clocks and hash trees are immutable and self-validating, making merging changes safe and efficient. Fireproof makes cryptographic proofs available for all of it's operations, making it an ideal verfiable document database for smart contracts and other applications running in trustless environments. [Proof chains provide performance benefits as well](https://purrfect-tracker-45c.notion.site/Data-Routing-23c37b269b4c4c3dacb60d0077113bcb), by allowing recipients to skip costly I/O operations and instead cryptographically verify that changes contain all of the required context.
+The [UCAN protocol](https://ucan.xyz) verifiably links Fireproof updates to authorized agents via cryptographic proof chains. These proofs are portable like bearer tokens, but because invocations are signed by end-user device keys, UCAN proofs don't need to be hidden to be secure, allowing for delegation of service capabilities across devices and parties. Additionally, Fireproof's Merkle clocks and hash trees are immutable and self-validating, making merging changes safe and efficient. Fireproof makes cryptographic proofs available for all of its operations, making it an ideal verifiable document database for smart contracts and other applications running in trustless environments. [Proof chains provide performance benefits as well](https://purrfect-tracker-45c.notion.site/Data-Routing-23c37b269b4c4c3dacb60d0077113bcb), by allowing recipients to skip costly I/O operations and instead cryptographically verify that changes contain all of the required context.
 
 ## Limitations 💣
 

package/hooks/use-fireproof.ts

@@ -4,44 +4,53 @@
 import { useEffect, useState, createContext } from 'react'
 import { Fireproof, Listener, Hydrator } from '../index'
 
-
 export interface FireproofCtxValue {
   addSubscriber: (label: String, fn: Function) => void
   database: Fireproof
   ready: boolean
+  persist: () => void
 }
 export const FireproofCtx = createContext<FireproofCtxValue>({
   addSubscriber: () => {},
   database: null,
-  ready: false,
+  ready: false
 })
 
 const inboundSubscriberQueue = new Map()
-const database = Fireproof.storage()
-const listener = new Listener(database)
-let startedSetup = false;
+
+let startedSetup = false
+let database
+let listener
+const initializeDatabase = name => {
+  if (database) return
+  database = Fireproof.storage(name)
+  listener = new Listener(database)
+}
 
 /**
  * @function useFireproof
  * React hook to initialize a Fireproof database, automatically saving and loading the clock.
+ * You might need to `import { nodePolyfills } from 'vite-plugin-node-polyfills'` in your vite.config.ts
  * @param [defineDatabaseFn] Synchronous function that defines the database, run this before any async calls
  * @param [setupDatabaseFn] Asynchronous function that sets up the database, run this to load fixture data etc
  * @returns {FireproofCtxValue} { addSubscriber, database, ready }
  */
-export function useFireproof(defineDatabaseFn: Function, setupDatabaseFn: Function): FireproofCtxValue {
+export function useFireproof(
+  defineDatabaseFn = (database: Fireproof) => {},
+  setupDatabaseFn = async (database: Fireproof) => {},
+  name: string
+): FireproofCtxValue {
   const [ready, setReady] = useState(false)
-  defineDatabaseFn = defineDatabaseFn || (() => {})
-  setupDatabaseFn = setupDatabaseFn || (() => {})
-  // console.log('useFireproof', database, ready)
-
+  initializeDatabase(name || 'useFireproof')
+  const localStorageKey = 'fp.' + database.name
 
   const addSubscriber = (label: String, fn: Function) => {
     inboundSubscriberQueue.set(label, fn)
   }
 
-  const listenerCallback = async () => {
-    // console.log ('listenerCallback', JSON.stringify(database))
-    localSet('fireproof', JSON.stringify(database))
+  const listenerCallback = async event => {
+      localSet(localStorageKey, JSON.stringify(database))
+    if (event._external) return
     for (const [, fn] of inboundSubscriberQueue) fn()
   }
 
@@ -51,14 +60,14 @@ export function useFireproof(defineDatabaseFn: Function, setupDatabaseFn: Functi
       if (startedSetup) return
       startedSetup = true
       defineDatabaseFn(database) // define indexes before querying them
-      const fp = localGet('fireproof')
+      console.log('Initializing database', database.name)
+      const fp = localGet(localStorageKey) // todo use db.name
       if (fp) {
-        const serialized = JSON.parse(fp)
-        // console.log('serialized', JSON.stringify(serialized.indexes.map(c => c.clock)))
-        console.log("Loading previous database clock. (localStorage.removeItem('fireproof') to reset)")
-        Hydrator.fromJSON(serialized, database)
-        // await database.setClock(clock)
         try {
+          const serialized = JSON.parse(fp)
+          // console.log('serialized', JSON.stringify(serialized.indexes.map(c => c.clock)))
+          console.log(`Loading previous database clock. (localStorage.removeItem('${localStorageKey}') to reset)`)
+          await Hydrator.fromJSON(serialized, database)
           const changes = await database.changesSince()
           if (changes.rows.length < 2) {
             // console.log('Resetting database')
@@ -66,16 +75,16 @@ export function useFireproof(defineDatabaseFn: Function, setupDatabaseFn: Functi
           }
         } catch (e) {
           console.error(`Error loading previous database clock. ${fp} Resetting.`, e)
-          await database.setClock([]) // todo this should be resetClock and also reset the indexes
+          await Hydrator.zoom(database, [])
           await setupDatabaseFn(database)
-          localSet('fireproof', JSON.stringify(database))
+          localSet(localStorageKey, JSON.stringify(database))
         }
       } else {
         await setupDatabaseFn(database)
-        localSet('fireproof', JSON.stringify(database))
+        localSet(localStorageKey, JSON.stringify(database))
       }
       setReady(true)
-      listener.on('*', hushed('*', listenerCallback, 250))
+      listener.on('*', listenerCallback)//hushed('*', listenerCallback, 250))
     }
     doSetup()
   }, [ready])
@@ -84,6 +93,9 @@ export function useFireproof(defineDatabaseFn: Function, setupDatabaseFn: Functi
     addSubscriber,
     database,
     ready,
+    persist: () => {
+      localSet(localStorageKey, JSON.stringify(database))
+    }
   }
 }
 
@@ -91,12 +103,17 @@ const husherMap = new Map()
 const husher = (id: string, workFn: { (): Promise<any> }, ms: number) => {
   if (!husherMap.has(id)) {
     const start: number = Date.now()
-    husherMap.set(id, workFn().finally(() => 
-      setTimeout(() => husherMap.delete(id), ms - (Date.now() - start))))
+    husherMap.set(
+      id,
+      workFn().finally(() => setTimeout(() => husherMap.delete(id), ms - (Date.now() - start)))
+    )
   }
   return husherMap.get(id)
 }
-const hushed = (id: string, workFn: { (): Promise<any> }, ms: number) => () => husher(id, workFn, ms)
+const hushed =
+  (id: string, workFn: { (...args): Promise<any> }, ms: number) =>
+  (...args) =>
+    husher(id, () => workFn(...args), ms)
 
 let storageSupported = false
 try {
@@ -116,4 +133,4 @@ function localSet(key: string, value: string) {
 //   if (storageSupported) {
 //     return localStorage && localStorage.removeItem(key)
 //   }
-// }
+// }

package/package.json

@@ -1,15 +1,17 @@
 {
   "name": "@fireproof/core",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Realtime database for IPFS",
   "main": "index.js",
   "type": "module",
   "scripts": {
-    "test": "standard && npm run test:mocha",
-    "test:mocha": "mocha test/*.test.js",
-    "test:watch": "npm run test:mocha -- -w --parallel test/*.test.js",
+    "keygen": "node scripts/keygen.js",
+    "test": "standard && npm run test:unencrypted && npm run test:mocha",
+    "test:unencrypted": "NO_ENCRYPT=true npm run test:mocha",
+    "test:mocha": "mocha --reporter list test/*.test.js",
+    "test:watch": "npm run test:mocha -- -w --parallel",
     "coverage": "c8 -r html -r text npm test",
-    "prepublishOnly" : "cp ../../README.md .",
+    "prepublishOnly": "cp ../../README.md .",
     "postpublish": "rm README.md",
     "lint": "standard",
     "lint:fix": "standard --fix"
@@ -37,6 +39,7 @@
     "car-transaction": "^1.0.1",
     "charwise": "^3.0.1",
     "cli-color": "^2.0.3",
+    "encrypted-block": "^0.0.3",
     "idb": "^7.1.1",
     "multiformats": "^11.0.1",
     "prolly-trees": "1.0.3",
@@ -69,7 +72,7 @@
   "bugs": {
     "url": "https://github.com/fireproof-storage/fireproof/issues"
   },
-  "homepage": "https://github.com/fireproof-storage/fireproof#readme",
+  "homepage": "https://fireproof.storage",
   "workspaces": [
     "examples/todomvc"
   ]

package/scripts/keygen.js

@@ -0,0 +1,3 @@
+import { randomBytes } from 'crypto'
+
+console.log(randomBytes(32).toString('hex'))

package/src/blockstore.js

@@ -1,10 +1,5 @@
 import { parse } from 'multiformats/link'
-import * as raw from 'multiformats/codecs/raw'
-import { sha256 } from 'multiformats/hashes/sha2'
-import * as Block from 'multiformats/block'
-import * as CBW from '@ipld/car/buffer-writer'
 import { CID } from 'multiformats'
-
 import Valet from './valet.js'
 
 // const sleep = ms => new Promise(r => setTimeout(r, ms))
@@ -34,15 +29,15 @@ const husher = (id, workFn) => {
  */
 export default class TransactionBlockstore {
   /** @type {Map<string, Uint8Array>} */
-  #oldBlocks = new Map()
+  #committedBlocks = new Map()
 
   valet = null
 
   #instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4)
   #inflightTransactions = new Set()
 
-  constructor (name) {
-    this.valet = new Valet(name)
+  constructor (name, encryptionKey) {
+    this.valet = new Valet(name, encryptionKey)
   }
 
   /**
@@ -54,7 +49,7 @@ export default class TransactionBlockstore {
   async get (cid) {
     const key = cid.toString()
     // it is safe to read from the in-flight transactions becauase they are immutable
-    const bytes = await Promise.any([this.#transactionsGet(key), this.commitedGet(key)]).catch((e) => {
+    const bytes = await Promise.any([this.#transactionsGet(key), this.committedGet(key)]).catch(e => {
       // console.log('networkGet', cid.toString(), e)
       return this.networkGet(key)
     })
@@ -72,18 +67,26 @@ export default class TransactionBlockstore {
     throw new Error('Missing block: ' + key)
   }
 
-  async commitedGet (key) {
-    const old = this.#oldBlocks.get(key)
+  async committedGet (key) {
+    const old = this.#committedBlocks.get(key)
     if (old) return old
-    return await this.valet.getBlock(key)
+    const got = await this.valet.getBlock(key)
+    // console.log('committedGet: ' + key)
+    this.#committedBlocks.set(key, got)
+    return got
+  }
+
+  async clearCommittedCache () {
+    this.#committedBlocks.clear()
   }
 
   async networkGet (key) {
     if (this.valet.remoteBlockFunction) {
+      // todo why is this on valet?
       const value = await husher(key, async () => await this.valet.remoteBlockFunction(key))
       if (value) {
         // console.log('networkGot: ' + key, value.length)
-        doTransaction('networkGot: ' + key, this, async (innerBlockstore) => {
+        doTransaction('networkGot: ' + key, this, async innerBlockstore => {
           await innerBlockstore.put(CID.parse(key), value)
         })
         return value
@@ -118,7 +121,7 @@ export default class TransactionBlockstore {
   //   // for (const [str, bytes] of this.#blocks) {
   //   //   yield { cid: parse(str), bytes }
   //   // }
-  //   for (const [str, bytes] of this.#oldBlocks) {
+  //   for (const [str, bytes] of this.#committedBlocks) {
   //     yield { cid: parse(str), bytes }
   //   }
   // }
@@ -145,39 +148,24 @@ export default class TransactionBlockstore {
   }
 
   // first get the transaction blockstore from the map of transaction blockstores
-  // then copy it to oldBlocks
+  // then copy it to committedBlocks
   // then write the transaction blockstore to a car
   // then write the car to the valet
   // then remove the transaction blockstore from the map of transaction blockstores
-  #doCommit = async (innerBlockstore) => {
+  #doCommit = async innerBlockstore => {
     const cids = new Set()
     for (const { cid, bytes } of innerBlockstore.entries()) {
       const stringCid = cid.toString() // unnecessary string conversion, can we fix upstream?
-      if (this.#oldBlocks.has(stringCid)) {
-        // console.log('Duplicate block: ' + stringCid)
+      if (this.#committedBlocks.has(stringCid)) {
+        // console.log('Duplicate block: ' + stringCid) // todo some of this can be avoided, cost is extra size on car files
       } else {
-        this.#oldBlocks.set(stringCid, bytes)
+        this.#committedBlocks.set(stringCid, bytes)
         cids.add(stringCid)
       }
     }
     if (cids.size > 0) {
       // console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
-      await this.#valetWriteTransaction(innerBlockstore, cids)
-    }
-  }
-
-  /**
-   * Group the blocks into a car and write it to the valet.
-   * @param {InnerBlockstore} innerBlockstore
-   * @param {Set<string>} cids
-   * @returns {Promise<void>}
-   * @memberof TransactionBlockstore
-   * @private
-   */
-  #valetWriteTransaction = async (innerBlockstore, cids) => {
-    if (innerBlockstore.lastCid) {
-      const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
-      await this.valet.parkCar(newCar.cid.toString(), newCar.bytes, cids)
+      await this.valet.writeTransaction(innerBlockstore, cids)
     }
   }
 
@@ -215,25 +203,6 @@ export const doTransaction = async (label, blockstore, doFun) => {
   }
 }
 
-const blocksToCarBlock = async (lastCid, blocks) => {
-  let size = 0
-  const headerSize = CBW.headerLength({ roots: [lastCid] })
-  size += headerSize
-  for (const { cid, bytes } of blocks.entries()) {
-    size += CBW.blockLength({ cid, bytes })
-  }
-  const buffer = new Uint8Array(size)
-  const writer = await CBW.createWriter(buffer, { headerSize })
-
-  writer.addRoot(lastCid)
-
-  for (const { cid, bytes } of blocks.entries()) {
-    writer.write({ cid, bytes })
-  }
-  await writer.close()
-  return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
-}
-
 /** @implements {BlockFetcher} */
 export class InnerBlockstore {
   /** @type {Map<string, Uint8Array>} */
@@ -254,8 +223,10 @@ export class InnerBlockstore {
   async get (cid) {
     const key = cid.toString()
     let bytes = this.#blocks.get(key)
-    if (bytes) { return { cid, bytes } }
-    bytes = await this.parentBlockstore.commitedGet(key)
+    if (bytes) {
+      return { cid, bytes }
+    }
+    bytes = await this.parentBlockstore.committedGet(key)
     if (bytes) {
       return { cid, bytes }
     }

package/src/clock.js

@@ -220,7 +220,6 @@ export async function findEventsToSync (blocks, head) {
 }
 
 const asyncFilter = async (arr, predicate) =>
-
   Promise.all(arr.map(predicate)).then((results) => arr.filter((_v, index) => results[index]))
 
 export async function findCommonAncestorWithSortedEvents (events, children) {

package/src/crypto.js

@@ -0,0 +1,65 @@
+import * as codec from 'encrypted-block'
+import {
+  create,
+  load
+} from 'prolly-trees/cid-set'
+import { CID } from 'multiformats'
+import { encode, decode, create as mfCreate } from 'multiformats/block'
+import * as dagcbor from '@ipld/dag-cbor'
+import { sha256 as hasher } from 'multiformats/hashes/sha2'
+
+const createBlock = (bytes, cid) => mfCreate({ cid, bytes, hasher, codec })
+
+const encrypt = async function * ({ get, cids, hasher, key, cache, chunker, root }) {
+  const set = new Set()
+  let eroot
+  for (const string of cids) {
+    const cid = CID.parse(string)
+    const unencrypted = await get(cid)
+    const block = await encode({ ...await codec.encrypt({ ...unencrypted, key }), codec, hasher })
+    // console.log(`encrypting ${string} as ${block.cid}`)
+    yield block
+    set.add(block.cid.toString())
+    if (unencrypted.cid.equals(root)) eroot = block.cid
+  }
+  if (!eroot) throw new Error('cids does not include root')
+  const list = [...set].map(s => CID.parse(s))
+  let last
+  for await (const node of create({ list, get, cache, chunker, hasher, codec: dagcbor })) {
+    const block = await node.block
+    yield block
+    last = block
+  }
+  const head = [eroot, last.cid]
+  const block = await encode({ value: head, codec: dagcbor, hasher })
+  yield block
+}
+
+const decrypt = async function * ({ root, get, key, cache, chunker, hasher }) {
+  const o = { ...await get(root), codec: dagcbor, hasher }
+  const decodedRoot = await decode(o)
+  // console.log('decodedRoot', decodedRoot)
+  const { value: [eroot, tree] } = decodedRoot
+  const rootBlock = await get(eroot) // should I decrypt?
+  const cidset = await load({ cid: tree, get, cache, chunker, codec, hasher })
+  const { result: nodes } = await cidset.getAllEntries()
+  const unwrap = async (eblock) => {
+    const { bytes, cid } = await codec.decrypt({ ...eblock, key }).catch(e => {
+      console.log('ekey', e)
+      throw new Error('bad key: ' + key.toString('hex'))
+    })
+    const block = await createBlock(bytes, cid)
+    return block
+  }
+  const promises = []
+  for (const { cid } of nodes) {
+    if (!rootBlock.cid.equals(cid)) promises.push(get(cid).then(unwrap))
+  }
+  yield * promises
+  yield unwrap(rootBlock)
+}
+
+export {
+  encrypt,
+  decrypt
+}

package/src/db-index.js

@@ -9,7 +9,7 @@ import { cidsToProof } from './fireproof.js'
 
 import * as codec from '@ipld/dag-cbor'
 // import { create as createBlock } from 'multiformats/block'
-import { doTransaction } from './blockstore.js'
+import TransactionBlockstore, { doTransaction } from './blockstore.js'
 import charwise from 'charwise'
 
 const ALWAYS_REBUILD = false // todo: make false
@@ -82,6 +82,7 @@ const indexEntriesForChanges = (changes, mapFn) => {
   changes.forEach(({ key, value, del }) => {
     if (del || !value) return
     mapFn(makeDoc({ key, value }), (k, v) => {
+      if (typeof v === 'undefined' || typeof k === 'undefined') return
       indexEntries.push({
         key: [charwise.encode(k), key],
         value: v
@@ -102,13 +103,16 @@ const indexEntriesForChanges = (changes, mapFn) => {
  *
  */
 export default class DbIndex {
-  constructor (database, mapFn, clock) {
+  constructor (database, mapFn, clock, opts = {}) {
     // console.log('DbIndex constructor', database.constructor.name, typeof mapFn, clock)
     /**
      * The database instance to DbIndex.
      * @type {Fireproof}
      */
     this.database = database
+    if (!database.indexBlocks) {
+      database.indexBlocks = new TransactionBlockstore(database.name + '.indexes', database.blocks.valet.getKeyMaterial())
+    }
     /**
      * The map function to apply to each entry in the database.
      * @type {Function}
@@ -130,7 +134,7 @@ export default class DbIndex {
     }
     this.instanceId = this.database.instanceId + `.DbIndex.${Math.random().toString(36).substring(2, 7)}`
     this.updateIndexPromise = null
-    DbIndex.registerWithDatabase(this, this.database)
+    if (!opts.temporary) { DbIndex.registerWithDatabase(this, this.database) }
   }
 
   static registerWithDatabase (inIndex, database) {
@@ -159,7 +163,7 @@ export default class DbIndex {
   }
 
   toJSON () {
-    const indexJson = { code: this.mapFn?.toString(), clock: { db: null, byId: null, byKey: null } }
+    const indexJson = { code: this.mapFnString, clock: { db: null, byId: null, byKey: null } }
     indexJson.clock.db = this.dbHead?.map(cid => cid.toString())
     indexJson.clock.byId = this.indexById.cid?.toString()
     indexJson.clock.byKey = this.indexByKey.cid?.toString()
@@ -185,18 +189,18 @@ export default class DbIndex {
    * @memberof DbIndex
    * @instance
    */
-  async query (query) {
+  async query (query, update = true) {
     // const callId = Math.random().toString(36).substring(2, 7)
     // if (!root) {
     // pass a root to query a snapshot
     // console.time(callId + '.#updateIndex')
-    await this.#updateIndex(this.database.blocks)
+    update && await this.#updateIndex(this.database.indexBlocks)
     // console.timeEnd(callId + '.#updateIndex')
 
     // }
     // console.time(callId + '.doIndexQuery')
     // console.log('query', query)
-    const response = await doIndexQuery(this.database.blocks, this.indexByKey, query)
+    const response = await doIndexQuery(this.database.indexBlocks, this.indexByKey, query)
     // console.timeEnd(callId + '.doIndexQuery')
 
     return {
@@ -266,6 +270,7 @@ export default class DbIndex {
       this.indexByKey = await bulkIndex(blocks, this.indexByKey, oldIndexEntries.concat(indexEntries), dbIndexOpts)
       this.dbHead = result.clock
     })
+    this.database.notifyExternal('dbIndex')
     // console.timeEnd(callTag + '.doTransaction#updateIndex')
     // console.log(`#updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
   }
@@ -317,7 +322,7 @@ async function loadIndex (blocks, index, indexOpts) {
   return index.root
 }
 
-async function doIndexQuery (blocks, indexByKey, query) {
+async function doIndexQuery (blocks, indexByKey, query = {}) {
   await loadIndex(blocks, indexByKey, dbIndexOpts)
   if (query.range) {
     const encodedRange = query.range.map((key) => charwise.encode(key))
@@ -325,5 +330,12 @@ async function doIndexQuery (blocks, indexByKey, query) {
   } else if (query.key) {
     const encodedKey = charwise.encode(query.key)
     return indexByKey.root.get(encodedKey)
+  } else {
+    if (indexByKey.root) {
+      const { result, ...all } = await indexByKey.root.getAllEntries()
+      return { result: result.map(({ key: [k, id], value }) => ({ key: k, id, row: value })), ...all }
+    } else {
+      return { result: [] }
+    }
   }
 }

package/src/fireproof.js

@@ -1,4 +1,5 @@
-import { vis, put, get, getAll, eventsSince } from './prolly.js'
+import { randomBytes } from 'crypto'
+import { visMerkleClock, visMerkleTree, vis, put, get, getAll, eventsSince } from './prolly.js'
 import TransactionBlockstore, { doTransaction } from './blockstore.js'
 import charwise from 'charwise'
 
@@ -27,8 +28,11 @@ export default class Fireproof {
    * @static
    * @returns {Fireproof} - a new Fireproof instance
    */
-  static storage = (name) => {
-    return new Fireproof(new TransactionBlockstore(name), [], { name })
+  static storage = (name = 'global') => {
+    const instanceKey = randomBytes(32).toString('hex') // pass null to disable encryption
+    // pick a random key from const validatedKeys
+    // const instanceKey = validatedKeys[Math.floor(Math.random() * validatedKeys.length)]
+    return new Fireproof(new TransactionBlockstore(name, instanceKey), [], { name })
   }
 
   constructor (blocks, clock, config, authCtx = {}) {
@@ -50,15 +54,22 @@ export default class Fireproof {
   toJSON () {
     // todo this also needs to return the index roots...
     return {
-      clock: this.clock.map(cid => cid.toString()),
+      clock: this.clockToJSON(),
       name: this.name,
+      key: this.blocks.valet.getKeyMaterial(),
       indexes: [...this.indexes.values()].map(index => index.toJSON())
     }
   }
 
-  hydrate ({ clock, name }) {
+  clockToJSON () {
+    return this.clock.map(cid => cid.toString())
+  }
+
+  hydrate ({ clock, name, key }) {
     this.name = name
     this.clock = clock
+    this.blocks.valet.setKeyMaterial(key)
+    this.indexBlocks = null
   }
 
   /**
@@ -71,7 +82,12 @@ export default class Fireproof {
    * @instance
    */
   async notifyReset () {
-    await this.#notifyListeners({ reset: true, clock: this.clock })
+    await this.#notifyListeners({ _reset: true, _clock: this.clockToJSON() })
+  }
+
+  // used be indexes etc to notify database listeners of new availability
+  async notifyExternal (source = 'unknown') {
+    await this.#notifyListeners({ _external: source, _clock: this.clockToJSON() })
   }
 
   /**
@@ -85,6 +101,7 @@ export default class Fireproof {
   async changesSince (event) {
     // console.log('changesSince', this.instanceId, event, this.clock)
     let rows, dataCIDs, clockCIDs
+    // if (!event) event = []
     if (event) {
       const resp = await eventsSince(this.blocks, this.clock, event)
       const docsMap = new Map()
@@ -106,11 +123,21 @@ export default class Fireproof {
     }
     return {
       rows,
-      clock: this.clock,
+      clock: this.clockToJSON(),
       proof: { data: await cidsToProof(dataCIDs), clock: await cidsToProof(clockCIDs) }
     }
   }
 
+  async allDocuments () {
+    const allResp = await getAll(this.blocks, this.clock)
+    const rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value }))).map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
+    return {
+      rows,
+      clock: this.clockToJSON(),
+      proof: await cidsToProof(allResp.cids)
+    }
+  }
+
   /**
    * Registers a Listener to be called when the Fireproof instance's clock is updated.
    * Recieves live changes from the database after they are committed.
@@ -196,7 +223,7 @@ export default class Fireproof {
    */
   async #putToProllyTree (decodedEvent, clock = null) {
     const event = encodeEvent(decodedEvent)
-    if (clock && JSON.stringify(clock) !== JSON.stringify(this.clock)) {
+    if (clock && JSON.stringify(clock) !== JSON.stringify(this.clockToJSON())) {
       // we need to check and see what version of the document exists at the clock specified
       // if it is the same as the one we are trying to put, then we can proceed
       const resp = await eventsSince(this.blocks, this.clock, event.value._clock)
@@ -219,7 +246,7 @@ export default class Fireproof {
     await this.#notifyListeners([decodedEvent]) // this type is odd
     return {
       id: decodedEvent.key,
-      clock: this.clock,
+      clock: this.clockToJSON(),
       proof: { data: await cidsToProof(result.cids), clock: await cidsToProof(result.clockCIDs) }
     }
     // todo should include additions (or split clock)
@@ -268,11 +295,11 @@ export default class Fireproof {
     }
     const doc = resp.result
     if (opts.mvcc === true) {
-      doc._clock = this.clock
+      doc._clock = this.clockToJSON()
     }
     doc._proof = {
       data: await cidsToProof(resp.cids),
-      clock: this.clock
+      clock: this.clockToJSON()
     }
     doc._id = key
     return doc
@@ -282,6 +309,14 @@ export default class Fireproof {
     return yield * vis(this.blocks, this.clock)
   }
 
+  async visTree () {
+    return await visMerkleTree(this.blocks, this.clock)
+  }
+
+  async visClock () {
+    return await visMerkleClock(this.blocks, this.clock)
+  }
+
   setCarUploader (carUploaderFn) {
     // console.log('registering car uploader')
     // https://en.wikipedia.org/wiki/Law_of_Demeter - this is a violation of the law of demeter

package/src/hydrator.js

@@ -6,16 +6,18 @@ const parseCID = cid => typeof cid === 'string' ? CID.parse(cid) : cid
 
 export default class Hydrator {
   static fromJSON (json, database) {
-    database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name })
-    for (const { code, clock: { byId, byKey, db } } of json.indexes) {
-      DbIndex.fromJSON(database, {
-        clock: {
-          byId: byId ? parseCID(byId) : null,
-          byKey: byKey ? parseCID(byKey) : null,
-          db: db ? db.map(c => parseCID(c)) : null
-        },
-        code
-      })
+    database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name, key: json.key })
+    if (json.indexes) {
+      for (const { code, clock: { byId, byKey, db } } of json.indexes) {
+        DbIndex.fromJSON(database, {
+          clock: {
+            byId: byId ? parseCID(byId) : null,
+            byKey: byKey ? parseCID(byKey) : null,
+            db: db ? db.map(c => parseCID(c)) : null
+          },
+          code
+        })
+      }
     }
     return database
   }
@@ -45,7 +47,7 @@ export default class Hydrator {
       index.dbHead = null
     })
     database.clock = clock.map(c => parseCID(c))
-    await database.notifyReset()
+    await database.notifyReset() // hmm... indexes should listen to this? might be more complex than worth it. so far this is the only caller
     return database
   }
 }

package/src/listener.js

@@ -19,7 +19,7 @@ export default class Listener {
      * @type {Fireproof}
      */
     this.database = database
-    this.#doStopListening = database.registerListener((changes) => this.#onChanges(changes))
+    this.#doStopListening = database.registerListener(changes => this.#onChanges(changes))
     /**
      * The map function to apply to each entry in the database.
      * @type {Function}
@@ -46,7 +46,7 @@ export default class Listener {
     if (typeof since !== 'undefined') {
       this.database.changesSince(since).then(({ rows: changes }) => {
         const keys = topicsForChanges(changes, this.routingFn).get(topic)
-        if (keys) keys.forEach((key) => subscriber(key))
+        if (keys) keys.forEach(key => subscriber(key))
       })
     }
     return () => {
@@ -60,18 +60,14 @@ export default class Listener {
       const seenTopics = topicsForChanges(changes, this.routingFn)
       for (const [topic, keys] of seenTopics) {
         const listOfTopicSubscribers = getTopicList(this.#subcribers, topic)
-        listOfTopicSubscribers.forEach((subscriber) => keys.forEach((key) => subscriber(key)))
+        listOfTopicSubscribers.forEach(subscriber => keys.forEach(key => subscriber(key)))
       }
     } else {
-      // reset event
-      if (changes.reset) {
-        for (const [, listOfTopicSubscribers] of this.#subcribers) {
-          listOfTopicSubscribers.forEach((subscriber) => subscriber(changes))
-        }
+      // non-arrays go to all subscribers
+      for (const [, listOfTopicSubscribers] of this.#subcribers) {
+        listOfTopicSubscribers.forEach(subscriber => subscriber(changes))
       }
     }
-    // if changes is special, notify all listeners?
-    // first make the example app use listeners
   }
 }
 
@@ -99,7 +95,7 @@ const topicsForChanges = (changes, routingFn) => {
   const seenTopics = new Map()
   changes.forEach(({ key, value, del }) => {
     if (del || !value) value = { _deleted: true }
-    routingFn(makeDoc({ key, value }), (t) => {
+    routingFn(makeDoc({ key, value }), t => {
       const topicList = getTopicList(seenTopics, t)
       topicList.push(key)
     })

package/src/prolly.js

@@ -3,7 +3,8 @@ import {
   EventFetcher,
   EventBlock,
   findCommonAncestorWithSortedEvents,
-  findEventsToSync
+  findEventsToSync,
+  vis as visClock
 } from './clock.js'
 import { create, load } from 'prolly-trees/map'
 // import { create, load } from '../../../../prolly-trees/src/map.js'
@@ -13,7 +14,7 @@ import * as codec from '@ipld/dag-cbor'
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
 import { doTransaction } from './blockstore.js'
 import { create as createBlock } from 'multiformats/block'
-const opts = { cache, chunker: bf(3), codec, hasher, compare }
+const blockOpts = { cache, chunker: bf(3), codec, hasher, compare }
 
 const withLog = async (label, fn) => {
   const resp = await fn()
@@ -135,7 +136,7 @@ const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
   const event = await events.get(ancestor)
   const { root } = event.value.data
   // console.log('prollyRootFromAncestor', root.cid, JSON.stringify(root.value))
-  return load({ cid: root.cid, get: getBlock, ...opts })
+  return load({ cid: root.cid, get: getBlock, ...blockOpts })
 }
 
 /**
@@ -155,7 +156,7 @@ export async function put (inBlocks, head, event, options) {
   if (!head.length) {
     const additions = new Map()
     let root
-    for await (const node of create({ get: getBlock, list: [event], ...opts })) {
+    for await (const node of create({ get: getBlock, list: [event], ...blockOpts })) {
       root = await node.block
       bigPut(root, additions)
     }
@@ -278,6 +279,28 @@ export async function * vis (blocks, head) {
   const lines = []
   for await (const line of prollyRootNode.vis()) {
     yield line
+    lines.push(line)
   }
   return { vis: lines.join('\n'), cids }
 }
+
+export async function visMerkleTree (blocks, head) {
+  if (!head.length) {
+    return { cids: new CIDCounter(), result: null }
+  }
+  const { node: prollyRootNode, cids } = await root(blocks, head)
+  const lines = []
+  for await (const line of prollyRootNode.vis()) {
+    lines.push(line)
+  }
+  return { vis: lines.join('\n'), cids }
+}
+
+export async function visMerkleClock (blocks, head) {
+  const lines = []
+  for await (const line of visClock(blocks, head)) {
+    // yield line
+    lines.push(line)
+  }
+  return { vis: lines.join('\n') }
+}

package/src/sha1.js

@@ -0,0 +1,82 @@
+// from https://github.com/duzun/sync-sha1/blob/master/rawSha1.js
+// MIT License Copyright (c) 2020 Dumitru Uzun
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// import {
+//   isLittleEndian, switchEndianness32
+// } from 'string-encode'
+
+/**
+* SHA1 on binary array
+*
+* @param   {Uint8Array}  b  Data to hash
+*
+* @return  {Uint8Array}  sha1 hash
+*/
+export default function rawSha1 (b) {
+  let i = b.byteLength
+  let bs = 0
+  let A; let B; let C; let D; let G
+  const H = Uint32Array.from([A = 0x67452301, B = 0xEFCDAB89, ~A, ~B, 0xC3D2E1F0])
+  const W = new Uint32Array(80)
+  const nrWords = (i / 4 + 2) | 15
+  const words = new Uint32Array(nrWords + 1)
+  let j
+
+  words[nrWords] = i * 8
+  words[i >> 2] |= 0x80 << (~i << 3)
+  for (;i--;) {
+    words[i >> 2] |= b[i] << (~i << 3)
+  }
+
+  for (A = H.slice(); bs < nrWords; bs += 16, A.set(H)) {
+    for (i = 0; i < 80;
+      A[0] = (
+        G = ((b = A[0]) << 5 | b >>> 27) +
+                  A[4] +
+                  (W[i] = (i < 16) ? words[bs + i] : G << 1 | G >>> 31) +
+                  0x5A827999,
+        B = A[1],
+        C = A[2],
+        D = A[3],
+        G + ((j = i / 5 >> 2) // eslint-disable-line no-cond-assign
+          ? j !== 2
+            ? (B ^ C ^ D) + (j & 2 ? 0x6FE0483D : 0x14577208)
+            : (B & C | B & D | C & D) + 0x34994343
+          : B & C | ~B & D
+        )
+      )
+      , A[1] = b
+      , A[2] = B << 30 | B >>> 2
+      , A[3] = C
+      , A[4] = D
+      , ++i
+    ) {
+      G = W[i - 3] ^ W[i - 8] ^ W[i - 14] ^ W[i - 16]
+    }
+
+    for (i = 5; i;) H[--i] = H[i] + A[i]
+  }
+
+  // if (isLittleEndian()) {
+  //   H = H.map(switchEndianness32)
+  // }
+
+  return new Uint8Array(H.buffer, H.byteOffset, H.byteLength)
+}

package/src/valet.js

@@ -1,12 +1,30 @@
 import { CarReader } from '@ipld/car'
 import { CID } from 'multiformats/cid'
+import { sha256 } from 'multiformats/hashes/sha2'
+import * as CBW from '@ipld/car/buffer-writer'
+import * as raw from 'multiformats/codecs/raw'
+import * as Block from 'multiformats/block'
+import * as dagcbor from '@ipld/dag-cbor'
 import { openDB } from 'idb'
 import cargoQueue from 'async/cargoQueue.js'
+import { bf } from 'prolly-trees/utils'
+import { nocache as cache } from 'prolly-trees/cache'
+import { encrypt, decrypt } from './crypto.js'
+import { Buffer } from 'buffer'
+import * as codec from 'encrypted-block'
+import sha1sync from './sha1.js'
+const chunker = bf(3)
+
+const NO_ENCRYPT =
+  typeof process !== 'undefined' ? process.env.NO_ENCRYPT : import.meta && import.meta.env.VITE_NO_ENCRYPT
 
 export default class Valet {
   idb = null
+  name = null
   #uploadQueue = null
   #alreadyEnqueued = new Set()
+  #keyMaterial = null
+  keyId = 'null'
 
   /**
    * Function installed by the database to upload car files
@@ -14,8 +32,9 @@ export default class Valet {
    */
   uploadFunction = null
 
-  constructor (name = 'default') {
+  constructor (name = 'default', keyMaterial) {
     this.name = name
+    this.setKeyMaterial(keyMaterial)
     this.#uploadQueue = cargoQueue(async (tasks, callback) => {
       console.log(
         'queue worker',
@@ -24,7 +43,7 @@ export default class Valet {
       )
       if (this.uploadFunction) {
         // todo we can coalesce these into a single car file
-        return await this.withDB(async (db) => {
+        return await this.withDB(async db => {
           for (const task of tasks) {
             await this.uploadFunction(task.carCid, task.value)
             // update the indexedb to mark this car as no longer pending
@@ -38,8 +57,8 @@ export default class Valet {
     })
 
     this.#uploadQueue.drain(async () => {
-      return await this.withDB(async (db) => {
-        const carKeys = (await db.getAllFromIndex('cidToCar', 'pending')).map((c) => c.car)
+      return await this.withDB(async db => {
+        const carKeys = (await db.getAllFromIndex('cidToCar', 'pending')).map(c => c.car)
         for (const carKey of carKeys) {
           await this.uploadFunction(carKey, await db.get('cars', carKey))
           const carMeta = await db.get('cidToCar', carKey)
@@ -50,9 +69,46 @@ export default class Valet {
     })
   }
 
-  withDB = async (dbWorkFun) => {
+  getKeyMaterial () {
+    return this.#keyMaterial
+  }
+
+  setKeyMaterial (km) {
+    if (km && !NO_ENCRYPT) {
+      const hex = Uint8Array.from(Buffer.from(km, 'hex'))
+      this.#keyMaterial = km
+      const hash = sha1sync(hex)
+      this.keyId = Buffer.from(hash).toString('hex')
+    } else {
+      this.#keyMaterial = null
+      this.keyId = 'null'
+    }
+    // console.trace('keyId', this.name, this.keyId)
+  }
+
+  /**
+   * Group the blocks into a car and write it to the valet.
+   * @param {InnerBlockstore} innerBlockstore
+   * @param {Set<string>} cids
+   * @returns {Promise<void>}
+   * @memberof Valet
+   */
+  async writeTransaction (innerBlockstore, cids) {
+    if (innerBlockstore.lastCid) {
+      if (this.#keyMaterial) {
+        // console.log('encrypting car', innerBlockstore.label)
+        const newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.#keyMaterial)
+        await this.parkCar(newCar.cid.toString(), newCar.bytes, cids)
+      } else {
+        const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
+        await this.parkCar(newCar.cid.toString(), newCar.bytes, cids)
+      }
+    }
+  }
+
+  withDB = async dbWorkFun => {
     if (!this.idb) {
-      this.idb = await openDB(`fp.${this.name}.valet`, 2, {
+      this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 2, {
         upgrade (db, oldVersion, newVersion, transaction) {
           if (oldVersion < 1) {
             db.createObjectStore('cars') // todo use database name
@@ -75,7 +131,7 @@ export default class Valet {
    * @param {*} value
    */
   async parkCar (carCid, value, cids) {
-    await this.withDB(async (db) => {
+    await this.withDB(async db => {
       const tx = db.transaction(['cars', 'cidToCar'], 'readwrite')
       await tx.objectStore('cars').put(value, carCid)
       await tx.objectStore('cidToCar').put({ pending: 'y', car: carCid, cids: Array.from(cids) })
@@ -100,7 +156,7 @@ export default class Valet {
   remoteBlockFunction = null
 
   async getBlock (dataCID) {
-    return await this.withDB(async (db) => {
+    return await this.withDB(async db => {
       const tx = db.transaction(['cars', 'cidToCar'], 'readonly')
       const indexResp = await tx.objectStore('cidToCar').index('cids').get(dataCID)
       const carCid = indexResp?.car
@@ -109,10 +165,112 @@ export default class Valet {
       }
       const carBytes = await tx.objectStore('cars').get(carCid)
       const reader = await CarReader.fromBytes(carBytes)
-      const gotBlock = await reader.get(CID.parse(dataCID))
-      if (gotBlock) {
-        return gotBlock.bytes
+      if (this.#keyMaterial) {
+        const roots = await reader.getRoots()
+        const readerGetWithCodec = async cid => {
+          const got = await reader.get(cid)
+          // console.log('got.', cid.toString())
+          let useCodec = codec
+          if (cid.toString().indexOf('bafy') === 0) {
+            useCodec = dagcbor
+          }
+          const decoded = await Block.decode({
+            ...got,
+            codec: useCodec,
+            hasher: sha256
+          })
+          // console.log('decoded', decoded.value)
+          return decoded
+        }
+        const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.#keyMaterial)
+        const block = blocks.find(b => b.cid.toString() === dataCID)
+        if (block) {
+          return block.bytes
+        }
+      } else {
+        const gotBlock = await reader.get(CID.parse(dataCID))
+        if (gotBlock) {
+          return gotBlock.bytes
+        }
       }
     })
   }
 }
+
+const blocksToCarBlock = async (lastCid, blocks) => {
+  let size = 0
+  const headerSize = CBW.headerLength({ roots: [lastCid] })
+  size += headerSize
+  if (!Array.isArray(blocks)) {
+    blocks = Array.from(blocks.entries())
+  }
+  for (const { cid, bytes } of blocks) {
+    size += CBW.blockLength({ cid, bytes })
+  }
+  const buffer = new Uint8Array(size)
+  const writer = await CBW.createWriter(buffer, { headerSize })
+
+  writer.addRoot(lastCid)
+
+  for (const { cid, bytes } of blocks) {
+    writer.write({ cid, bytes })
+  }
+  await writer.close()
+  return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
+}
+
+const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
+  const encryptionKey = Buffer.from(keyMaterial, 'hex')
+  const encryptedBlocks = []
+  const theCids = []
+  for (const { cid } of blocks.entries()) {
+    theCids.push(cid.toString())
+  }
+
+  let last
+  for await (const block of encrypt({
+    cids: theCids,
+    get: async cid => blocks.get(cid), // maybe we can just use blocks.get
+    key: encryptionKey,
+    hasher: sha256,
+    chunker,
+    codec: dagcbor, // should be crypto?
+    root: innerBlockStoreClockRootCid
+  })) {
+    encryptedBlocks.push(block)
+    last = block
+  }
+  // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
+  const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks)
+  return encryptedCar
+}
+// { root, get, key, cache, chunker, hasher }
+
+const memoizeDecryptedCarBlocks = new Map()
+const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
+  if (memoizeDecryptedCarBlocks.has(cid.toString())) {
+    return memoizeDecryptedCarBlocks.get(cid.toString())
+  } else {
+    const blocksPromise = (async () => {
+      const decryptionKey = Buffer.from(keyMaterial, 'hex')
+      // console.log('decrypting', keyMaterial, cid.toString())
+      const cids = new Set()
+      const decryptedBlocks = []
+      for await (const block of decrypt({
+        root: cid,
+        get,
+        key: decryptionKey,
+        chunker,
+        hasher: sha256,
+        cache,
+        codec: dagcbor
+      })) {
+        decryptedBlocks.push(block)
+        cids.add(block.cid.toString())
+      }
+      return { blocks: decryptedBlocks, cids }
+    })()
+    memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise)
+    return blocksPromise
+  }
+}

package/test/db-index.test.js

@@ -30,10 +30,10 @@ describe('DbIndex query', () => {
     })
   })
   it('query index range', async () => {
-    const result = await index.query({ range: [41, 44] })
+    const result = await index.query({ range: [41, 49] })
     assert(result, 'did return result')
     assert(result.rows)
-    assert.equal(result.rows.length, 1, 'one row matched')
+    assert.equal(result.rows.length, 2, 'two row matched')
     assert.equal(result.rows[0].key, 43)
     assert(result.rows[0].value === 'carol', 'correct value')
   })
@@ -47,6 +47,33 @@ describe('DbIndex query', () => {
     assert.equal(result.rows[0].key, 43)
     assert(result.rows[0].value === 'carol', 'correct value')
   })
+  it('query index all', async () => {
+    const result = await index.query()
+    assert(result, 'did return result')
+    assert(result.rows)
+    assert.equal(result.rows.length, 6, 'six row matched')
+    assert.equal(result.rows[0].key, 4)
+    assert.equal(result.rows[0].value, 'emily')
+    assert.equal(result.rows[result.rows.length - 1].value, 'dave')
+  })
+  it('query index NaN', async () => {
+    const result = await index.query({ range: [NaN, 44] })
+    assert(result, 'did return result')
+    assert(result.rows)
+    assert.equal(result.rows.length, 5, 'six row matched')
+    assert.equal(result.rows[0].key, 4)
+    assert.equal(result.rows[0].value, 'emily')
+    assert.equal(result.rows[result.rows.length - 1].value, 'carol')
+  })
+  it('query index Infinity', async () => {
+    const result = await index.query({ range: [42, Infinity] })
+    assert(result, 'did return result')
+    assert(result.rows)
+    assert.equal(result.rows.length, 2, 'six row matched')
+    assert.equal(result.rows[0].key, 43)
+    assert.equal(result.rows[0].value, 'carol')
+    assert.equal(result.rows[result.rows.length - 1].value, 'dave')
+  })
   it('query twice', async () => {
     let result = await index.query({ range: [41, 44] })
     assert(result, 'did return result')

package/test/fireproof.test.js

@@ -20,10 +20,13 @@ describe('Fireproof', () => {
   })
   it('takes an optional name', () => {
     assert.equal(database.name, 'helloName')
+    const km = database.blocks.valet.getKeyMaterial()
+    if (process.env.NO_ENCRYPT) { assert.equal(km, null) } else { assert.equal(km.length, 64) }
     const x = database.blocks.valet.idb
-    assert.equal(x.name.toString(), 'fp.helloName.valet')
+    const keyId = database.blocks.valet.keyId
+    assert.equal(x.name.toString(), `fp.${keyId}.helloName.valet`)
   })
-  it('put and get document', async () => {
+  it('only put and get document', async () => {
     assert(resp0.id, 'should have id')
     assert.equal(resp0.id, '1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
     const avalue = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
@@ -103,7 +106,11 @@ describe('Fireproof', () => {
     console.log('err', err)
     assert.match(err.message, /MVCC conflict/)
   })
-
+  it('allDocuments', async () => {
+    await database.put({ name: 'bob' })
+    const allDocs = await database.allDocuments()
+    assert.equal(allDocs.rows.length, 2)
+  })
   it('has a factory for making new instances with default settings', async () => {
     // TODO if you pass it an email it asks the local keyring, and if no key, does the email validation thing
     const db = await Fireproof.storage({ email: 'jchris@gmail.com' })
@@ -367,6 +374,24 @@ describe('Fireproof', () => {
         }
       })
 
+      database.blocks.clearCommittedCache() // clear cache to force re-reading from encrypted store
+
+      doc = await database.get(resp.id).catch(e => {
+        console.log('failed', e)
+        assert.fail(`get failed on _id: ${id}, error: ${e.message}`)
+      })
+
+      assert.equal(doc.index, index, `doc.index is not equal to index for _id: ${id}`)
+      changes = await database.changesSince().catch(async e => {
+        assert.fail(`changesSince failed on _id: ${id}, error: ${e.message}`)
+      })
+      changes.rows.forEach(row => {
+        for (const key in row) {
+          const value = row[key]
+          assert(!/^bafy/.test(value), `Unexpected "bafy..." value found at index ${index} in row ${JSON.stringify(row)}`)
+        }
+      })
+
       // console.log('changes: ', index, changes.rows.length, JSON.stringify(changes.rows))
       assert.equal(changes.rows.length, index + 2, `failed on ${index}, with ${changes.rows.length} ${id}`)
     }
@@ -411,4 +436,25 @@ describe('Fireproof', () => {
     assert.equal(serialized.name, 'helloName')
     assert.equal(serialized.clock.length, 1)
   })
+  it('clocked changes in order', async () => {
+    await database.put({ _id: '2' })
+    await database.put({ _id: 'three' })
+    await database.put({ _id: '4' })
+    const changes = await database.changesSince(resp0.clock)
+    assert.equal(changes.rows.length, 3)
+    assert.equal(changes.rows[0].key, '2')
+    assert.equal(changes.rows[1].key, 'three')
+    assert.equal(changes.rows[2].key, '4')
+  })
+  it.skip('changes in order', async () => {
+    await database.put({ _id: '2' })
+    await database.put({ _id: 'three' })
+    await database.put({ _id: '4' })
+    const changes = await database.changesSince()
+    assert.equal(changes.rows.length, 4)
+    assert.equal(changes.rows[0].key, resp0.id)
+    assert.equal(changes.rows[1].key, '2')
+    assert.equal(changes.rows[2].key, 'three')
+    assert.equal(changes.rows[3].key, '4')
+  })
 })

package/test/hydrator.test.js

@@ -3,7 +3,6 @@ import assert from 'node:assert'
 import Fireproof from '../src/fireproof.js'
 import DbIndex from '../src/db-index.js'
 import Hydrator from '../src/hydrator.js'
-console.x = function () {}
 
 describe('DbIndex query', () => {
   let database, index
@@ -36,6 +35,9 @@ describe('DbIndex query', () => {
     const serialized = database.toJSON()
     // console.log('serialized', serialized)
     assert.equal(serialized.name, 'global')
+    if (database.blocks.valet.keyId !== 'null') {
+      assert.equal(serialized.key.length, 64)
+    }
     assert.equal(serialized.clock.length, 1)
     assert.equal(serialized.clock[0].constructor.name, 'String')
     assert.equal(serialized.indexes.length, 1)