hyperbee2 2.7.1 → 2.9.0

package/README.md

@@ -7,16 +7,525 @@ npm install hyperbee2
 Next major version for [hyperbee](https://github.com/holepunchto/hyperbee).
 Will be merged in there and released a new major when fully done.
 
+An append-only B-tree on top of a [Hypercore][hypercore].
+
 ## Usage
 
 ```js
-const Hyperbee2 = require('hyperbee2')
+import Hyperbee2 from 'hyperbee2'
+import Corestore from 'corestore'
+
+// Create/open tree
+const tree = new Hyperbee2(new Corestore('./mystore'))
+
+// Wait for tree to become available
+await tree.ready()
 
-const db = new Hyperbee2(store)
+// Write some values to tree
+const batch = tree.write()
+batch.tryPut(Buffer.from('name'), Buffer.from('example'))
+batch.tryPut(Buffer.from('email'), Buffer.from('example@example.com'))
+await batch.flush()
 
-// see tests for more
+// Read values from tree
+const name = await tree.get(Buffer.from('name'))
+const email = await tree.get(Buffer.from('email'))
+
+// Print values
+console.log(name.value.toString()) // example
+console.log(email.value.toString()) // example@example.com
 ```
 
 ## License
 
 Apache-2.0
+
+## API
+
+### Hyperbee
+
+#### `const db = new Hyperbee(store, [options])`
+
+Make a new Hyperbee2 instance. The `store` argument expects a
+[Corestore][corestore].
+
+Options include:
+
+```js
+{
+  key: null,               // Buffer or String. Key of Hypercore to load via Corestore
+  maxCacheSize: 4096,      // Max number of nodes to keep in NodeCache
+  core: Hypercore(...),    // Hypercore within the Corestore to use (defaults to loading key, or using name='bee')
+  view: false,             // Is this a view of an open Hyperbee? (i.e. do not close underlying store)
+  writable: true,          // Is append / truncate allowed on the underlying Hypercore?
+  unbatch: 0,              // Number of write batches to rollback during bootstrap
+  autoUpdate: false,       // Reload root node when underlying Hypercore is appended to?
+  preload: null,           // A function called by ready() after the Hypercore is ready. Can be async.
+  wait: true,              // Wait for Hypercore to download blocks
+}
+```
+
+#### `await db.ready()`
+
+Ensures the underlying [Hypercore][hypercore] is ready and prepares the Hyperbee
+for use. If `autoUpdate` was set to true in the constructor's options,
+this will start watching the [Hypercore][hypercore] for new writes.
+
+Calling `get()` will call this automatically for you.
+
+#### `await db.close()`
+
+Fully close the Hyperbee. If it is not a view on another Hyperbee,
+this will also close it's [Hypercore][hypercore].
+
+#### `db.head()`
+
+Returns an object with the following properties:
+
+```
+{
+  length,  // Number of blocks from the start of Hypercore that apply to this tree.
+  key,     // Buffer or null. The key of the underlying Hypercore.
+}
+```
+
+If the Hyperbee is not ready, this will return null.
+
+#### `db.cache`
+
+Read only. The NodeCache used by this Hyperbee.
+
+#### `db.core`
+
+Read only. The [Hypercore][hypercore] used by this Hyperbee.
+
+#### `db.opening`
+
+Read only. A Promise that resolves to `undefined` once the [Corestore][corestore]
+is ready.
+
+#### `db.closing`
+
+Read only. Initially null. When `db.close()` is called, this is set to
+a Promise that resolves to `undefined` when the close completes.
+
+#### `db.opened`
+
+Read only. Boolean indicating whether the [Corestore][corestore] has opened.
+
+#### `db.closed`
+
+Read only. Boolean indicating whether the [Corestore][corestore] has closed.
+
+#### `db.replicate()`
+
+Calls `replicate` on underlying [Corestore][corestore].
+
+#### `db.checkout([options])`
+
+Returns a new Hyperbee as a view on the underlying [Hypercore][hypercore].
+
+Options:
+
+```js
+{
+  writable: false,           // Boolean. Will the new tree be writable?
+  length: this.core.length,  // Number. Length of blocks used from the Hypercore
+  key: null,                 // Buffer or null. Key of the Hypercore
+}
+```
+
+#### `db.move([options])`
+
+Replaces the root of the tree with a new entry, potentially using a new
+[Hypercore][hypercore].
+
+Options:
+
+```js
+{
+  writable: this.writable,   // Boolean. Is this tree writable after the move?
+  length: this.core.length,  // Number. Length of blocks used from the Hypercore
+  key: null,                 // Buffer or null. Key of the Hypercore
+}
+```
+
+#### `db.snapshot()`
+
+Returns a new Hyperbee that is a read only view of the current tree.
+
+#### `db.undo(n)`
+
+Returns a new Hyperbee that is a writable view of the current tree,
+with the last `n` write batches ignored.
+
+#### `db.write([options])`
+
+Returns a [WriteBatch](#writebatch) object through which the tree can be updated.
+
+Options:
+
+```js
+{
+  length: -1,                     // Length of blocks used from the Hypercore
+                                  // (i.e. what point in the hypercore is the write
+                                  // going to extend when flush() is called?).
+                                  // If -1, the length will be calculated on flush()
+                                  // using root.seq + 1.
+  key: null,                      // Buffer or null. Key of the Hypercore.
+  autoUpdate: true,               // Boolean .Should Hyperbee automatically reflect updates
+                                  // after each flush()?
+  compat: false,                  // Boolean. Write blocks compatible with Hyperbee 1?
+  type: encoding.TYPE_LATEST,     // Number. Block format to use.
+  inlineValueSize: 1024,          // Values smaller than this byte length are
+                                  // written inline in the node. Larger values
+                                  // are referenced via a pointer into the block.
+  preferredBlockSize: 4096,       // Try to write blocks of approximately this size
+                                  // when flushing updates.
+}
+```
+
+Errors:
+
+- If the tree is not writable this will throw an Error
+
+#### `db.createReadStream([options])`
+
+Returns a [streamx][streamx] Readable Stream. This is async iterable.
+
+Options:
+
+```js
+{
+  prefetch: true,        // Prefetch future blocks after yielding an entry?
+  reverse: false,        // Iterate backwards over keys?
+  limit: -1,             // Max number of entries to yield (-1 means no limit)
+  gte: undefined,        // Buffer. Key lower bound (inclusive)
+  gt: undefined,         // Buffer. Key lower bound (exclusive)
+  lte: undefined,        // Buffer. Key upper bound (inclusive)
+  lt: undefined,         // Buffer. Key upper bound (exclusive)
+  highWaterMark: 16384,  // Size of read ahead buffer calculated
+                         // as: number of entries * 1024
+}
+```
+
+Iterating over the stream will yield the following properties:
+
+```js
+{
+  core,    // Hypercore: the hypercore the entry is stored in
+  offset,  // Number: the index of the entry in the block
+  seq,     // Number: the sequence number of the block in the hypercore
+  key,     // Buffer: the key of the entry
+  value,   // Buffer: the value of the entry
+}
+```
+
+Example:
+
+```js
+for await (const data of b.createReadStream()) {
+  console.log(data.key, '-->', data.value)
+}
+```
+
+#### `db.createDiffStream(right, [options])`
+
+Returns a [streamx][streamx] Readable Stream that provides
+synchronized iteration over two trees. This is async iterable.
+
+Options: Same as options for `createReadStream()`.
+
+Iterating over the stream will yield the same properties as
+`createReadStream()` split into left (`db` in this case) and right
+(`right` parameter).
+
+```js
+{
+  left: {core, offset, seq, key, value},   // see createReadStream()
+  right: {core, offset, seq, key, value},  // see createReadStream()
+}
+```
+
+Example:
+
+```js
+import Hyperbee from './index.js'
+import Corestore from 'corestore'
+
+const b1 = new Hyperbee(new Corestore('./store1'))
+const b2 = new Hyperbee(new Corestore('./store2'))
+
+await b1.ready()
+await b2.ready()
+
+const w1 = b1.write()
+w1.tryPut(Buffer.from('A'), Buffer.from('A'))
+w1.tryPut(Buffer.from('B'), Buffer.from('B'))
+w1.tryPut(Buffer.from('C'), Buffer.from('C'))
+await w1.flush()
+
+const w2 = b2.write()
+w2.tryPut(Buffer.from('A'), Buffer.from('A'))
+w2.tryPut(Buffer.from('C'), Buffer.from('C'))
+w2.tryPut(Buffer.from('E'), Buffer.from('E'))
+await w2.flush()
+
+for await (const data of b1.createDiffStream(b2)) {
+  console.log(data.left?.key.toString(), data.right?.key.toString())
+}
+```
+
+Example output:
+
+```js
+A A
+B undefined
+C C
+undefined E
+```
+
+#### `db.createChangesStream([options])`
+
+Returns a [streamx][streamx] Readable Stream for iterating over all
+batches previously written to the tree. This is async iterable.
+
+Options:
+
+```js
+{
+  head: null,            // null means use this.tree.head()
+  highWaterMark: 16384,  // Size of read ahead buffer calculated
+                         // as: number of entries * 1024
+  timeout: 0,            // Wait at most this many milliseconds (0 means no timeout).
+                         // Defaults to the value of the Hyperbee's timeout option.
+  wait: true,            // Wait for Hypercore to download blocks
+                         // Defaults to the value of the Hyperbee's wait option.
+}
+```
+
+Iterating over the stream will yield:
+
+```js
+{
+  head: {
+    length,  // Number: number of blocks from the start of Hypercore
+             // that apply to this version of the tree
+    key,     // Buffer or null: the key of the Hypercore for this version
+  },
+  tail: {
+    length,  // Number: number of blocks from the start of Hypercore
+             // that apply to the previous version of the tree
+    key,     // Buffer or null: the key of the Hypercore for the previous
+             // version
+  },
+  batch: [   // Blocks written in this batch
+    {tree, keys, values, cohorts, metadata, ...},
+    ...
+  ],
+}
+```
+
+#### `await db.peek([range])`
+
+Attempts to get the first entry within the given range.
+
+Returns `null` if no entry exists in the range, or an object with
+the following properties on success:
+
+```js
+{
+  core,    // Hypercore: the hypercore the entry is stored in
+  offset,  // Number: the index of the entry in the block
+  seq,     // Number: the sequence number of the block in the hypercore
+  key,     // Buffer: the key of the entry
+  value,   // Buffer: the value of the entry
+}
+```
+
+The `range` argument accepts the same properties as the options for
+[`createReadStream()`](#dbcreatereadstreamoptions).
+
+#### `await db.download([range])`
+
+Fetches all entries in the given range. The promise resolves once
+all matching entries have been fetched.
+
+The `range` argument accepts the same properties as the options for
+[`createReadStream()`](#dbcreatereadstreamoptions).
+
+#### `await db.get(key, [options])`
+
+Attempt to find an entry by its key.
+
+Returns `null` if no entry exists in the range, or an object with
+the following properties on success:
+
+```js
+{
+  core,    // Hypercore: the hypercore the entry is stored in
+  offset,  // Number: the index of the entry in the block
+  seq,     // Number: the sequence number of the block in the hypercore
+  key,     // Buffer: the key of the entry
+  value,   // Buffer: the value of the entry
+}
+```
+
+Options:
+
+```js
+{
+  timeout,  // Number: wait at most this many milliseconds (0 means no timeout)
+  wait,     // Boolean: wait for Hypercore to download blocks
+}
+```
+
+#### `b.on('ready', listener)`
+
+Emitted once the Hyperbee is ready for use.
+
+##### `b.on('update', listener)`
+
+Emitted in the following scenarios:
+
+- When a `WriteBatch` is flushed and its `autoUpdate` option is `true`
+  (the default).
+- When the underlying core is appended to (locally or remotely) and the
+  `Hyperbee`'s `autoUpdate` option is `true`.
+- After a `move()` call on the `Hyperbee`.
+- After a rollback completes because the `unbatch` option to the
+  `Hyperbee` constructor was greater than `0`.
+
+Warning: this event may be emitted multiple times for the same
+update if `autoUpdate` is set on the `Hyperbee` and on `WriteBatch`:
+
+```js
+const b = new Hyperbee(store, { autoUpdate: true })
+b.on('update', () => console.log('New update'))
+
+const w = b.write(/* autoUpdate is true by default */)
+w.tryPut(Buffer.from('key'), Buffer.from('value'))
+await w.flush()
+
+// The following will be printed:
+// New udpate
+// New udpate
+```
+
+##### `b.on('close', listener)`
+
+Emitted one the `close()` method completes.
+
+### WriteBatch
+
+A WriteBatch can be constructed via Hyperbee's [write()](#dbwriteoptions)
+method. Using a batch, multiple updates can be queued then applied together.
+
+```js
+const batch = b.write();
+batch.tryPut(Buffer.from('key'), Buffer.from('value');
+await batch.flush();
+```
+
+**Warning:** WriteBatch does not hold an exclusive lock on the database
+while queuing operations unless you call `lock()`. By default, the
+lock is only acquired when the operations are flushed. So be careful
+about building concurrent batches:
+
+```js
+import Hyperbee from './index.js'
+import Corestore from 'corestore'
+
+const b = new Hyperbee(new Corestore('./store'))
+await b.ready()
+
+const w1 = b.write()
+w1.tryPut(Buffer.from('name'), Buffer.from('Sneezy'))
+
+const w2 = b.write()
+w2.tryPut(Buffer.from('name'), Buffer.from('Sleepy'))
+w2.tryPut(Buffer.from('email'), Buffer.from('sleepy@example.com'))
+
+// Be careful about application order vs. order the operations
+// were queued when building batches concurrently.
+await w2.flush()
+await w1.flush()
+
+for await (const data of b.createReadStream(b)) {
+  console.log(data.key.toString(), '-->', data.value.toString())
+}
+
+// Output: (email and name mismatch)
+//
+// email --> sleepy@example.com
+// name --> Sneezy
+```
+
+### `await batch.lock()`
+
+Aquires an exclusive write lock now instead of waiting for `flush()`
+to be called. No writes will occur until this batch is closed allowing
+you to keep a consistent view of the database while building the batch.
+
+#### `batch.tryPut(key, value)`
+
+Queues an operation to associate `key` with `value`. Any existing entry
+for `key` will be overwritten.
+
+#### `batch.tryDelete(key)`
+
+Queues an operation to remove the entry with `key`, if it exists. If it
+does not exist, this method does nothing (and will not throw).
+
+#### `batch.tryClear()`
+
+Queues an operation to clear all entries from the tree.
+
+#### `await batch.flush()`
+
+Aquires an exclusive write lock and applies the operations queued in this
+batch to the tree, clearing the queue.
+
+**Warning:** continuing to use the batch after flushing can cause unpredictable
+behavior. Batches applied after the first flush will be 'unapplied' if you
+flush again later. This can lead to accidentally removing data from the tree.
+
+```js
+import Hyperbee from './index.js'
+import Corestore from 'corestore'
+
+const b = new Hyperbee(new Corestore('./store'))
+await b.ready()
+
+const w1 = b.write()
+w1.tryPut(Buffer.from('name'), Buffer.from('Sneezy'))
+
+const w2 = b.write()
+w2.tryPut(Buffer.from('name'), Buffer.from('Sleepy'))
+w2.tryPut(Buffer.from('email'), Buffer.from('sleepy@example.com'))
+
+await w1.flush()
+await w2.flush()
+
+// Warning: flushing w1 again will unapply w2!
+w1.tryPut(Buffer.from('active'), Buffer.from('false'))
+await w1.flush()
+
+for await (const data of b.createReadStream(b)) {
+  console.log(data.key.toString(), '-->', data.value.toString())
+}
+
+// Output: (name has reverted to 'Sneezy', email is missing)
+//
+// active --> false
+// name --> Sneezy
+```
+
+#### `batch.close()`
+
+Closes the batch without flushing operations. Subsequent attempts
+to flush the batch will result in an error.
+
+[hypercore]: https://github.com/holepunchto/hypercore
+[corestore]: https://github.com/holepunchto/corestore
+[streamx]: https://github.com/mafintosh/streamx

package/index.js

@@ -8,36 +8,47 @@ const NodeCache = require('./lib/cache.js')
 const WriteBatch = require('./lib/write.js')
 const CoreContext = require('./lib/context.js')
 const SessionConfig = require('./lib/session-config.js')
-const { Pointer, KeyPointer, ValuePointer, TreeNode, EMPTY } = require('./lib/tree.js')
-const { DeltaOp, DeltaCohort, OP_COHORT } = require('./lib/compression.js')
+const { inflate, inflateValue } = require('./lib/inflate.js')
+const { EMPTY } = require('./lib/tree.js')
 
 class Hyperbee extends EventEmitter {
-  constructor(store, options = {}) {
+  constructor(store, opts = {}) {
     super()
 
     const {
       t = 128, // legacy number for now, should be 128 now
       key = null,
       encryption = null,
+      getEncryptionProvider = toEncryptionProvider(encryption),
       maxCacheSize = 4096,
-      config = new SessionConfig([], 0, true),
+      config = new SessionConfig([], 0, true, null),
       activeRequests = config.activeRequests,
       timeout = config.timeout,
       wait = config.wait,
-      core = key ? store.get(key) : store.get({ key, name: 'bee', encryption }),
-      context = new CoreContext(store, core, new NodeCache(maxCacheSize), core, encryption, t),
+      trace = config.trace,
+      core = key
+        ? store.get({ key, encryption: getEncryptionProvider(key) })
+        : store.get({ key, name: 'bee', encryption: getEncryptionProvider(key) }),
+      context = new CoreContext(
+        store,
+        core,
+        new NodeCache(maxCacheSize),
+        core,
+        getEncryptionProvider,
+        t
+      ),
       root = null,
       view = false,
       writable = true,
       unbatch = 0,
-      autoUpdate = false,
+      autoUpdate = !writable && !view,
       preload = null
-    } = options
+    } = opts
 
     this.store = store
     this.root = root
     this.context = context
-    this.config = config.sub(activeRequests, timeout, wait)
+    this.config = config.sub(activeRequests, timeout, wait, trace)
     this.view = view
     this.writable = writable
     this.unbatch = unbatch
@@ -107,12 +118,9 @@ class Hyperbee extends EventEmitter {
   }
 
   move({ length = this.core.length, key = null, writable = this.writable } = {}) {
-    const context = key ? this.context.getContextByKey(key) : this.context
-    const root = length === 0 ? EMPTY : context.createTreeNode(0, length - 1, 0, false, null)
-    this.context = context
+    this.context = key ? this.context.getContextByKey(key) : this.context
     this.writable = writable
-    this.root = root
-    this.emit('update')
+    this._setRoot(this._nodeAtSeq(length - 1), true)
   }
 
   snapshot() {
@@ -128,21 +136,23 @@ class Hyperbee extends EventEmitter {
     return new WriteBatch(this, opts)
   }
 
+  _lastNodeInCore() {
+    return this._nodeAtSeq(this.context.core.length - 1)
+  }
+
+  _nodeAtSeq(seq) {
+    return seq < 0 ? EMPTY : this.context.createTreeNode(0, seq, 0, false, null)
+  }
+
   async ready() {
     if (!this.core.opened) await this.core.ready()
     if (this.root) return
     if (this.preload) await this.preload()
     if (this.root) return
 
-    this.root =
-      this.context.core.length === 0
-        ? EMPTY
-        : this.context.createTreeNode(0, this.core.length - 1, 0, false, null)
-
+    this._setRoot(this._lastNodeInCore(), false)
     if (this.autoUpdate) {
-      this.core.on('append', () => {
-        this.update()
-      })
+      this.core.on('append', () => this._setRoot(this._lastNodeInCore(), true))
     }
 
     this.emit('ready')
@@ -192,45 +202,16 @@ class Hyperbee extends EventEmitter {
     return ptr.value
   }
 
-  // TODO: unslab these
   async inflate(ptr, config) {
-    if (ptr.value) {
-      this.bump(ptr)
-      return ptr.value
+    if (!ptr.value) {
+      await inflate(ptr, config)
     }
-
-    const [block, context] = await Promise.all([
-      ptr.context.getBlock(ptr.seq, ptr.core, config),
-      ptr.context.getContext(ptr.core, config)
-    ])
-
-    const tree = block.tree[ptr.offset]
-
-    const keys = new Array(tree.keys.length)
-    const children = new Array(tree.children.length)
-
-    for (let i = 0; i < keys.length; i++) {
-      const d = tree.keys[i]
-      keys[i] = inflateKey(context, d, ptr, block, config)
-    }
-
-    for (let i = 0; i < children.length; i++) {
-      const d = tree.children[i]
-      children[i] = inflateChild(context, d, ptr, block, config)
-    }
-
-    const [k, c] = await Promise.all([Promise.all(keys), Promise.all(children)])
-
-    const value = new TreeNode(k, c)
-    if (!ptr.value) ptr.value = value
-
     this.bump(ptr)
-
     return ptr.value
   }
 
   async finalizeKeyPointer(key, config) {
-    const value = key.value || (await this.inflateValue(key, config))
+    const value = key.value || (await inflateValue(key, config))
 
     return {
       core: key.context.getCore(key.core),
@@ -241,30 +222,6 @@ class Hyperbee extends EventEmitter {
     }
   }
 
-  async inflateValue(key, config) {
-    if (key.value) return key.value
-    if (!key.valuePointer) return null
-
-    const ptr = key.valuePointer
-
-    if (ptr.split === 0) {
-      const block = await ptr.context.getBlock(ptr.seq, ptr.core, config)
-      return block.values[ptr.offset]
-    }
-
-    const blockPromises = new Array(ptr.split + 1)
-    for (let i = 0; i < blockPromises.length; i++) {
-      blockPromises[i] = ptr.context.getBlock(ptr.seq - ptr.split + i, ptr.core, config)
-    }
-    const blocks = await Promise.all(blockPromises)
-    const splitValue = new Array(blockPromises.length)
-    for (let i = 0; i < splitValue.length - 1; i++) {
-      splitValue[i] = blocks[i].values[0]
-    }
-    splitValue[splitValue.length - 1] = blocks[blocks.length - 1].buffer[ptr.offset]
-    return b4a.concat(splitValue)
-  }
-
   async bootstrap(config) {
     if (!this.root) await this.ready()
     if (this.unbatch) await this._rollback(config)
@@ -294,16 +251,16 @@ class Hyperbee extends EventEmitter {
 
     if (expected === this.unbatch) {
       this.context = context
-      this.root = length === 0 ? EMPTY : context.createTreeNode(0, length - 1, 0, false, null)
-      this.unbatch = 0
-      this.emit('update')
+      this._setRoot(this._nodeAtSeq(length - 1), true)
     }
   }
 
-  update(root = null) {
-    this.root = root
+  _setRoot(root, emit) {
+    if (!root.equivalentTo(this.root)) {
+      this.root = root
+      if (emit) this.emit('update')
+    }
     this.unbatch = 0
-    this.emit('update')
   }
 
   async get(key, opts) {
@@ -343,85 +300,7 @@ module.exports = Hyperbee
 
 function noop() {}
 
-function inflateKey(context, d, ptr, block, config) {
-  if (d.type === OP_COHORT) return inflateKeyCohort(context, d, ptr, block, config)
-  return inflateKeyDelta(context, d, ptr, block, config)
-}
-
-async function inflateKeyDelta(context, d, ptr, block, config) {
-  const k = d.pointer
-
-  if (!k) return new DeltaOp(false, d.type, d.index, null)
-
-  const blk =
-    k.seq === ptr.seq && k.core === 0 && ptr.core === 0
-      ? block
-      : await context.getBlock(k.seq, k.core, config)
-
-  const bk = blk.keys[k.offset]
-
-  let vp = null
-
-  if (bk.valuePointer) {
-    const p = bk.valuePointer
-    const ctx = await context.getContext(k.core, config)
-    vp = new ValuePointer(ctx, p.core, p.seq, p.offset, p.split)
-  }
-
-  const kp = new KeyPointer(context, k.core, k.seq, k.offset, false, bk.key, bk.value, vp)
-  return new DeltaOp(false, d.type, d.index, kp)
-}
-
-async function inflateKeyCohort(context, d, ptr, block, config) {
-  const co = d.pointer
-
-  const blk =
-    co.seq === ptr.seq && co.core === 0 && ptr.core === 0
-      ? block
-      : await context.getBlock(co.seq, co.core, config)
-
-  const cohort = blk.cohorts[co.offset]
-  const promises = new Array(cohort.length)
-
-  for (let i = 0; i < cohort.length; i++) {
-    const p = cohort[i]
-    const k = inflateKeyDelta(context, p, co, blk, config)
-    promises[i] = k
-  }
-
-  const p = new Pointer(context, co.core, co.seq, co.offset)
-  return new DeltaCohort(false, p, await Promise.all(promises))
-}
-
-async function inflateChild(context, d, ptr, block, config) {
-  if (d.type === OP_COHORT) return inflateChildCohort(context, d, ptr, block, config)
-  if (d.pointer && !context.hasCore(d.pointer.core)) await context.update(config)
-  return inflateChildDelta(context, d, ptr, block, config)
-}
-
-function inflateChildDelta(context, d, ptr, block, config) {
-  const p = d.pointer
-  const c = p && context.createTreeNode(p.core, p.seq, p.offset, false, null)
-  return new DeltaOp(false, d.type, d.index, c)
-}
-
-async function inflateChildCohort(context, d, ptr, block, config) {
-  const co = d.pointer
-
-  const blk =
-    co.seq === ptr.seq && co.core === 0 && ptr.core === 0
-      ? block
-      : await context.getBlock(co.seq, co.core, config)
-
-  const cohort = blk.cohorts[co.offset]
-  const deltas = new Array(cohort.length)
-
-  for (let i = 0; i < cohort.length; i++) {
-    const c = cohort[i]
-    if (c.pointer && !context.hasCore(c.pointer.core)) await context.update(config)
-    deltas[i] = inflateChildDelta(context, c, co, blk, config)
-  }
-
-  const p = new Pointer(context, co.core, co.seq, co.offset)
-  return new DeltaCohort(false, p, deltas)
+function toEncryptionProvider(encryption) {
+  if (encryption) return (key) => encryption
+  return () => null
 }

package/lib/context.js

@@ -5,7 +5,16 @@ const { TreeNodePointer } = require('./tree.js')
 const { decodeBlock } = require('./encoding.js')
 
 class CoreContext {
-  constructor(store, local, cache, core, encryption, t, lock = new ScopeLock(), other = new Map()) {
+  constructor(
+    store,
+    local,
+    cache,
+    core,
+    getEncryptionProvider,
+    t,
+    lock = new ScopeLock(),
+    other = new Map()
+  ) {
     this.store = store
     this.local = local
     this.cache = cache
@@ -13,7 +22,7 @@ class CoreContext {
     this.t = t
     this.minKeys = t - 1
     this.maxKeys = 2 * t - 1
-    this.encryption = encryption
+    this.getEncryptionProvider = getEncryptionProvider
     this.lock = lock
     this.other = other
     this.length = 0
@@ -132,7 +141,7 @@ class CoreContext {
     if (index > this.cores.length) throw new Error('Bad core index: ' + index)
     if (this.opened[index - 1] === null) {
       const key = this.cores[index - 1].key
-      this.opened[index - 1] = this.store.get({ key, encryption: this.encryption })
+      this.opened[index - 1] = this.store.get({ key, encryption: this.getEncryptionProvider(key) })
     }
     return this.opened[index - 1]
   }
@@ -152,6 +161,9 @@ class CoreContext {
     const hc = this.getCore(core)
     const buffer = await hc.get(seq, config)
     if (buffer === null) throw BLOCK_NOT_AVAILABLE()
+
+    if (config.trace !== null) config.trace(core, seq)
+
     const block = decodeBlock(buffer, seq)
     return block
   }
@@ -166,7 +178,9 @@ class CoreContext {
     const hex = b4a.toString(key, 'hex')
     if (this.other.has(hex)) return this.other.get(hex)
 
-    const ctx = this._createContext(this.store.get({ key, encryption: this.encryption }))
+    const ctx = this._createContext(
+      this.store.get({ key, encryption: this.getEncryptionProvider(key) })
+    )
     this.other.set(hex, ctx)
     return ctx
   }
@@ -195,11 +209,11 @@ class CoreContext {
     const store = this.store
     const local = this.local
     const cache = this.cache
-    const encryption = this.encryption
+    const getEncryptionProvider = this.getEncryptionProvider
     const t = this.t
     const lock = this.lock
     const other = this.other
-    return new CoreContext(store, local, cache, core, encryption, t, lock, other)
+    return new CoreContext(store, local, cache, core, getEncryptionProvider, t, lock, other)
   }
 }
 

package/lib/inflate.js

@@ -0,0 +1,140 @@
+const b4a = require('b4a')
+const { Pointer, KeyPointer, ValuePointer, TreeNode } = require('./tree.js')
+const { DeltaOp, DeltaCohort, OP_COHORT } = require('./compression.js')
+
+exports.inflate = async function inflate(ptr, config) {
+  if (ptr.value) return ptr.value
+
+  const [block, context] = await Promise.all([
+    ptr.context.getBlock(ptr.seq, ptr.core, config),
+    ptr.context.getContext(ptr.core, config)
+  ])
+
+  const tree = block.tree[ptr.offset]
+
+  const keys = new Array(tree.keys.length)
+  const children = new Array(tree.children.length)
+
+  for (let i = 0; i < keys.length; i++) {
+    const d = tree.keys[i]
+    keys[i] = inflateKey(context, d, ptr, block, config)
+  }
+
+  for (let i = 0; i < children.length; i++) {
+    const d = tree.children[i]
+    children[i] = inflateChild(context, d, ptr, block, config)
+  }
+
+  const [k, c] = await Promise.all([Promise.all(keys), Promise.all(children)])
+
+  const value = new TreeNode(k, c)
+  if (!ptr.value) ptr.value = value
+  return ptr.value
+}
+
+function inflateKey(context, d, ptr, block, config) {
+  if (d.type === OP_COHORT) return inflateKeyCohort(context, d, ptr, block, config)
+  return inflateKeyDelta(context, d, ptr, block, config)
+}
+
+async function inflateKeyDelta(context, d, ptr, block, config) {
+  const k = d.pointer
+
+  if (!k) return new DeltaOp(false, d.type, d.index, null)
+
+  const blk =
+    k.seq === ptr.seq && k.core === 0 && ptr.core === 0
+      ? block
+      : await context.getBlock(k.seq, k.core, config)
+
+  const bk = blk.keys[k.offset]
+
+  let vp = null
+
+  if (bk.valuePointer) {
+    const p = bk.valuePointer
+    const ctx = await context.getContext(k.core, config)
+    vp = new ValuePointer(ctx, p.core, p.seq, p.offset, p.split)
+  }
+
+  const kp = new KeyPointer(context, k.core, k.seq, k.offset, false, bk.key, bk.value, vp)
+  return new DeltaOp(false, d.type, d.index, kp)
+}
+
+exports.inflateValue = async function inflateValue(key, config) {
+  if (key.value) return key.value
+  if (!key.valuePointer) return null
+
+  const ptr = key.valuePointer
+
+  if (ptr.split === 0) {
+    const block = await ptr.context.getBlock(ptr.seq, ptr.core, config)
+    return block.values[ptr.offset]
+  }
+
+  const blockPromises = new Array(ptr.split + 1)
+  for (let i = 0; i < blockPromises.length; i++) {
+    blockPromises[i] = ptr.context.getBlock(ptr.seq - ptr.split + i, ptr.core, config)
+  }
+  const blocks = await Promise.all(blockPromises)
+  const splitValue = new Array(blockPromises.length)
+  for (let i = 0; i < splitValue.length - 1; i++) {
+    splitValue[i] = blocks[i].values[0]
+  }
+  splitValue[splitValue.length - 1] = blocks[blocks.length - 1].buffer[ptr.offset]
+  return b4a.concat(splitValue)
+}
+
+async function inflateKeyCohort(context, d, ptr, block, config) {
+  const co = d.pointer
+
+  const blk =
+    co.seq === ptr.seq && co.core === 0 && ptr.core === 0
+      ? block
+      : await context.getBlock(co.seq, co.core, config)
+
+  const cohort = blk.cohorts[co.offset]
+  const promises = new Array(cohort.length)
+
+  for (let i = 0; i < cohort.length; i++) {
+    const p = cohort[i]
+    const k = inflateKeyDelta(context, p, co, blk, config)
+    promises[i] = k
+  }
+
+  const p = new Pointer(context, co.core, co.seq, co.offset)
+  return new DeltaCohort(false, p, await Promise.all(promises))
+}
+
+async function inflateChild(context, d, ptr, block, config) {
+  if (d.type === OP_COHORT) return inflateChildCohort(context, d, ptr, block, config)
+  if (d.pointer && !context.hasCore(d.pointer.core)) await context.update(config)
+  return inflateChildDelta(context, d, ptr, block, config)
+}
+
+function inflateChildDelta(context, d, ptr, block, config) {
+  const p = d.pointer
+  const c = p && context.createTreeNode(p.core, p.seq, p.offset, false, null)
+  return new DeltaOp(false, d.type, d.index, c)
+}
+
+async function inflateChildCohort(context, d, ptr, block, config) {
+  const co = d.pointer
+
+  const blk =
+    co.seq === ptr.seq && co.core === 0 && ptr.core === 0
+      ? block
+      : await context.getBlock(co.seq, co.core, config)
+
+  const cohort = blk.cohorts[co.offset]
+  const deltas = new Array(cohort.length)
+
+  for (let i = 0; i < cohort.length; i++) {
+    const c = cohort[i]
+    if (c.pointer && !context.hasCore(c.pointer.core)) await context.update(config)
+    deltas[i] = inflateChildDelta(context, c, co, blk, config)
+  }
+
+  const p = new Pointer(context, co.core, co.seq, co.offset)
+  return new DeltaCohort(false, p, deltas)
+}

package/lib/ranges.js

@@ -119,6 +119,7 @@ class RangeIterator {
     // TODO: dbl check this for off-by-ones with the offset and keys and children
     let limit = this.limit
 
+    // TODO: if limit === -1, don't return early here
     if (limit < this.tree.context.minKeys) return
 
     const parent = this.stack[this.stack.length - 1]
@@ -128,17 +129,21 @@ class RangeIterator {
     this.prefetching = pv
 
     for (let i = parent.offset >> 1; i < pv.children.length; i++) {
-      const k = pv.keys.get(i)
+      // If the preceding key in parent is beyond upper bound,
+      // stop fetching child nodes.
+      if (i > 0) {
+        const k = pv.keys.get(i - 1)
 
-      const cmp = this.reverse
-        ? this.start
-          ? b4a.compare(this.start, k.key)
-          : -1
-        : this.end
-          ? b4a.compare(k.key, this.end)
-          : -1
+        const cmp = this.reverse
+          ? this.start
+            ? b4a.compare(this.start, k.key)
+            : -1
+          : this.end
+            ? b4a.compare(k.key, this.end)
+            : -1
 
-      if (cmp > this.compare) break
+        if (cmp > this.compare) return
+      }
 
       const c = pv.children.get(i)
       if (!c.value) this.tree.inflate(c, this.config).catch(noop)

package/lib/session-config.js

@@ -1,22 +1,33 @@
 class SessionConfig {
-  constructor(activeRequests, timeout, wait) {
+  constructor(activeRequests, timeout, wait, trace) {
     this.activeRequests = activeRequests
     this.timeout = timeout
     this.wait = wait
+    this.trace = trace
   }
 
-  sub(activeRequests, timeout, wait) {
-    if (this.activeRequests === activeRequests && this.timeout === timeout && this.wait === wait) {
+  sub(activeRequests, timeout, wait, trace) {
+    if (
+      this.activeRequests === activeRequests &&
+      this.timeout === timeout &&
+      this.wait === wait &&
+      this.trace === trace
+    ) {
       return this
     }
 
-    return new SessionConfig(activeRequests, timeout, wait)
+    return new SessionConfig(activeRequests, timeout, wait, trace)
   }
 
   options(opts) {
     if (!opts) return this
-    const { activeRequests = this.activeRequests, timeout = this.timeout, wait = this.wait } = opts
-    return this.sub(activeRequests, timeout, wait)
+    const {
+      activeRequests = this.activeRequests,
+      timeout = this.timeout,
+      wait = this.wait,
+      trace = this.trace
+    } = opts
+    return this.sub(activeRequests, timeout, wait, trace)
   }
 }
 

package/lib/tree.js

@@ -28,6 +28,22 @@ class Pointer {
     this.changedBy = null
   }
 
+  // Compare two pointers to see if they point to equivalent positions
+  equivalentTo(other) {
+    // EMPTY is a special case that can look equivalent to a first entry
+    // in a hypercore but actually contains different data.
+    if (other === exports.EMPTY) return this === exports.EMPTY
+    if (!other) return false
+
+    return (
+      this.seq === other.seq &&
+      this.offset === other.offset &&
+      this.changed === other.changed &&
+      this.context === other.context &&
+      this.core === other.core
+    )
+  }
+
   retain() {
     this.retained = this.context.cache.retained + 1
   }

package/lib/write.js

@@ -1,8 +1,8 @@
 const b4a = require('b4a')
-const c = require('compact-encoding')
 const { OP_COHORT } = require('./compression.js')
 const { encodeBlock, TYPE_COMPAT, TYPE_LATEST } = require('./encoding.js')
 const { Pointer, KeyPointer, ValuePointer, TreeNode, INSERTED, NEEDS_SPLIT } = require('./tree.js')
+const { inflateValue } = require('./inflate.js')
 
 const PREFERRED_BLOCK_SIZE = 4096
 const INLINE_VALUE_SIZE = 1024
@@ -33,6 +33,7 @@ module.exports = class WriteBatch {
     this.length = length
     this.key = key
     this.type = type
+    this.hasLock = false
     this.closed = false
     this.applied = 0
     this.root = null
@@ -40,14 +41,17 @@ module.exports = class WriteBatch {
   }
 
   tryPut(key, value) {
+    this.checkIfClosed()
     this.ops.push({ put: true, applied: false, key, value })
   }
 
   tryDelete(key) {
+    this.checkIfClosed()
     this.ops.push({ put: false, applied: false, key, value: null })
   }
 
   tryClear() {
+    this.checkIfClosed()
     this.ops = []
     this.length = 0
   }
@@ -62,11 +66,25 @@ module.exports = class WriteBatch {
     return root ? root.seq + 1 : 0
   }
 
+  async lock() {
+    if (this.hasLock) return
+    await this.tree.context.lock.lock()
+    this.hasLock = true
+  }
+
+  _unlock() {
+    if (!this.hasLock) return
+    this.tree.context.lock.unlock()
+    this.hasLock = false
+  }
+
   async flush() {
-    const lock = this.tree.context.lock
-    await lock.lock()
+    this.checkIfClosed()
+    await this.lock()
 
     try {
+      this.checkIfClosed()
+
       const ops = this.ops
 
       const root = await this.tree.bootstrap(this.config)
@@ -88,13 +106,13 @@ module.exports = class WriteBatch {
       }
 
       await this._flush()
-      await this.snapshot.close()
+      await this.close()
 
       if (this.autoUpdate) {
-        this.tree.update(this.root)
+        this.tree._setRoot(this.root, true)
       }
     } finally {
-      lock.unlock()
+      this._unlock()
     }
   }
 
@@ -128,7 +146,7 @@ module.exports = class WriteBatch {
         c = b4a.compare(target, m.key)
 
         if (c === 0) {
-          const existing = await snap.inflateValue(m, conf)
+          const existing = await inflateValue(m, conf)
           if (b4a.equals(existing, value)) return false
           v.setValue(this.tree.context, mid, value)
           for (let i = 0; i < stack.length; i++) stack[i].changed = true
@@ -149,7 +167,7 @@ module.exports = class WriteBatch {
     if (status >= 0) {
       // already exists, upsert if changed
       const m = v.keys.uget(status)
-      const existing = await snap.inflateValue(m, conf)
+      const existing = await inflateValue(m, conf)
       if (b4a.equals(existing, value)) return false
       v.setValue(this.tree.context, status, value)
     }
@@ -505,9 +523,7 @@ module.exports = class WriteBatch {
       buffers[i] = encodeBlock(blocks[i])
     }
 
-    if (this.closed) {
-      throw new Error('Write batch is closed')
-    }
+    this.checkIfClosed()
 
     await context.core.append(buffers)
 
@@ -523,6 +539,12 @@ module.exports = class WriteBatch {
     context.cache.retained++
     context.cache.gc()
   }
+
+  checkIfClosed() {
+    if (this.closed) {
+      throw new Error('Write batch is closed')
+    }
+  }
 }
 
 async function toCompatType(context, batch, ops) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hyperbee2",
-  "version": "2.7.1",
+  "version": "2.9.0",
   "description": "Scalable P2P BTree",
   "main": "index.js",
   "files": [