hyperbee2 2.8.0 → 2.9.0

package/index.js

@@ -8,8 +8,8 @@ const NodeCache = require('./lib/cache.js')
 const WriteBatch = require('./lib/write.js')
 const CoreContext = require('./lib/context.js')
 const SessionConfig = require('./lib/session-config.js')
-const { Pointer, KeyPointer, ValuePointer, TreeNode, EMPTY } = require('./lib/tree.js')
-const { DeltaOp, DeltaCohort, OP_COHORT } = require('./lib/compression.js')
+const { inflate, inflateValue } = require('./lib/inflate.js')
+const { EMPTY } = require('./lib/tree.js')
 
 class Hyperbee extends EventEmitter {
   constructor(store, opts = {}) {
@@ -21,10 +21,11 @@ class Hyperbee extends EventEmitter {
       encryption = null,
       getEncryptionProvider = toEncryptionProvider(encryption),
       maxCacheSize = 4096,
-      config = new SessionConfig([], 0, true),
+      config = new SessionConfig([], 0, true, null),
       activeRequests = config.activeRequests,
       timeout = config.timeout,
       wait = config.wait,
+      trace = config.trace,
       core = key
         ? store.get({ key, encryption: getEncryptionProvider(key) })
         : store.get({ key, name: 'bee', encryption: getEncryptionProvider(key) }),
@@ -40,14 +41,14 @@ class Hyperbee extends EventEmitter {
       view = false,
       writable = true,
       unbatch = 0,
-      autoUpdate = false,
+      autoUpdate = !writable && !view,
       preload = null
     } = opts
 
     this.store = store
     this.root = root
     this.context = context
-    this.config = config.sub(activeRequests, timeout, wait)
+    this.config = config.sub(activeRequests, timeout, wait, trace)
     this.view = view
     this.writable = writable
     this.unbatch = unbatch
@@ -117,12 +118,9 @@ class Hyperbee extends EventEmitter {
   }
 
   move({ length = this.core.length, key = null, writable = this.writable } = {}) {
-    const context = key ? this.context.getContextByKey(key) : this.context
-    const root = length === 0 ? EMPTY : context.createTreeNode(0, length - 1, 0, false, null)
-    this.context = context
+    this.context = key ? this.context.getContextByKey(key) : this.context
     this.writable = writable
-    this.root = root
-    this.emit('update')
+    this._setRoot(this._nodeAtSeq(length - 1), true)
   }
 
   snapshot() {
@@ -138,21 +136,23 @@ class Hyperbee extends EventEmitter {
     return new WriteBatch(this, opts)
   }
 
+  _lastNodeInCore() {
+    return this._nodeAtSeq(this.context.core.length - 1)
+  }
+
+  _nodeAtSeq(seq) {
+    return seq < 0 ? EMPTY : this.context.createTreeNode(0, seq, 0, false, null)
+  }
+
   async ready() {
     if (!this.core.opened) await this.core.ready()
     if (this.root) return
     if (this.preload) await this.preload()
     if (this.root) return
 
-    this.root =
-      this.context.core.length === 0
-        ? EMPTY
-        : this.context.createTreeNode(0, this.core.length - 1, 0, false, null)
-
+    this._setRoot(this._lastNodeInCore(), false)
     if (this.autoUpdate) {
-      this.core.on('append', () => {
-        this.update()
-      })
+      this.core.on('append', () => this._setRoot(this._lastNodeInCore(), true))
     }
 
     this.emit('ready')
@@ -202,45 +202,16 @@ class Hyperbee extends EventEmitter {
     return ptr.value
   }
 
-  // TODO: unslab these
   async inflate(ptr, config) {
-    if (ptr.value) {
-      this.bump(ptr)
-      return ptr.value
-    }
-
-    const [block, context] = await Promise.all([
-      ptr.context.getBlock(ptr.seq, ptr.core, config),
-      ptr.context.getContext(ptr.core, config)
-    ])
-
-    const tree = block.tree[ptr.offset]
-
-    const keys = new Array(tree.keys.length)
-    const children = new Array(tree.children.length)
-
-    for (let i = 0; i < keys.length; i++) {
-      const d = tree.keys[i]
-      keys[i] = inflateKey(context, d, ptr, block, config)
-    }
-
-    for (let i = 0; i < children.length; i++) {
-      const d = tree.children[i]
-      children[i] = inflateChild(context, d, ptr, block, config)
+    if (!ptr.value) {
+      await inflate(ptr, config)
     }
-
-    const [k, c] = await Promise.all([Promise.all(keys), Promise.all(children)])
-
-    const value = new TreeNode(k, c)
-    if (!ptr.value) ptr.value = value
-
     this.bump(ptr)
-
     return ptr.value
   }
 
   async finalizeKeyPointer(key, config) {
-    const value = key.value || (await this.inflateValue(key, config))
+    const value = key.value || (await inflateValue(key, config))
 
     return {
       core: key.context.getCore(key.core),
@@ -251,30 +222,6 @@ class Hyperbee extends EventEmitter {
     }
   }
 
-  async inflateValue(key, config) {
-    if (key.value) return key.value
-    if (!key.valuePointer) return null
-
-    const ptr = key.valuePointer
-
-    if (ptr.split === 0) {
-      const block = await ptr.context.getBlock(ptr.seq, ptr.core, config)
-      return block.values[ptr.offset]
-    }
-
-    const blockPromises = new Array(ptr.split + 1)
-    for (let i = 0; i < blockPromises.length; i++) {
-      blockPromises[i] = ptr.context.getBlock(ptr.seq - ptr.split + i, ptr.core, config)
-    }
-    const blocks = await Promise.all(blockPromises)
-    const splitValue = new Array(blockPromises.length)
-    for (let i = 0; i < splitValue.length - 1; i++) {
-      splitValue[i] = blocks[i].values[0]
-    }
-    splitValue[splitValue.length - 1] = blocks[blocks.length - 1].buffer[ptr.offset]
-    return b4a.concat(splitValue)
-  }
-
   async bootstrap(config) {
     if (!this.root) await this.ready()
     if (this.unbatch) await this._rollback(config)
@@ -304,16 +251,16 @@ class Hyperbee extends EventEmitter {
 
     if (expected === this.unbatch) {
       this.context = context
-      this.root = length === 0 ? EMPTY : context.createTreeNode(0, length - 1, 0, false, null)
-      this.unbatch = 0
-      this.emit('update')
+      this._setRoot(this._nodeAtSeq(length - 1), true)
     }
   }
 
-  update(root = null) {
-    this.root = root
+  _setRoot(root, emit) {
+    if (!root.equivalentTo(this.root)) {
+      this.root = root
+      if (emit) this.emit('update')
+    }
     this.unbatch = 0
-    this.emit('update')
   }
 
   async get(key, opts) {
@@ -353,89 +300,6 @@ module.exports = Hyperbee
 
 function noop() {}
 
-function inflateKey(context, d, ptr, block, config) {
-  if (d.type === OP_COHORT) return inflateKeyCohort(context, d, ptr, block, config)
-  return inflateKeyDelta(context, d, ptr, block, config)
-}
-
-async function inflateKeyDelta(context, d, ptr, block, config) {
-  const k = d.pointer
-
-  if (!k) return new DeltaOp(false, d.type, d.index, null)
-
-  const blk =
-    k.seq === ptr.seq && k.core === 0 && ptr.core === 0
-      ? block
-      : await context.getBlock(k.seq, k.core, config)
-
-  const bk = blk.keys[k.offset]
-
-  let vp = null
-
-  if (bk.valuePointer) {
-    const p = bk.valuePointer
-    const ctx = await context.getContext(k.core, config)
-    vp = new ValuePointer(ctx, p.core, p.seq, p.offset, p.split)
-  }
-
-  const kp = new KeyPointer(context, k.core, k.seq, k.offset, false, bk.key, bk.value, vp)
-  return new DeltaOp(false, d.type, d.index, kp)
-}
-
-async function inflateKeyCohort(context, d, ptr, block, config) {
-  const co = d.pointer
-
-  const blk =
-    co.seq === ptr.seq && co.core === 0 && ptr.core === 0
-      ? block
-      : await context.getBlock(co.seq, co.core, config)
-
-  const cohort = blk.cohorts[co.offset]
-  const promises = new Array(cohort.length)
-
-  for (let i = 0; i < cohort.length; i++) {
-    const p = cohort[i]
-    const k = inflateKeyDelta(context, p, co, blk, config)
-    promises[i] = k
-  }
-
-  const p = new Pointer(context, co.core, co.seq, co.offset)
-  return new DeltaCohort(false, p, await Promise.all(promises))
-}
-
-async function inflateChild(context, d, ptr, block, config) {
-  if (d.type === OP_COHORT) return inflateChildCohort(context, d, ptr, block, config)
-  if (d.pointer && !context.hasCore(d.pointer.core)) await context.update(config)
-  return inflateChildDelta(context, d, ptr, block, config)
-}
-
-function inflateChildDelta(context, d, ptr, block, config) {
-  const p = d.pointer
-  const c = p && context.createTreeNode(p.core, p.seq, p.offset, false, null)
-  return new DeltaOp(false, d.type, d.index, c)
-}
-
-async function inflateChildCohort(context, d, ptr, block, config) {
-  const co = d.pointer
-
-  const blk =
-    co.seq === ptr.seq && co.core === 0 && ptr.core === 0
-      ? block
-      : await context.getBlock(co.seq, co.core, config)
-
-  const cohort = blk.cohorts[co.offset]
-  const deltas = new Array(cohort.length)
-
-  for (let i = 0; i < cohort.length; i++) {
-    const c = cohort[i]
-    if (c.pointer && !context.hasCore(c.pointer.core)) await context.update(config)
-    deltas[i] = inflateChildDelta(context, c, co, blk, config)
-  }
-
-  const p = new Pointer(context, co.core, co.seq, co.offset)
-  return new DeltaCohort(false, p, deltas)
-}
-
 function toEncryptionProvider(encryption) {
   if (encryption) return (key) => encryption
   return () => null

package/lib/context.js

@@ -161,6 +161,9 @@ class CoreContext {
     const hc = this.getCore(core)
     const buffer = await hc.get(seq, config)
     if (buffer === null) throw BLOCK_NOT_AVAILABLE()
+
+    if (config.trace !== null) config.trace(core, seq)
+
     const block = decodeBlock(buffer, seq)
     return block
   }

package/lib/inflate.js

@@ -0,0 +1,140 @@
+const b4a = require('b4a')
+const { Pointer, KeyPointer, ValuePointer, TreeNode } = require('./tree.js')
+const { DeltaOp, DeltaCohort, OP_COHORT } = require('./compression.js')
+
+exports.inflate = async function inflate(ptr, config) {
+  if (ptr.value) return ptr.value
+
+  const [block, context] = await Promise.all([
+    ptr.context.getBlock(ptr.seq, ptr.core, config),
+    ptr.context.getContext(ptr.core, config)
+  ])
+
+  const tree = block.tree[ptr.offset]
+
+  const keys = new Array(tree.keys.length)
+  const children = new Array(tree.children.length)
+
+  for (let i = 0; i < keys.length; i++) {
+    const d = tree.keys[i]
+    keys[i] = inflateKey(context, d, ptr, block, config)
+  }
+
+  for (let i = 0; i < children.length; i++) {
+    const d = tree.children[i]
+    children[i] = inflateChild(context, d, ptr, block, config)
+  }
+
+  const [k, c] = await Promise.all([Promise.all(keys), Promise.all(children)])
+
+  const value = new TreeNode(k, c)
+  if (!ptr.value) ptr.value = value
+  return ptr.value
+}
+
+function inflateKey(context, d, ptr, block, config) {
+  if (d.type === OP_COHORT) return inflateKeyCohort(context, d, ptr, block, config)
+  return inflateKeyDelta(context, d, ptr, block, config)
+}
+
+async function inflateKeyDelta(context, d, ptr, block, config) {
+  const k = d.pointer
+
+  if (!k) return new DeltaOp(false, d.type, d.index, null)
+
+  const blk =
+    k.seq === ptr.seq && k.core === 0 && ptr.core === 0
+      ? block
+      : await context.getBlock(k.seq, k.core, config)
+
+  const bk = blk.keys[k.offset]
+
+  let vp = null
+
+  if (bk.valuePointer) {
+    const p = bk.valuePointer
+    const ctx = await context.getContext(k.core, config)
+    vp = new ValuePointer(ctx, p.core, p.seq, p.offset, p.split)
+  }
+
+  const kp = new KeyPointer(context, k.core, k.seq, k.offset, false, bk.key, bk.value, vp)
+  return new DeltaOp(false, d.type, d.index, kp)
+}
+
+exports.inflateValue = async function inflateValue(key, config) {
+  if (key.value) return key.value
+  if (!key.valuePointer) return null
+
+  const ptr = key.valuePointer
+
+  if (ptr.split === 0) {
+    const block = await ptr.context.getBlock(ptr.seq, ptr.core, config)
+    return block.values[ptr.offset]
+  }
+
+  const blockPromises = new Array(ptr.split + 1)
+  for (let i = 0; i < blockPromises.length; i++) {
+    blockPromises[i] = ptr.context.getBlock(ptr.seq - ptr.split + i, ptr.core, config)
+  }
+  const blocks = await Promise.all(blockPromises)
+  const splitValue = new Array(blockPromises.length)
+  for (let i = 0; i < splitValue.length - 1; i++) {
+    splitValue[i] = blocks[i].values[0]
+  }
+  splitValue[splitValue.length - 1] = blocks[blocks.length - 1].buffer[ptr.offset]
+  return b4a.concat(splitValue)
+}
+
+async function inflateKeyCohort(context, d, ptr, block, config) {
+  const co = d.pointer
+
+  const blk =
+    co.seq === ptr.seq && co.core === 0 && ptr.core === 0
+      ? block
+      : await context.getBlock(co.seq, co.core, config)
+
+  const cohort = blk.cohorts[co.offset]
+  const promises = new Array(cohort.length)
+
+  for (let i = 0; i < cohort.length; i++) {
+    const p = cohort[i]
+    const k = inflateKeyDelta(context, p, co, blk, config)
+    promises[i] = k
+  }
+
+  const p = new Pointer(context, co.core, co.seq, co.offset)
+  return new DeltaCohort(false, p, await Promise.all(promises))
+}
+
+async function inflateChild(context, d, ptr, block, config) {
+  if (d.type === OP_COHORT) return inflateChildCohort(context, d, ptr, block, config)
+  if (d.pointer && !context.hasCore(d.pointer.core)) await context.update(config)
+  return inflateChildDelta(context, d, ptr, block, config)
+}
+
+function inflateChildDelta(context, d, ptr, block, config) {
+  const p = d.pointer
+  const c = p && context.createTreeNode(p.core, p.seq, p.offset, false, null)
+  return new DeltaOp(false, d.type, d.index, c)
+}
+
+async function inflateChildCohort(context, d, ptr, block, config) {
+  const co = d.pointer
+
+  const blk =
+    co.seq === ptr.seq && co.core === 0 && ptr.core === 0
+      ? block
+      : await context.getBlock(co.seq, co.core, config)
+
+  const cohort = blk.cohorts[co.offset]
+  const deltas = new Array(cohort.length)
+
+  for (let i = 0; i < cohort.length; i++) {
+    const c = cohort[i]
+    if (c.pointer && !context.hasCore(c.pointer.core)) await context.update(config)
+    deltas[i] = inflateChildDelta(context, c, co, blk, config)
+  }
+
+  const p = new Pointer(context, co.core, co.seq, co.offset)
+  return new DeltaCohort(false, p, deltas)
+}

package/lib/session-config.js

@@ -1,22 +1,33 @@
 class SessionConfig {
-  constructor(activeRequests, timeout, wait) {
+  constructor(activeRequests, timeout, wait, trace) {
     this.activeRequests = activeRequests
     this.timeout = timeout
     this.wait = wait
+    this.trace = trace
   }
 
-  sub(activeRequests, timeout, wait) {
-    if (this.activeRequests === activeRequests && this.timeout === timeout && this.wait === wait) {
+  sub(activeRequests, timeout, wait, trace) {
+    if (
+      this.activeRequests === activeRequests &&
+      this.timeout === timeout &&
+      this.wait === wait &&
+      this.trace === trace
+    ) {
       return this
     }
 
-    return new SessionConfig(activeRequests, timeout, wait)
+    return new SessionConfig(activeRequests, timeout, wait, trace)
   }
 
   options(opts) {
     if (!opts) return this
-    const { activeRequests = this.activeRequests, timeout = this.timeout, wait = this.wait } = opts
-    return this.sub(activeRequests, timeout, wait)
+    const {
+      activeRequests = this.activeRequests,
+      timeout = this.timeout,
+      wait = this.wait,
+      trace = this.trace
+    } = opts
+    return this.sub(activeRequests, timeout, wait, trace)
   }
 }
 

package/lib/tree.js

@@ -28,6 +28,22 @@ class Pointer {
     this.changedBy = null
   }
 
+  // Compare two pointers to see if they point to equivalent positions
+  equivalentTo(other) {
+    // EMPTY is a special case that can look equivalent to a first entry
+    // in a hypercore but actually contains different data.
+    if (other === exports.EMPTY) return this === exports.EMPTY
+    if (!other) return false
+
+    return (
+      this.seq === other.seq &&
+      this.offset === other.offset &&
+      this.changed === other.changed &&
+      this.context === other.context &&
+      this.core === other.core
+    )
+  }
+
   retain() {
     this.retained = this.context.cache.retained + 1
   }

package/lib/write.js

@@ -1,8 +1,8 @@
 const b4a = require('b4a')
-const c = require('compact-encoding')
 const { OP_COHORT } = require('./compression.js')
 const { encodeBlock, TYPE_COMPAT, TYPE_LATEST } = require('./encoding.js')
 const { Pointer, KeyPointer, ValuePointer, TreeNode, INSERTED, NEEDS_SPLIT } = require('./tree.js')
+const { inflateValue } = require('./inflate.js')
 
 const PREFERRED_BLOCK_SIZE = 4096
 const INLINE_VALUE_SIZE = 1024
@@ -41,14 +41,17 @@ module.exports = class WriteBatch {
   }
 
   tryPut(key, value) {
+    this.checkIfClosed()
     this.ops.push({ put: true, applied: false, key, value })
   }
 
   tryDelete(key) {
+    this.checkIfClosed()
     this.ops.push({ put: false, applied: false, key, value: null })
   }
 
   tryClear() {
+    this.checkIfClosed()
     this.ops = []
     this.length = 0
   }
@@ -76,9 +79,12 @@ module.exports = class WriteBatch {
   }
 
   async flush() {
+    this.checkIfClosed()
     await this.lock()
 
     try {
+      this.checkIfClosed()
+
       const ops = this.ops
 
       const root = await this.tree.bootstrap(this.config)
@@ -100,10 +106,10 @@ module.exports = class WriteBatch {
       }
 
       await this._flush()
-      await this.snapshot.close()
+      await this.close()
 
       if (this.autoUpdate) {
-        this.tree.update(this.root)
+        this.tree._setRoot(this.root, true)
       }
     } finally {
       this._unlock()
@@ -140,7 +146,7 @@ module.exports = class WriteBatch {
         c = b4a.compare(target, m.key)
 
         if (c === 0) {
-          const existing = await snap.inflateValue(m, conf)
+          const existing = await inflateValue(m, conf)
           if (b4a.equals(existing, value)) return false
           v.setValue(this.tree.context, mid, value)
           for (let i = 0; i < stack.length; i++) stack[i].changed = true
@@ -161,7 +167,7 @@ module.exports = class WriteBatch {
     if (status >= 0) {
       // already exists, upsert if changed
       const m = v.keys.uget(status)
-      const existing = await snap.inflateValue(m, conf)
+      const existing = await inflateValue(m, conf)
       if (b4a.equals(existing, value)) return false
       v.setValue(this.tree.context, status, value)
     }
@@ -517,9 +523,7 @@ module.exports = class WriteBatch {
       buffers[i] = encodeBlock(blocks[i])
     }
 
-    if (this.closed) {
-      throw new Error('Write batch is closed')
-    }
+    this.checkIfClosed()
 
     await context.core.append(buffers)
 
@@ -535,6 +539,12 @@ module.exports = class WriteBatch {
     context.cache.retained++
     context.cache.gc()
   }
+
+  checkIfClosed() {
+    if (this.closed) {
+      throw new Error('Write batch is closed')
+    }
+  }
 }
 
 async function toCompatType(context, batch, ops) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hyperbee2",
-  "version": "2.8.0",
+  "version": "2.9.0",
   "description": "Scalable P2P BTree",
   "main": "index.js",
   "files": [