hyperbee2 2.2.0 → 2.4.1

package/lib/context.js

@@ -1,11 +1,14 @@
 const b4a = require('b4a')
 const ScopeLock = require('scope-lock')
+const { BLOCK_NOT_AVAILABLE } = require('hypercore-errors')
+const { TreeNodePointer } = require('./tree.js')
 const { decodeBlock } = require('./encoding.js')
 
 class CoreContext {
-  constructor(store, local, core, encryption, t, lock = new ScopeLock(), other = new Map()) {
+  constructor(store, local, cache, core, encryption, t, lock = new ScopeLock(), other = new Map()) {
     this.store = store
     this.local = local
+    this.cache = cache
     this.core = core
     this.t = t
     this.minKeys = t - 1
@@ -25,20 +28,26 @@ class CoreContext {
   //   return '[CoreContext]'
   // }
 
-  async update(activeRequests) {
+  async update(config) {
     await this.core.ready()
 
     if (this.length === this.core.length || this.core.length === 0) return
 
     const length = this.core.length
     const seq = length - 1
-    const buffer = await this.core.get(seq, { activeRequests })
+    const buffer = await this.core.get(seq, config)
     const block = decodeBlock(buffer, seq)
+
+    await this._inflateCheckpoint(length, block)
+  }
+
+  async _inflateCheckpoint(length, block) {
     const checkpoint = block.checkpoint
 
     if (checkpoint > 0) {
-      const buffer = await this.core.get(checkpoint - 1)
-      const block = decodeBlock(buffer, checkpoint - 1)
+      const seq = checkpoint - 1
+      const buffer = await this.core.get(seq)
+      const block = decodeBlock(buffer, seq)
 
       if (length < this.length) return
 
@@ -52,6 +61,14 @@ class CoreContext {
     this.length = length
   }
 
+  createTreeNode(core, seq, offset, changed, value) {
+    const ptr = new TreeNodePointer(this, core, seq, offset, changed, value)
+    if (ptr.value || changed) return ptr
+    const existing = this.cache.get(ptr)
+    if (existing) return existing
+    return ptr
+  }
+
   getCoreOffsetLocal(context, core) {
     if (context === this) return core
 
@@ -74,12 +91,12 @@ class CoreContext {
     return this.cores.length
   }
 
-  async getCoreOffset(context, core, activeRequests) {
-    if (core !== 0 && core - 1 >= context.cores.length) await context.update(activeRequests)
+  async getCoreOffset(context, core, config) {
+    if (core !== 0 && core - 1 >= context.cores.length) await context.update(config)
     return this.getCoreOffsetLocal(context, core)
   }
 
-  async getCoreOffsetByKey(key, activeRequests) {
+  async getCoreOffsetByKey(key, config) {
     await this.core.ready()
 
     if (b4a.equals(key, this.core.key)) return 0
@@ -91,7 +108,7 @@ class CoreContext {
       }
     }
 
-    await this.update(activeRequests)
+    await this.update(config)
 
     for (let i = 0; i < this.cores.length; i++) {
       const k = this.cores[i].key
@@ -125,11 +142,18 @@ class CoreContext {
     return this.cores[index - 1].key
   }
 
-  async getBlock(seq, core, activeRequests) {
-    if (core !== 0 && core - 1 >= this.cores.length) await this.update(activeRequests)
+  async getBlock(seq, core, config) {
+    if (core !== 0 && core - 1 >= this.cores.length) await this.update(config)
+
     const hc = this.getCore(core)
-    const buffer = await hc.get(seq, { activeRequests })
+    const buffer = await hc.get(seq, config)
+    if (buffer === null) throw BLOCK_NOT_AVAILABLE()
     const block = decodeBlock(buffer, seq)
+
+    if (block.checkpoint > this.checkpoint) {
+      await this._inflateCheckpoint(seq + 1, block)
+    }
+
     return block
   }
 
@@ -148,9 +172,9 @@ class CoreContext {
     return ctx
   }
 
-  async getContext(core, activeRequests) {
+  async getContext(core, config) {
     if (core === 0) return this
-    if (core > this.cores.length) await this.update(activeRequests)
+    if (core > this.cores.length) await this.update(config)
     if (core > this.cores.length) throw new Error('Bad core index: ' + core)
 
     const hex = b4a.toString(this.cores[core - 1].key, 'hex')
@@ -171,11 +195,12 @@ class CoreContext {
   _createContext(core) {
     const store = this.store
     const local = this.local
+    const cache = this.cache
     const encryption = this.encryption
     const t = this.t
     const lock = this.lock
     const other = this.other
-    return new CoreContext(store, local, core, encryption, t, lock, other)
+    return new CoreContext(store, local, cache, core, encryption, t, lock, other)
   }
 }
 

package/lib/diff.js

@@ -3,7 +3,9 @@ const b4a = require('b4a')
 const { RangeIterator } = require('./ranges.js')
 
 class DiffIterator {
-  constructor(tree, left, right, { limit = -1, activeRequests = [] } = {}) {
+  constructor(tree, left, right, opts = {}) {
+    const { limit = -1 } = opts
+
     this.tree = tree
 
     this.left = left
@@ -11,7 +13,7 @@ class DiffIterator {
 
     this.left.limit = this.right.limit = -1
 
-    this.activeRequests = activeRequests
+    this.config = tree.config.options(opts)
     this.limit = limit
 
     this.nextLeft = null
@@ -23,14 +25,14 @@ class DiffIterator {
   }
 
   async _shiftRight() {
-    const right = await this.tree.finalizeKeyPointer(this.nextRight, this.activeRequests)
+    const right = await this.tree.finalizeKeyPointer(this.nextRight, this.config)
     this.nextRight = null
     if (this.limit > 0) this.limit--
     return { left: null, right }
   }
 
   async _shiftLeft() {
-    const left = await this.tree.finalizeKeyPointer(this.nextLeft, this.activeRequests)
+    const left = await this.tree.finalizeKeyPointer(this.nextLeft, this.config)
     this.nextLeft = null
     if (this.limit > 0) this.limit--
     return { left, right: null }
@@ -77,8 +79,8 @@ class DiffIterator {
         }
 
         const [left, right] = await Promise.all([
-          this.tree.finalizeKeyPointer(leftKey, this.activeRequests),
-          this.tree.finalizeKeyPointer(rightKey, this.activeRequests)
+          this.tree.finalizeKeyPointer(leftKey, this.config),
+          this.tree.finalizeKeyPointer(rightKey, this.config)
         ])
 
         if (this.limit > 0) this.limit--

package/lib/ranges.js

@@ -3,21 +3,12 @@ const b4a = require('b4a')
 
 class RangeIterator {
   constructor(tree, opts = {}) {
-    const {
-      activeRequests = [],
-      prefetch = true,
-      reverse = false,
-      limit = -1,
-      gte,
-      gt,
-      lte,
-      lt
-    } = opts
+    const { prefetch = true, reverse = false, limit = -1, gte, gt, lte, lt } = opts
 
     this.tree = tree
     this.root = null
     this.stack = []
-    this.activeRequests = activeRequests
+    this.config = tree.config.options(opts)
     this.reverse = reverse
     this.limit = limit
     this.start = gte || gt || null
@@ -29,7 +20,7 @@ class RangeIterator {
   }
 
   async open() {
-    if (!this.root) this.root = await this.tree.bootstrap(this.activeRequests)
+    if (!this.root) this.root = await this.tree.bootstrap(this.config)
     if (!this.root) return
 
     if (this.limit === 0) return
@@ -44,7 +35,7 @@ class RangeIterator {
       const top = this.stack[this.stack.length - 1]
       const v = top.node.value
         ? this.tree.bump(top.node)
-        : await this.tree.inflate(top.node, this.activeRequests)
+        : await this.tree.inflate(top.node, this.config)
 
       for (let i = 0; i < v.keys.length; i++) {
         const j = this.reverse ? v.keys.length - 1 - i : i
@@ -75,7 +66,7 @@ class RangeIterator {
 
   async next() {
     const key = await this.nextKey()
-    return key === null ? key : this.tree.finalizeKeyPointer(key, this.activeRequests)
+    return key === null ? key : this.tree.finalizeKeyPointer(key, this.config)
   }
 
   async nextKey() {
@@ -83,7 +74,7 @@ class RangeIterator {
       const top = this.stack.pop()
       const v = top.node.value
         ? this.tree.bump(top.node)
-        : await this.tree.inflate(top.node, this.activeRequests)
+        : await this.tree.inflate(top.node, this.config)
 
       const offset = top.offset++
       const child = (offset & 1) === 0
@@ -150,7 +141,7 @@ class RangeIterator {
       if (cmp > this.compare) break
 
       const c = pv.children.get(i)
-      if (!c.value) this.tree.inflate(c, this.activeRequests).catch(noop)
+      if (!c.value) this.tree.inflate(c, this.config).catch(noop)
 
       limit = Math.max(limit - this.tree.context.minKeys, 0)
       if (limit === 0) break

package/lib/session-config.js

@@ -0,0 +1,23 @@
+class SessionConfig {
+  constructor(activeRequests, timeout, wait) {
+    this.activeRequests = activeRequests
+    this.timeout = timeout
+    this.wait = wait
+  }
+
+  sub(activeRequests, timeout, wait) {
+    if (this.activeRequests === activeRequests && this.timeout === timeout && this.wait === wait) {
+      return this
+    }
+
+    return new SessionConfig(activeRequests, timeout, wait)
+  }
+
+  options(opts) {
+    if (!opts) return this
+    const { activeRequests = this.activeRequests, timeout = this.timeout, wait = this.wait } = opts
+    return this.sub(activeRequests, timeout, wait)
+  }
+}
+
+module.exports = SessionConfig

package/lib/tree.js

@@ -23,9 +23,14 @@ class Pointer {
     this.seq = seq
     this.offset = offset
     this.changed = changed
+    this.retained = 0
 
     this.changedBy = null
   }
+
+  retain() {
+    this.retained = this.context.cache.retained + 1
+  }
 }
 
 class KeyPointer extends Pointer {
@@ -38,11 +43,9 @@ class KeyPointer extends Pointer {
   }
 
   // TODO: remove, left here for easier debugging for now
-  // [Symbol.for('nodejs.util.inspect.custom')]() {
-  //   return (
-  //     `[KeyPointer core=${this.core} seq=${this.seq}, offset=${this.offset}, key="${b4a.toString(this.key)}"]`
-  //   )
-  // }
+  [Symbol.for('nodejs.util.inspect.custom')]() {
+    return `[KeyPointer core=${this.core} seq=${this.seq}, offset=${this.offset}, key="${b4a.toString(this.key)}"]`
+  }
 }
 
 class TreeNodePointer extends Pointer {
@@ -55,12 +58,19 @@ class TreeNodePointer extends Pointer {
     this.prev = null
   }
 
+  commit() {
+    this.changed = false
+    const value = new TreeNode([], [])
+    value.keys = this.value.keys.commit()
+    value.children = this.value.children.commit()
+    const ptr = this.context.createTreeNode(this.core, 0, 0, true, value)
+    return ptr
+  }
+
   // TODO: remove, left here for easier debugging for now
-  // [Symbol.for('nodejs.util.inspect.custom')]() {
-  //   return (
-  //     `[TreeNodePointer core=${this.core} seq=${this.seq} offset=${this.offset} changed=${this.changed}]`
-  //   )
-  // }
+  [Symbol.for('nodejs.util.inspect.custom')]() {
+    return `[TreeNodePointer core=${this.core} seq=${this.seq} offset=${this.offset} changed=${this.changed}]`
+  }
 }
 
 class TreeNode {
@@ -70,17 +80,17 @@ class TreeNode {
   }
 
   isEmpty() {
-    return this.keys.length === 0 && this.children.length === 0
+    return this.keys.ulength === 0 && this.children.ulength === 0
   }
 
   insertLeaf(context, key, value) {
     let s = 0
-    let e = this.keys.length
+    let e = this.keys.ulength
     let c = 0
 
     while (s < e) {
       const mid = (s + e) >> 1
-      const k = this.keys.get(mid)
+      const k = this.keys.uget(mid)
 
       c = b4a.compare(key, k.key)
 
@@ -91,19 +101,20 @@ class TreeNode {
     }
 
     const i = c < 0 ? e : s
+
     this.keys.insert(i, new KeyPointer(context, 0, 0, 0, true, key, value, null))
 
-    return this.keys.length <= context.maxKeys ? INSERTED : NEEDS_SPLIT
+    return this.keys.ulength <= context.maxKeys ? INSERTED : NEEDS_SPLIT
   }
 
   insertNode(context, keyPointer, treePointer) {
     let s = 0
-    let e = this.keys.length
+    let e = this.keys.ulength
     let c = 0
 
     while (s < e) {
       const mid = (s + e) >> 1
-      const k = this.keys.get(mid)
+      const k = this.keys.uget(mid)
 
       c = b4a.compare(keyPointer.key, k.key)
 
@@ -117,16 +128,16 @@ class TreeNode {
     this.keys.insert(i, keyPointer)
     this.children.insert(i + 1, treePointer)
 
-    return this.keys.length <= context.maxKeys ? INSERTED : NEEDS_SPLIT
+    return this.keys.ulength <= context.maxKeys ? INSERTED : NEEDS_SPLIT
   }
 
   setValue(context, i, value) {
-    this.keys.set(i, new KeyPointer(context, 0, 0, 0, true, this.keys.get(i).key, value, null))
+    this.keys.set(i, new KeyPointer(context, 0, 0, 0, true, this.keys.uget(i).key, value, null))
   }
 
   removeKey(i) {
     this.keys.delete(i)
-    if (this.children.length) {
+    if (this.children.ulength) {
       this.children.delete(i + 1)
     }
   }
@@ -134,11 +145,11 @@ class TreeNode {
   siblings(parent) {
     const pc = parent.children
 
-    for (let i = 0; i < pc.length; i++) {
-      if (pc.get(i).value !== this) continue // TODO: move to a seq/offset check instead
+    for (let i = 0; i < pc.ulength; i++) {
+      if (pc.uget(i).value !== this) continue // TODO: move to a seq/offset check instead
 
-      const left = i ? pc.get(i - 1) : null
-      const right = i < pc.length - 1 ? pc.get(i + 1) : null
+      const left = i ? pc.uget(i - 1) : null
+      const right = i < pc.ulength - 1 ? pc.uget(i + 1) : null
       return { left, index: i, right }
     }
 
@@ -152,13 +163,13 @@ class TreeNode {
 
     this.keys.push(median)
 
-    for (let i = 0; i < keys.length; i++) this.keys.push(keys.get(i))
-    for (let i = 0; i < children.length; i++) this.children.push(children.get(i))
+    for (let i = 0; i < keys.ulength; i++) this.keys.push(keys.uget(i))
+    for (let i = 0; i < children.ulength; i++) this.children.push(children.uget(i))
   }
 
   split(context) {
-    const len = this.keys.length >> 1
-    const right = new TreeNodePointer(context, 0, 0, 0, true, new TreeNode([], []))
+    const len = this.keys.ulength >> 1
+    const right = context.createTreeNode(0, 0, 0, true, new TreeNode([], []))
 
     const k = []
     while (k.length < len) k.push(this.keys.pop())
@@ -166,7 +177,7 @@ class TreeNode {
 
     const median = this.keys.pop()
 
-    if (this.children.length) {
+    if (this.children.ulength) {
       const c = []
       while (c.length < len + 1) c.push(this.children.pop())
       for (let i = c.length - 1; i >= 0; i--) right.value.children.push(c[i])