hyperbee2 2.2.0 → 2.4.1

package/index.js

@@ -6,14 +6,8 @@ const { ChangesStream } = require('./lib/changes.js')
 const NodeCache = require('./lib/cache.js')
 const WriteBatch = require('./lib/write.js')
 const CoreContext = require('./lib/context.js')
-const {
-  Pointer,
-  KeyPointer,
-  ValuePointer,
-  TreeNode,
-  TreeNodePointer,
-  EMPTY
-} = require('./lib/tree.js')
+const SessionConfig = require('./lib/session-config.js')
+const { Pointer, KeyPointer, ValuePointer, TreeNode, EMPTY } = require('./lib/tree.js')
 const { DeltaOp, DeltaCohort, OP_COHORT } = require('./lib/compression.js')
 
 class Hyperbee {
@@ -22,12 +16,14 @@ class Hyperbee {
       t = 128, // legacy number for now, should be 128 now
       key = null,
       encryption = null,
-      core = key ? store.get(key) : store.get({ key, name: 'bee', encryption }),
-      context = new CoreContext(store, core, core, encryption, t),
       maxCacheSize = 4096,
-      cache = new NodeCache(maxCacheSize),
+      config = new SessionConfig([], 0, true),
+      activeRequests = config.activeRequests,
+      timeout = config.timeout,
+      wait = config.wait,
+      core = key ? store.get(key) : store.get({ key, name: 'bee', encryption }),
+      context = new CoreContext(store, core, new NodeCache(maxCacheSize), core, encryption, t),
       root = null,
-      activeRequests = [],
       view = false,
       writable = true,
       unbatch = 0,
@@ -36,9 +32,8 @@ class Hyperbee {
 
     this.store = store
     this.root = root
-    this.cache = cache
     this.context = context
-    this.activeRequests = activeRequests
+    this.config = config.sub(activeRequests, timeout, wait)
     this.view = view
     this.writable = writable
     this.unbatch = unbatch
@@ -58,6 +53,10 @@ class Hyperbee {
     return { length: this.root.seq + 1, key: this.root.context.core.key }
   }
 
+  get cache() {
+    return this.context.cache
+  }
+
   get core() {
     return this.context.core
   }
@@ -84,8 +83,9 @@ class Hyperbee {
 
   _makeView(context, root, writable, unbatch) {
     return new Hyperbee(this.store, {
+      config: this.config,
+      core: context.core,
       context,
-      cache: this.cache,
       root,
       view: true,
       writable,
@@ -95,7 +95,7 @@ class Hyperbee {
 
   checkout({ length = this.core.length, key = null, writable = false } = {}) {
     const context = key ? this.context.getContextByKey(key) : this.context
-    const root = length === 0 ? EMPTY : new TreeNodePointer(context, 0, length - 1, 0, false, null)
+    const root = length === 0 ? EMPTY : context.createTreeNode(0, length - 1, 0, false, null)
     return this._makeView(context, root, writable, 0)
   }
 
@@ -119,7 +119,7 @@ class Hyperbee {
     this.root =
       this.context.core.length === 0
         ? EMPTY
-        : new TreeNodePointer(this.context, 0, this.core.length - 1, 0, false, null)
+        : this.context.createTreeNode(0, this.core.length - 1, 0, false, null)
 
     if (this._autoUpdate) {
       this.core.on('append', () => {
@@ -129,7 +129,7 @@ class Hyperbee {
   }
 
   async close() {
-    if (this.activeRequests.length) Hypercore.clearRequests(this.activeRequests)
+    if (this.config.activeRequests.length) Hypercore.clearRequests(this.config.activeRequests)
     if (!this.view) await this.store.close()
   }
 
@@ -163,21 +163,21 @@ class Hyperbee {
 
   bump(ptr) {
     if (ptr.changed) return ptr.value
-    this.cache.bump(ptr)
-    this.cache.gc()
+    this.context.cache.bump(ptr)
+    this.context.cache.gc()
     return ptr.value
   }
 
-  // TODO: unslab these and parallize
-  async inflate(ptr, activeRequests = this.activeRequests) {
+  // TODO: unslab these
+  async inflate(ptr, config) {
     if (ptr.value) {
       this.bump(ptr)
       return ptr.value
     }
 
     const [block, context] = await Promise.all([
-      ptr.context.getBlock(ptr.seq, ptr.core, activeRequests),
-      ptr.context.getContext(ptr.core, activeRequests)
+      ptr.context.getBlock(ptr.seq, ptr.core, config),
+      ptr.context.getContext(ptr.core, config)
     ])
 
     const tree = block.tree[ptr.offset]
@@ -187,15 +187,17 @@ class Hyperbee {
 
     for (let i = 0; i < keys.length; i++) {
       const d = tree.keys[i]
-      keys[i] = inflateKey(context, d, ptr, block, activeRequests)
+      keys[i] = inflateKey(context, d, ptr, block, config)
     }
 
     for (let i = 0; i < children.length; i++) {
       const d = tree.children[i]
-      children[i] = inflateChild(context, d, ptr, block, activeRequests)
+      children[i] = inflateChild(context, d, ptr, block, config)
     }
 
-    const value = new TreeNode(await Promise.all(keys), await Promise.all(children))
+    const [k, c] = await Promise.all([Promise.all(keys), Promise.all(children)])
+
+    const value = new TreeNode(k, c)
     if (!ptr.value) ptr.value = value
 
     this.bump(ptr)
@@ -203,8 +205,8 @@ class Hyperbee {
     return ptr.value
   }
 
-  async finalizeKeyPointer(key, activeRequests = this.activeRequests) {
-    const value = key.value || (await this.inflateValue(key, activeRequests))
+  async finalizeKeyPointer(key, config) {
+    const value = key.value || (await this.inflateValue(key, config))
 
     return {
       core: key.context.getCore(key.core),
@@ -215,20 +217,20 @@ class Hyperbee {
     }
   }
 
-  async inflateValue(key, activeRequests = this.activeRequests) {
+  async inflateValue(key, config) {
     if (key.value) return key.value
     if (!key.valuePointer) return null
 
     const ptr = key.valuePointer
 
     if (ptr.split === 0) {
-      const block = await ptr.context.getBlock(ptr.seq, ptr.core, activeRequests)
+      const block = await ptr.context.getBlock(ptr.seq, ptr.core, config)
       return block.values[ptr.offset]
     }
 
     const blockPromises = new Array(ptr.split + 1)
     for (let i = 0; i < blockPromises.length; i++) {
-      blockPromises[i] = ptr.context.getBlock(ptr.seq - ptr.split + i, ptr.core, activeRequests)
+      blockPromises[i] = ptr.context.getBlock(ptr.seq - ptr.split + i, ptr.core, config)
     }
     const blocks = await Promise.all(blockPromises)
     const splitValue = new Array(blockPromises.length)
@@ -239,13 +241,13 @@ class Hyperbee {
     return b4a.concat(splitValue)
   }
 
-  async bootstrap(activeRequests = this.activeRequests) {
+  async bootstrap(config) {
     if (!this.root) await this.ready()
-    if (this.unbatch) await this._rollback(activeRequests)
+    if (this.unbatch) await this._rollback(config)
     return this.root === EMPTY ? null : this.root
   }
 
-  async _rollback(activeRequests) {
+  async _rollback(config) {
     const expected = this.unbatch
 
     let n = expected
@@ -254,21 +256,21 @@ class Hyperbee {
 
     while (n > 0 && length > 0 && expected === this.unbatch) {
       const seq = length - 1
-      const blk = await context.getBlock(seq, 0, activeRequests)
+      const blk = await context.getBlock(seq, 0, config)
 
       if (!blk.previous) {
         length = 0
         break
       }
 
-      context = await context.getContext(blk.previous.core, activeRequests)
+      context = await context.getContext(blk.previous.core, config)
       length = blk.previous.seq + 1
       n--
     }
 
     if (expected === this.unbatch) {
       this.context = context
-      this.root = length === 0 ? EMPTY : new TreeNodePointer(context, 0, length - 1, 0, false, null)
+      this.root = length === 0 ? EMPTY : context.createTreeNode(0, length - 1, 0, false, null)
       this.unbatch = 0
     }
   }
@@ -278,12 +280,14 @@ class Hyperbee {
     this.unbatch = 0
   }
 
-  async get(key, { activeRequests = this.activeRequests } = {}) {
-    let ptr = await this.bootstrap(activeRequests)
+  async get(key, opts) {
+    const config = this.config.options(opts)
+
+    let ptr = await this.bootstrap(config)
     if (!ptr) return null
 
     while (true) {
-      const v = ptr.value ? this.bump(ptr) : await this.inflate(ptr, activeRequests)
+      const v = ptr.value ? this.bump(ptr) : await this.inflate(ptr, config)
 
       let s = 0
       let e = v.keys.length
@@ -295,7 +299,7 @@ class Hyperbee {
 
         c = b4a.compare(key, m.key)
 
-        if (c === 0) return this.finalizeKeyPointer(m, activeRequests)
+        if (c === 0) return this.finalizeKeyPointer(m, config)
 
         if (c < 0) e = mid
         else s = mid + 1
@@ -313,12 +317,12 @@ module.exports = Hyperbee
 
 function noop() {}
 
-function inflateKey(context, d, ptr, block, activeRequests) {
-  if (d.type === OP_COHORT) return inflateKeyCohort(context, d, ptr, block, activeRequests)
-  return inflateKeyDelta(context, d, ptr, block, activeRequests)
+function inflateKey(context, d, ptr, block, config) {
+  if (d.type === OP_COHORT) return inflateKeyCohort(context, d, ptr, block, config)
+  return inflateKeyDelta(context, d, ptr, block, config)
 }
 
-async function inflateKeyDelta(context, d, ptr, block, activeRequests) {
+async function inflateKeyDelta(context, d, ptr, block, config) {
   const k = d.pointer
 
   if (!k) return new DeltaOp(false, d.type, d.index, null)
@@ -326,7 +330,7 @@ async function inflateKeyDelta(context, d, ptr, block, activeRequests) {
   const blk =
     k.seq === ptr.seq && k.core === 0 && ptr.core === 0
       ? block
-      : await context.getBlock(k.seq, k.core, activeRequests)
+      : await context.getBlock(k.seq, k.core, config)
 
   const bk = blk.keys[k.offset]
 
@@ -334,7 +338,7 @@ async function inflateKeyDelta(context, d, ptr, block, activeRequests) {
 
   if (bk.valuePointer) {
     const p = bk.valuePointer
-    const ctx = await context.getContext(k.core, activeRequests)
+    const ctx = await context.getContext(k.core, config)
     vp = new ValuePointer(ctx, p.core, p.seq, p.offset, p.split)
   }
 
@@ -342,20 +346,20 @@ async function inflateKeyDelta(context, d, ptr, block, activeRequests) {
   return new DeltaOp(false, d.type, d.index, kp)
 }
 
-async function inflateKeyCohort(context, d, ptr, block, activeRequests) {
+async function inflateKeyCohort(context, d, ptr, block, config) {
   const co = d.pointer
 
   const blk =
     co.seq === ptr.seq && co.core === 0 && ptr.core === 0
       ? block
-      : await context.getBlock(co.seq, co.core, activeRequests)
+      : await context.getBlock(co.seq, co.core, config)
 
   const cohort = blk.cohorts[co.offset]
   const promises = new Array(cohort.length)
 
   for (let i = 0; i < cohort.length; i++) {
     const p = cohort[i]
-    const k = inflateKeyDelta(context, p, co, blk, activeRequests)
+    const k = inflateKeyDelta(context, p, co, blk, config)
     promises[i] = k
   }
 
@@ -363,31 +367,31 @@ async function inflateKeyCohort(context, d, ptr, block, activeRequests) {
   return new DeltaCohort(false, p, await Promise.all(promises))
 }
 
-function inflateChild(context, d, ptr, block, activeRequests) {
-  if (d.type === OP_COHORT) return inflateChildCohort(context, d, ptr, block, activeRequests)
-  return Promise.resolve(inflateChildDelta(context, d, ptr, block, activeRequests))
+function inflateChild(context, d, ptr, block, config) {
+  if (d.type === OP_COHORT) return inflateChildCohort(context, d, ptr, block, config)
+  return Promise.resolve(inflateChildDelta(context, d, ptr, block, config))
 }
 
-function inflateChildDelta(context, d, ptr, block, activeRequests) {
+function inflateChildDelta(context, d, ptr, block, config) {
   const p = d.pointer
-  const c = p && new TreeNodePointer(context, p.core, p.seq, p.offset, false, null)
+  const c = p && context.createTreeNode(p.core, p.seq, p.offset, false, null)
   return new DeltaOp(false, d.type, d.index, c)
 }
 
-async function inflateChildCohort(context, d, ptr, block, activeRequests) {
+async function inflateChildCohort(context, d, ptr, block, config) {
   const co = d.pointer
 
   const blk =
     co.seq === ptr.seq && co.core === 0 && ptr.core === 0
       ? block
-      : await context.getBlock(co.seq, co.core, activeRequests)
+      : await context.getBlock(co.seq, co.core, config)
 
   const cohort = blk.cohorts[co.offset]
   const deltas = new Array(cohort.length)
 
   for (let i = 0; i < cohort.length; i++) {
     const c = cohort[i]
-    deltas[i] = inflateChildDelta(context, c, co, blk, activeRequests)
+    deltas[i] = inflateChildDelta(context, c, co, blk, config)
   }
 
   const p = new Pointer(context, co.core, co.seq, co.offset)

package/lib/cache.js

@@ -3,6 +3,8 @@ module.exports = class NodeCache {
     this.size = 0
     this.maxSize = maxSize
     this.latest = null
+    this.retained = 0
+    this.byId = new Map()
   }
 
   oldest() {
@@ -20,7 +22,7 @@ module.exports = class NodeCache {
   gc() {
     while (this.size > this.maxSize) {
       const old = this.oldest()
-      if (old.changed) break
+      if (old.retained > this.retained) break
       this.remove(old)
       old.value = null
     }
@@ -29,6 +31,10 @@ module.exports = class NodeCache {
   bump(node) {
     if (node === this.latest) return
 
+    if (node.next === null && node.prev === null) {
+      this.byId.set(cacheId(node), node)
+    }
+
     if (node.prev) this.remove(node)
     this.size++
 
@@ -44,6 +50,10 @@ module.exports = class NodeCache {
     }
   }
 
+  get(node) {
+    return this.byId.get(cacheId(node)) || null
+  }
+
   remove(node) {
     if (node.prev) {
       this.size--
@@ -57,6 +67,11 @@ module.exports = class NodeCache {
     }
 
     node.prev = node.next = null
+
+    const id = cacheId(node)
+    if (this.byId.get(id) === node) {
+      this.byId.delete(id)
+    }
   }
 
   *[Symbol.iterator]() {
@@ -71,3 +86,8 @@ module.exports = class NodeCache {
     } while (node !== next)
   }
 }
+
+function cacheId(node) {
+  const id = node.core === 0 ? node.context.core.id : node.context.opened[node.core - 1].id
+  return id + '@' + node.seq + '.' + node.offset
+}

package/lib/changes.js

@@ -1,43 +1,41 @@
 const { Readable } = require('streamx')
 
 class ChangesStream extends Readable {
-  constructor(tree, options = {}) {
-    const { highWaterMark } = options
+  constructor(tree, opts = {}) {
+    const { highWaterMark, head = null } = opts
     super({ eagerOpen: true, highWaterMark })
 
-    const { head = null, activeRequests = tree.activeRequests } = options
-
     this.tree = tree
 
-    this._head = head
-    this._context = null
-    this._activeRequests = activeRequests
+    this.head = head
+    this.context = null
+    this.config = tree.config.options(opts)
   }
 
   async _openp() {
-    await this.tree.bootstrap(this._activeRequests)
-    if (this._head === null) this._head = this.tree.head()
-    if (this._head !== null) this._context = this.tree.context.getContextByKey(this._head.key)
+    await this.tree.bootstrap(this.config)
+    if (this.head === null) this.head = this.tree.head()
+    if (this.head !== null) this.context = this.tree.context.getContextByKey(this.head.key)
   }
 
   async _readp() {
-    if (!this._context || this._head.length === 0) {
+    if (!this.context || this.head.length === 0) {
       this.push(null)
       return
     }
 
     const data = {
-      head: this._head,
+      head: this.head,
       batch: []
     }
 
-    const seq = this._head.length - 1
-    const blk = await this._context.getBlock(seq, 0, this._activeRequests)
+    const seq = this.head.length - 1
+    const blk = await this.context.getBlock(seq, 0, this.config)
     const batchStart = seq - blk.batch.start
     const remaining = new Array(blk.batch.start)
 
     for (let i = 0; i < remaining.length; i++) {
-      remaining[i] = this._context.getBlock(batchStart + i, 0, this._activeRequests)
+      remaining[i] = this.context.getBlock(batchStart + i, 0, this.config)
     }
 
     for (const blk of await Promise.all(remaining)) data.batch.push(blk)
@@ -46,13 +44,13 @@ class ChangesStream extends Readable {
     this.push(data)
 
     if (!blk.previous) {
-      this._head = null
-      this._context = null
+      this.head = null
+      this.context = null
       return
     }
 
-    this._context = await this._context.getContext(blk.previous.core, this._activeRequests)
-    this._head = { key: this._context.core.key, length: blk.previous.seq + 1 }
+    this.context = await this.context.getContext(blk.previous.core, this.config)
+    this.head = { key: this.context.core.key, length: blk.previous.seq + 1 }
   }
 
   async _open(cb) {

package/lib/compression.js

@@ -23,6 +23,8 @@ class CompressedArray {
   constructor(delta) {
     this.entries = []
     this.delta = delta
+    this.uentries = null
+    this.updates = 0
 
     for (const d of delta) {
       if (d.type === OP_COHORT) {
@@ -39,8 +41,32 @@ class CompressedArray {
     return this.entries.length
   }
 
-  touch(index) {
-    const pointer = this.entries[index]
+  get ulength() {
+    return this.uentries ? this.uentries.length : this.entries.length
+  }
+
+  commit() {
+    const c = new CompressedArray([])
+
+    c.delta = this.delta
+    c.entries = this.uentries || this.entries.slice(0)
+
+    this.delta = this.delta.slice(0, this.updates)
+    this.uentries = null
+    this.updates = 0
+
+    return c
+  }
+
+  _update() {
+    if (this.uentries) return
+    this.uentries = this.entries.slice(0)
+  }
+
+  touch(index, pointer) {
+    if (pointer) this.entries[index] = pointer
+    else pointer = this.entries[index]
+
     if (pointer.changedBy === this) return
     this.set(index, pointer)
   }
@@ -49,8 +75,13 @@ class CompressedArray {
     return this.entries[index]
   }
 
+  uget(index) {
+    return this.uentries ? this.uentries[index] : this.entries[index]
+  }
+
   push(pointer) {
-    this.insert(this.entries.length, pointer)
+    if (!this.uentries) this._update()
+    this.insert(this.uentries.length, pointer)
   }
 
   unshift(pointer) {
@@ -58,37 +89,43 @@ class CompressedArray {
   }
 
   pop() {
-    if (this.entries.length === 0) return
-    const head = this.entries[this.entries.length - 1]
-    this.delete(this.entries.length - 1)
+    if (!this.uentries) this._update()
+    if (this.uentries.length === 0) return
+    const head = this.uentries[this.uentries.length - 1]
+    this.delete(this.uentries.length - 1)
     return head
   }
 
   shift() {
-    if (this.entries.length === 0) return
-    const tail = this.entries[0]
+    if (!this.uentries) this._update()
+    if (this.uentries.length === 0) return
+    const tail = this.uentries[0]
     this.delete(0)
     return tail
   }
 
   _touch(pointer) {
+    this.updates++
     if (pointer) pointer.changedBy = this
   }
 
   insert(index, pointer) {
-    if (!insert(this.entries, index, pointer)) return
+    if (!this.uentries) this._update()
+    if (!insert(this.uentries, index, pointer)) return
     this._touch(pointer)
     this.delta.push(new DeltaOp(true, OP_INSERT, index, pointer))
   }
 
   delete(index) {
-    if (!del(this.entries, index)) return
+    if (!this.uentries) this._update()
+    if (!del(this.uentries, index)) return
     this._touch(null)
     this.delta.push(new DeltaOp(true, OP_DEL, index, null))
   }
 
   set(index, pointer) {
-    if (!set(this.entries, index, pointer)) return
+    if (!this.uentries) this._update()
+    if (!set(this.uentries, index, pointer)) return
     this._touch(pointer)
     this.delta.push(new DeltaOp(true, OP_SET, index, pointer))
   }