hyperbee2 1.2.0 → 2.0.0

package/lib/context.js

@@ -3,10 +3,13 @@ const ScopeLock = require('scope-lock')
 const { decodeBlock } = require('./encoding.js')
 
 class CoreContext {
-  constructor(store, local, core, encryption, lock = new ScopeLock(), other = new Map()) {
+  constructor(store, local, core, encryption, t, lock = new ScopeLock(), other = new Map()) {
     this.store = store
     this.local = local
     this.core = core
+    this.t = t
+    this.minKeys = t - 1
+    this.maxKeys = 2 * t - 1
     this.encryption = encryption
     this.lock = lock
     this.other = other
@@ -17,6 +20,11 @@ class CoreContext {
     this.changed = false
   }
 
+  // TODO: remove, left here for easier debugging for now
+  // [Symbol.for('nodejs.util.inspect.custom')]() {
+  //   return '[CoreContext]'
+  // }
+
   async update(activeRequests) {
     await this.core.ready()
 
@@ -34,7 +42,7 @@ class CoreContext {
 
       if (length < this.length) return
 
-      this.cores = block.cores || []
+      this.cores = block.metadata ? block.metadata.cores : []
       while (this.opened.length < this.cores.length) this.opened.push(null)
     }
 
@@ -44,8 +52,9 @@ class CoreContext {
     this.length = length
   }
 
-  async getCoreOffset(context, core, activeRequests) {
-    if (core !== 0 && core - 1 >= context.cores.length) await context.update(activeRequests)
+  getCoreOffsetLocal(context, core) {
+    if (context === this) return core
+
     const key = core === 0 ? context.core.key : context.cores[core - 1].key
 
     if (b4a.equals(key, this.core.key)) return 0
@@ -65,6 +74,11 @@ class CoreContext {
     return this.cores.length
   }
 
+  async getCoreOffset(context, core, activeRequests) {
+    if (core !== 0 && core - 1 >= context.cores.length) await context.update(activeRequests)
+    return this.getCoreOffsetLocal(context, core)
+  }
+
   async getCoreOffsetByKey(key, activeRequests) {
     await this.core.ready()
 
@@ -112,7 +126,7 @@ class CoreContext {
   }
 
   async getBlock(seq, core, activeRequests) {
-    if (core !== 0) await this.update(activeRequests)
+    if (core !== 0 && core - 1 >= this.cores.length) await this.update(activeRequests)
     const hc = this.getCore(core)
     const buffer = await hc.get(seq, { activeRequests })
     const block = decodeBlock(buffer, seq)
@@ -129,8 +143,7 @@ class CoreContext {
     const hex = b4a.toString(key, 'hex')
     if (this.other.has(hex)) return this.other.get(hex)
 
-    const hc = this.store.get({ key, encryption: this.encryption })
-    const ctx = new CoreContext(this.store, this.local, hc, this.encryption, this.lock, this.other)
+    const ctx = this._createContext(this.store.get({ key, encryption: this.encryption }))
     this.other.set(hex, ctx)
     return ctx
   }
@@ -143,11 +156,27 @@ class CoreContext {
     const hex = b4a.toString(this.cores[core - 1].key, 'hex')
     if (this.other.has(hex)) return this.other.get(hex)
 
-    const hc = this.getCore(core)
-    const ctx = new CoreContext(this.store, this.local, hc, this.encryption, this.lock, this.other)
+    const ctx = this._createContext(this.getCore(core))
     this.other.set(hex, ctx)
     return ctx
   }
+
+  flush() {
+    this.changed = false
+    return {
+      cores: this.cores
+    }
+  }
+
+  _createContext(core) {
+    const store = this.store
+    const local = this.local
+    const encryption = this.encryption
+    const t = this.t
+    const lock = this.lock
+    const other = this.other
+    return new CoreContext(store, local, core, encryption, t, lock, other)
+  }
 }
 
 module.exports = CoreContext

package/lib/diff.js

@@ -3,11 +3,15 @@ const b4a = require('b4a')
 const { RangeIterator } = require('./ranges.js')
 
 class DiffIterator {
-  constructor(left, right, { limit = -1 } = {}) {
+  constructor(tree, left, right, { limit = -1, activeRequests = [] } = {}) {
+    this.tree = tree
+
     this.left = left
     this.right = right
 
     this.left.limit = this.right.limit = -1
+
+    this.activeRequests = activeRequests
     this.limit = limit
 
     this.nextLeft = null
@@ -18,15 +22,15 @@ class DiffIterator {
     await Promise.all([this.left.open(), this.right.open()])
   }
 
-  _shiftRight() {
-    const right = this.nextRight
+  async _shiftRight() {
+    const right = await this.tree.finalizeKeyPointer(this.nextRight, this.activeRequests)
     this.nextRight = null
     if (this.limit > 0) this.limit--
     return { left: null, right }
   }
 
-  _shiftLeft() {
-    const left = this.nextLeft
+  async _shiftLeft() {
+    const left = await this.tree.finalizeKeyPointer(this.nextLeft, this.activeRequests)
     this.nextLeft = null
     if (this.limit > 0) this.limit--
     return { left, right: null }
@@ -47,8 +51,8 @@ class DiffIterator {
 
   async next() {
     while (this.limit === -1 || this.limit > 0) {
-      const leftPromise = this.nextLeft ? null : this.left.next()
-      const rightPromise = this.nextRight ? null : this.right.next()
+      const leftPromise = this.nextLeft ? null : this.left.nextKey()
+      const rightPromise = this.nextRight ? null : this.right.nextKey()
 
       const [l, r] = await Promise.all([leftPromise, rightPromise])
 
@@ -62,16 +66,21 @@ class DiffIterator {
       const cmp = b4a.compare(this.nextLeft.key, this.nextRight.key)
 
       if (cmp === 0) {
-        const left = this.nextLeft
-        const right = this.nextRight
+        const leftKey = this.nextLeft
+        const rightKey = this.nextRight
 
         this.nextRight = this.nextLeft = null
 
-        if (isSame(left, right)) {
+        if (isSame(leftKey, rightKey)) {
           this._fastForward()
           continue
         }
 
+        const [left, right] = await Promise.all([
+          this.tree.finalizeKeyPointer(leftKey, this.activeRequests),
+          this.tree.finalizeKeyPointer(rightKey, this.activeRequests)
+        ])
+
         if (this.limit > 0) this.limit--
         return { left, right }
       }
@@ -92,6 +101,7 @@ class DiffStream extends Readable {
     this.left = left
     this.right = right
     this.iterator = new DiffIterator(
+      left,
       new RangeIterator(left, options),
       new RangeIterator(right, options),
       options

package/lib/encoding.js

@@ -1,19 +1,144 @@
 const c = require('compact-encoding')
 const { getEncoding } = require('../spec/hyperschema')
+const { OP_COHORT, OP_INSERT } = require('./compression.js')
 const compat = require('./compat.js')
 
-const Block = getEncoding('@bee/block')
+const Block0 = getEncoding('@bee/block-0')
+const Block1 = getEncoding('@bee/block-1')
 
-exports.Block = Block
+const TYPE_COMPAT = 10 // compat with bee1, all blocks starts with 0x10
+
+exports.TYPE_COMPAT = TYPE_COMPAT
+exports.TYPE_LATEST = 1
+
+exports.Block = Block1
 exports.encodeBlock = encodeBlock
 exports.decodeBlock = decodeBlock
 
-function encodeBlock(block, format) {
-  return format === 2 ? compat.encode(block) : c.encode(Block, block)
+function encodeBlock(block) {
+  if (block.type === 1) {
+    return c.encode(Block1, block)
+  }
+
+  if (block.type === 0) {
+    return encodeBlock0(block)
+  }
+
+  if (block.type === TYPE_COMPAT) {
+    return compat.encode(block)
+  }
+
+  throw new Error('Unknown block type: ' + block.type)
 }
 
 function decodeBlock(buffer, seq) {
-  const isCompat = buffer.length > 0 && buffer[0] === 0x0a
-  const block = isCompat ? compat.decode(buffer, seq) : c.decode(Block, buffer)
-  return block
+  const state = { start: 0, end: buffer.byteLength, buffer }
+  const type = state.end > 0 ? c.uint.decode(state) : 0
+
+  state.start = 0
+
+  if (type === 1) {
+    return Block1.decode(state)
+  }
+
+  if (type === 0) {
+    return decodeBlock0(state, seq)
+  }
+
+  if (type === TYPE_COMPAT) {
+    return compat.decode(buffer, seq)
+  }
+
+  throw new Error('Unknown block type: ' + type)
+}
+
+function dataToKey(d) {
+  return { ...d, valuePointer: null }
+}
+
+function keyToData(d) {
+  return { key: d.key, value: d.value }
+}
+
+function decodeBlock0(state, seq) {
+  const cohorts = []
+  const blk = Block0.decode(state)
+  const tree = []
+
+  for (const t of blk.tree) {
+    const next = {
+      keys: toCohort(seq, t.keys, cohorts),
+      children: toCohort(seq, t.children, cohorts)
+    }
+
+    tree.push(next)
+  }
+
+  return {
+    type: 0,
+    checkpoint: blk.checkpoint,
+    batch: blk.batch,
+    previous: blk.previous,
+    metadata: { cores: blk.cores },
+    tree,
+    keys: blk.data.map(dataToKey),
+    values: null,
+    cohorts
+  }
+}
+
+function encodeBlock0(blk) {
+  const tree = []
+
+  for (const t of blk.tree) {
+    tree.push({
+      keys: fromCohort(t.keys, blk.cohorts),
+      children: fromCohort(t.children, blk.cohorts)
+    })
+  }
+
+  return c.encode(Block0, {
+    type: 0,
+    checkpoint: blk.checkpoint,
+    batch: blk.batch,
+    previous: blk.previous,
+    tree,
+    data: blk.keys.map(keyToData),
+    cores: blk.metadata ? blk.metadata.cores : null
+  })
+}
+
+function fromCohort(deltas, cohorts) {
+  if (deltas.length === 0) return []
+  const cohort = cohorts[deltas[0].pointer.offset]
+  const pointers = []
+  for (const d of cohort) pointers.push(d.pointer)
+  return pointers
+}
+
+function toCohort(seq, pointers, cohorts) {
+  if (!pointers.length) return []
+
+  const cohort = []
+  const offset = cohorts.push(cohort) - 1
+
+  for (let i = 0; i < pointers.length; i++) {
+    cohort.push({
+      type: OP_INSERT,
+      index: i,
+      pointer: pointers[i]
+    })
+  }
+
+  return [
+    {
+      type: OP_COHORT,
+      index: 0,
+      pointer: {
+        core: 0,
+        seq,
+        offset
+      }
+    }
+  ]
 }

package/lib/ranges.js

@@ -37,8 +37,8 @@ class RangeIterator {
         const j = this.reverse ? v.keys.length - 1 - i : i
 
         const c = this.reverse
-          ? b4a.compare(v.keys[j].key, this.end)
-          : b4a.compare(this.start, v.keys[j].key)
+          ? b4a.compare(v.keys.get(j).key, this.end)
+          : b4a.compare(this.start, v.keys.get(j).key)
 
         if (c < 0) break
         top.offset = 2 * i + 1 + (c === 0 ? offset : 1)
@@ -52,7 +52,7 @@ class RangeIterator {
       const j = this.reverse ? v.children.length - 1 - k : k
 
       this.stack.push({
-        node: v.children[j],
+        node: v.children.get(j),
         offset: 0
       })
 
@@ -61,6 +61,11 @@ class RangeIterator {
   }
 
   async next() {
+    const key = await this.nextKey()
+    return key === null ? key : this.tree.finalizeKeyPointer(key, this.activeRequests)
+  }
+
+  async nextKey() {
     while (this.stack.length && (this.limit === -1 || this.limit > 0)) {
       const top = this.stack.pop()
       const v = top.node.value
@@ -75,7 +80,7 @@ class RangeIterator {
         this.stack.push(top)
         if (k < v.children.length) {
           const j = this.reverse ? v.children.length - 1 - k : k
-          this.stack.push({ node: v.children[j], offset: 0 })
+          this.stack.push({ node: v.children.get(j), offset: 0 })
         }
         continue
       }
@@ -83,7 +88,7 @@ class RangeIterator {
       if (k < v.keys.length) {
         const j = this.reverse ? v.keys.length - 1 - k : k
 
-        const data = top.node.value.keys[j]
+        const data = top.node.value.keys.get(j)
 
         const c = this.reverse
           ? this.start

package/lib/tree.js

@@ -1,29 +1,22 @@
 const b4a = require('b4a')
+const { CompressedArray } = require('./compression.js')
 
-const T = 5
-const MIN_KEYS = T - 1
-const MAX_CHILDREN = MIN_KEYS * 2 + 1
+const INSERTED = -1
+const NEEDS_SPLIT = -2
 
-const UNCHANGED = 0
-const CHANGED = 1
-const NEEDS_SPLIT = 2
-
-class DataPointer {
-  constructor(context, core, seq, offset, changed, key, value) {
+class ValuePointer {
+  constructor(context, core, seq, offset, split) {
     this.context = context
 
     this.core = core
     this.seq = seq
     this.offset = offset
-    this.changed = changed
-
-    this.key = key
-    this.value = value
+    this.split = split
   }
 }
 
-class TreeNodePointer {
-  constructor(context, core, seq, offset, changed, value) {
+class Pointer {
+  constructor(context, core, seq, offset, changed) {
     this.context = context
 
     this.core = core
@@ -31,68 +24,121 @@ class TreeNodePointer {
     this.offset = offset
     this.changed = changed
 
+    this.changedBy = null
+  }
+}
+
+class KeyPointer extends Pointer {
+  constructor(context, core, seq, offset, changed, key, value, valuePointer) {
+    super(context, core, seq, offset, changed)
+
+    this.key = key
+    this.value = value
+    this.valuePointer = valuePointer
+  }
+
+  // TODO: remove, left here for easier debugging for now
+  // [Symbol.for('nodejs.util.inspect.custom')]() {
+  //   return (
+  //     `[KeyPointer core=${this.core} seq=${this.seq}, offset=${this.offset}, key="${b4a.toString(this.key)}"]`
+  //   )
+  // }
+}
+
+class TreeNodePointer extends Pointer {
+  constructor(context, core, seq, offset, changed, value) {
+    super(context, core, seq, offset, changed)
+
     this.value = value
 
     this.next = null
     this.prev = null
   }
+
+  // TODO: remove, left here for easier debugging for now
+  // [Symbol.for('nodejs.util.inspect.custom')]() {
+  //   return (
+  //     `[TreeNodePointer core=${this.core} seq=${this.seq} offset=${this.offset} changed=${this.changed}]`
+  //   )
+  // }
 }
 
 class TreeNode {
   constructor(keys, children) {
-    this.keys = keys
-    this.children = children
+    this.keys = new CompressedArray(keys)
+    this.children = new CompressedArray(children)
   }
 
   isEmpty() {
     return this.keys.length === 0 && this.children.length === 0
   }
 
-  put(context, key, value, child) {
+  insertLeaf(context, key, value) {
     let s = 0
     let e = this.keys.length
-    let c
+    let c = 0
 
     while (s < e) {
       const mid = (s + e) >> 1
-      const k = this.keys[mid]
+      const k = this.keys.get(mid)
 
       c = b4a.compare(key, k.key)
 
-      if (c === 0) {
-        if (b4a.equals(k.value, value)) return UNCHANGED
-        this.keys[mid] = new DataPointer(context, 0, 0, 0, true, key, value)
-        return CHANGED
-      }
+      if (c === 0) return mid
 
       if (c < 0) e = mid
       else s = mid + 1
     }
 
     const i = c < 0 ? e : s
-    this.keys.splice(i, 0, new DataPointer(context, 0, 0, 0, true, key, value))
-    if (child) this.children.splice(i + 1, 0, child)
+    this.keys.insert(i, new KeyPointer(context, 0, 0, 0, true, key, value, null))
 
-    return this.keys.length < MAX_CHILDREN ? CHANGED : NEEDS_SPLIT
+    return this.keys.length <= context.maxKeys ? INSERTED : NEEDS_SPLIT
+  }
+
+  insertNode(context, keyPointer, treePointer) {
+    let s = 0
+    let e = this.keys.length
+    let c = 0
+
+    while (s < e) {
+      const mid = (s + e) >> 1
+      const k = this.keys.get(mid)
+
+      c = b4a.compare(keyPointer.key, k.key)
+
+      if (c === 0) return mid
+
+      if (c < 0) e = mid
+      else s = mid + 1
+    }
+
+    const i = c < 0 ? e : s
+    this.keys.insert(i, keyPointer)
+    this.children.insert(i + 1, treePointer)
+
+    return this.keys.length <= context.maxKeys ? INSERTED : NEEDS_SPLIT
   }
 
   setValue(context, i, value) {
-    this.keys[i] = new DataPointer(context, 0, 0, 0, true, this.keys[i].key, value)
+    this.keys.set(i, new KeyPointer(context, 0, 0, 0, true, this.keys.get(i).key, value, null))
   }
 
   removeKey(i) {
-    this.keys.splice(i, 1)
+    this.keys.delete(i)
     if (this.children.length) {
-      this.children.splice(i + 1, 1)
+      this.children.delete(i + 1)
     }
   }
 
   siblings(parent) {
-    for (let i = 0; i < parent.children.length; i++) {
-      if (parent.children[i].value !== this) continue // TODO: move to a seq/offset check instead
+    const pc = parent.children
 
-      const left = i ? parent.children[i - 1] : null
-      const right = i < parent.children.length - 1 ? parent.children[i + 1] : null
+    for (let i = 0; i < pc.length; i++) {
+      if (pc.get(i).value !== this) continue // TODO: move to a seq/offset check instead
+
+      const left = i ? pc.get(i - 1) : null
+      const right = i < pc.length - 1 ? pc.get(i + 1) : null
       return { left, index: i, right }
     }
 
@@ -101,23 +147,29 @@ class TreeNode {
   }
 
   merge(node, median) {
+    const keys = node.keys
+    const children = node.children
+
     this.keys.push(median)
-    for (let i = 0; i < node.keys.length; i++) this.keys.push(node.keys[i])
-    for (let i = 0; i < node.children.length; i++) this.children.push(node.children[i])
+
+    for (let i = 0; i < keys.length; i++) this.keys.push(keys.get(i))
+    for (let i = 0; i < children.length; i++) this.children.push(children.get(i))
   }
 
   split(context) {
     const len = this.keys.length >> 1
     const right = new TreeNodePointer(context, 0, 0, 0, true, new TreeNode([], []))
 
-    while (right.value.keys.length < len) right.value.keys.push(this.keys.pop())
-    right.value.keys.reverse()
+    const k = []
+    while (k.length < len) k.push(this.keys.pop())
+    for (let i = k.length - 1; i >= 0; i--) right.value.keys.push(k[i])
 
     const median = this.keys.pop()
 
     if (this.children.length) {
-      while (right.value.children.length < len + 1) right.value.children.push(this.children.pop())
-      right.value.children.reverse()
+      const c = []
+      while (c.length < len + 1) c.push(this.children.pop())
+      for (let i = c.length - 1; i >= 0; i--) right.value.children.push(c[i])
     }
 
     return {
@@ -128,16 +180,13 @@ class TreeNode {
   }
 }
 
-exports.T = T
-exports.MIN_KEYS = MIN_KEYS
-exports.MAX_CHILDREN = MAX_CHILDREN
-
-exports.UNCHANGED = UNCHANGED
-exports.CHANGED = CHANGED
+exports.INSERTED = INSERTED
 exports.NEEDS_SPLIT = NEEDS_SPLIT
 
 exports.TreeNodePointer = TreeNodePointer
 exports.TreeNode = TreeNode
-exports.DataPointer = DataPointer
+exports.KeyPointer = KeyPointer
+exports.ValuePointer = ValuePointer
+exports.Pointer = Pointer
 
 exports.EMPTY = new TreeNodePointer(null, 0, 0, 0, false, null)