hyperbee2 1.2.0 → 2.0.0

package/lib/write.js

@@ -1,24 +1,46 @@
 const b4a = require('b4a')
 const c = require('compact-encoding')
-const { encodeBlock } = require('./encoding.js')
+const { OP_COHORT } = require('./compression.js')
+const { encodeBlock, TYPE_COMPAT, TYPE_LATEST } = require('./encoding.js')
 const {
-  DataPointer,
+  Pointer,
+  KeyPointer,
+  ValuePointer,
   TreeNode,
   TreeNodePointer,
   MIN_KEYS,
-  UNCHANGED,
-  CHANGED,
+  INSERTED,
   NEEDS_SPLIT
 } = require('./tree.js')
 
+const PREFERRED_BLOCK_SIZE = 4096
+const INLINE_VALUE_SIZE = 4096
+const ESTIMATED_POINTER_SIZE = 12 // conversative, seq=8b, offset=2b, core=2b
+
 module.exports = class WriteBatch {
-  constructor(tree, { length = -1, key = null, autoUpdate = true, blockFormat = 3 } = {}) {
+  constructor(tree, opts = {}) {
+    const {
+      length = -1,
+      key = null,
+      autoUpdate = true,
+      compat = false,
+      type = compat ? TYPE_COMPAT : TYPE_LATEST,
+      deltaMax = supportsCompression(type) ? 16 : 0,
+      deltaMin = supportsCompression(type) ? 1 : Infinity,
+      inlineValueSize = INLINE_VALUE_SIZE,
+      preferredBlockSize = PREFERRED_BLOCK_SIZE
+    } = opts
+
     this.tree = tree
+    this.deltaMax = deltaMax
+    this.deltaMin = deltaMin
+    this.inlineValueSize = inlineValueSize
+    this.preferredBlockSize = preferredBlockSize
     this.snapshot = tree.snapshot()
     this.autoUpdate = autoUpdate
     this.length = length
     this.key = key
-    this.blockFormat = blockFormat
+    this.type = type
     this.closed = false
     this.applied = 0
     this.root = null
@@ -62,16 +84,10 @@ module.exports = class WriteBatch {
 
       const changed = length === 0
       const seq = length === 0 ? 0 : length - 1
+      const value = changed ? new TreeNode([], []) : null
 
       this.length = length
-      this.root = new TreeNodePointer(
-        context,
-        0,
-        seq,
-        0,
-        changed,
-        changed ? new TreeNode([], []) : null
-      )
+      this.root = new TreeNodePointer(context, 0, seq, 0, changed, value)
 
       for (const op of ops) {
         if (op.put) op.applied = await this._put(op.key, op.value)
@@ -113,12 +129,13 @@ module.exports = class WriteBatch {
 
       while (s < e) {
         const mid = (s + e) >> 1
-        const m = v.keys[mid]
+        const m = v.keys.get(mid)
 
         c = b4a.compare(target, m.key)
 
         if (c === 0) {
-          if (b4a.equals(m.value, value)) return false
+          const existing = await this.snapshot.inflateValue(m)
+          if (b4a.equals(existing, value)) return false
           v.setValue(this.tree.context, mid, value)
           for (let i = 0; i < stack.length; i++) stack[i].changed = true
           return true
@@ -129,16 +146,21 @@ module.exports = class WriteBatch {
       }
 
       const i = c < 0 ? e : s
-      ptr = v.children[i]
+      ptr = v.children.get(i)
     }
 
     const v = ptr.value ? this.snapshot.bump(ptr) : await this.snapshot.inflate(ptr)
-    let status = v.put(this.tree.context, target, value, null)
-
-    if (status === UNCHANGED) return false
+    let status = v.insertLeaf(this.tree.context, target, value)
+
+    if (status >= 0) {
+      // already exists, upsert if changed
+      const m = v.keys.get(status)
+      const existing = await this.snapshot.inflateValue(m)
+      if (b4a.equals(existing, value)) return false
+      v.setValue(this.tree.context, status, value)
+    }
 
     ptr.changed = true
-
     for (let i = 0; i < stack.length; i++) stack[i].changed = true
 
     while (status === NEEDS_SPLIT) {
@@ -148,14 +170,15 @@ module.exports = class WriteBatch {
 
       if (parent) {
         const p = parent.value ? this.snapshot.bump(parent) : await this.snapshot.inflate(parent)
-        status = p.put(this.tree.context, median.key, median.value, right)
+        status = p.insertNode(this.tree.context, median, right)
         ptr = parent
       } else {
         this.root = new TreeNodePointer(this.tree.context, 0, 0, 0, true, new TreeNode([], []))
         this.root.value.keys.push(median)
-        this.root.value.children.push(ptr, right)
+        this.root.value.children.push(ptr)
+        this.root.value.children.push(right)
         this.snapshot.bump(this.root)
-        status = UNCHANGED
+        status = INSERTED
       }
     }
 
@@ -177,12 +200,11 @@ module.exports = class WriteBatch {
 
       while (s < e) {
         const mid = (s + e) >> 1
-        c = b4a.compare(key, v.keys[mid].key)
+        c = b4a.compare(key, v.keys.get(mid).key)
 
         if (c === 0) {
           if (v.children.length) await this._setKeyToNearestLeaf(v, mid, stack)
           else v.removeKey(mid)
-
           // we mark these as changed late, so we don't rewrite them if it is a 404
           for (let i = 0; i < stack.length; i++) stack[i].changed = true
           this.root = await this._rebalance(stack)
@@ -196,15 +218,15 @@ module.exports = class WriteBatch {
       if (!v.children.length) return false
 
       const i = c < 0 ? e : s
-      ptr = v.children[i]
+      ptr = v.children.get(i)
     }
 
     return false
   }
 
   async _setKeyToNearestLeaf(v, index, stack) {
-    let left = v.children[index]
-    let right = v.children[index + 1]
+    let left = v.children.get(index)
+    let right = v.children.get(index + 1)
 
     const [ls, rs] = await Promise.all([this._leafSize(left, false), this._leafSize(right, true)])
 
@@ -213,28 +235,28 @@ module.exports = class WriteBatch {
       stack.push(right)
       let r = right.value ? this.snapshot.bump(right) : await this.snapshot.inflate(right)
       while (r.children.length) {
-        right = r.children[0]
+        right = r.children.get(0)
         stack.push(right)
         r = right.value ? this.snapshot.bump(right) : await this.snapshot.inflate(right)
       }
-      v.keys[index] = r.keys.shift()
+      v.keys.set(index, r.keys.shift())
     } else {
       // if fewer leaves on the right
       stack.push(left)
       let l = left.value ? this.snapshot.bump(left) : await this.snapshot.inflate(left)
       while (l.children.length) {
-        left = l.children[l.children.length - 1]
+        left = l.children.get(l.children.length - 1)
         stack.push(left)
         l = left.value ? this.snapshot.bump(left) : await this.snapshot.inflate(left)
       }
-      v.keys[index] = l.keys.pop()
+      v.keys.set(index, l.keys.pop())
     }
   }
 
   async _leafSize(ptr, goLeft) {
     let v = ptr.value ? this.snapshot.bump(ptr) : await this.snapshot.inflate(ptr)
     while (v.children.length) {
-      ptr = v.children[goLeft ? 0 : v.children.length - 1]
+      ptr = v.children.get(goLeft ? 0 : v.children.length - 1)
       v = ptr.value ? this.snapshot.bump(ptr) : await this.snapshot.inflate(ptr)
     }
     return v.keys.length
@@ -242,6 +264,7 @@ module.exports = class WriteBatch {
 
   async _rebalance(stack) {
     const root = stack[0]
+    const minKeys = this.tree.context.minKeys
 
     while (stack.length > 1) {
       const ptr = stack.pop()
@@ -258,11 +281,11 @@ module.exports = class WriteBatch {
       let l = left && (left.value ? this.snapshot.bump(left) : await this.snapshot.inflate(left))
 
       // maybe borrow from left sibling?
-      if (l && l.keys.length > MIN_KEYS) {
+      if (l && l.keys.length > minKeys) {
         left.changed = true
-        v.keys.unshift(p.keys[index - 1])
+        v.keys.unshift(p.keys.get(index - 1))
         if (l.children.length) v.children.unshift(l.children.pop())
-        p.keys[index - 1] = l.keys.pop()
+        p.keys.set(index - 1, l.keys.pop())
         return root
       }
 
@@ -270,11 +293,11 @@ module.exports = class WriteBatch {
         right && (right.value ? this.snapshot.bump(right) : await this.snapshot.inflate(right))
 
       // maybe borrow from right sibling?
-      if (r && r.keys.length > MIN_KEYS) {
+      if (r && r.keys.length > minKeys) {
         right.changed = true
-        v.keys.push(p.keys[index])
+        v.keys.push(p.keys.get(index))
         if (r.children.length) v.children.push(r.children.shift())
-        p.keys[index] = r.keys.shift()
+        p.keys.set(index, r.keys.shift())
         return root
       }
 
@@ -289,7 +312,7 @@ module.exports = class WriteBatch {
       }
 
       left.changed = true
-      l.merge(r, p.keys[index])
+      l.merge(r, p.keys.get(index))
 
       parent.changed = true
       p.removeKey(index)
@@ -297,69 +320,100 @@ module.exports = class WriteBatch {
 
     const r = root.value ? this.snapshot.bump(root) : await this.snapshot.inflate(root)
     // check if the tree shrunk
-    if (!r.keys.length && r.children.length) return r.children[0]
+    if (!r.keys.length && r.children.length) return r.children.get(0)
     return root
   }
 
+  _shouldInlineValue(k) {
+    if (!k.value || supportsCompression(this.type)) return true
+    if (k.valuePointer) return true
+    return k.value.byteLength <= this.inlineValueSize
+  }
+
   async _flush() {
-    if (!this.root || !this.root.changed) return
+    if (!this.root || !this.root.changed) {
+      return
+    }
+
+    if (!this.root.value) await this.snapshot.inflate(this.root)
+
+    let update = { size: 0, nodes: [], keys: [], values: [] }
+    let minValue = -1
 
-    const update = { node: [], keys: [] }
     const batch = [update]
-    const stack = [{ update, node: this.root }]
+    const stack = [this.root]
+    const values = []
+
     const context = this.tree.context.getLocalContext()
+    const activeRequests = this.tree.activeRequests
 
-    await context.update(this.tree.activeRequests)
+    await context.update(activeRequests)
 
     while (stack.length > 0) {
-      const { update, node } = stack.pop()
+      const node = stack.pop()
 
-      node.changed = false
-      update.node.push(node)
-
-      for (let i = 0; i < node.value.keys.length; i++) {
-        const k = node.value.keys[i]
+      if (this.type !== TYPE_COMPAT && update.size >= this.preferredBlockSize) {
+        update = { size: 0, nodes: [], keys: [], values: [] }
+        batch.push(update)
+      }
 
-        if (!k.changed) {
-          k.core = await context.getCoreOffset(k.context, k.core, this.tree.activeRequests)
-          k.context = context
-          continue
-        }
+      update.nodes.push(node)
+      update.size += getEstimatedNodeSize(node.value)
 
-        k.changed = false
-        update.keys.push(k)
-      }
+      const keys = node.value.keys
+      const children = node.value.children
 
-      let first = true
+      for (let i = 0; i < keys.entries.length; i++) {
+        const k = keys.entries[i]
+        if (!k.changed) continue
 
-      for (let i = 0; i < node.value.children.length; i++) {
-        const n = node.value.children[i]
+        if (!this._shouldInlineValue(k)) {
+          values.push(k)
+          k.valuePointer = new ValuePointer(context, 0, 0, 0, 0)
 
-        if (!n.changed) {
-          n.core = await context.getCoreOffset(n.context, n.core, this.tree.activeRequests)
-          n.context = context
-          continue
+          if (minValue === -1 || minValue < k.value.byteLength) {
+            minValue = k.value.byteLength
+          }
         }
 
-        if (first || this.blockFormat === 2) {
-          stack.push({ update, node: n })
-          first = false
-        } else {
-          const update = { node: [], keys: [] }
-          batch.push(update)
-          stack.push({ update, node: n })
-        }
+        update.keys.push(k)
+        update.size += getEstimatedKeySize(k)
+      }
+
+      for (let i = 0; i < children.entries.length; i++) {
+        const c = children.entries[i]
+        if (!c.value || !c.changed) continue
+        children.touch(i)
+        stack.push(c)
       }
     }
 
+    if (this.type === TYPE_COMPAT) await toCompatType(context, batch, this.ops)
+
     const length = context.core.length
 
     // if noop and not genesis, bail early
-    if (this.applied === 0 && length > 0) {
-      return
+    if (this.applied === 0 && length > 0) return
+
+    if (minValue > -1 && minValue + update.size < this.preferredBlockSize) {
+      // TODO: repack the value into the block
     }
 
-    if (this.blockFormat === 2) toBlockFormat2(context, batch, this.ops)
+    if (values.length) {
+      update = { size: 0, nodes: [], keys: [], values: [] }
+
+      for (let i = 0; i < values.length; i++) {
+        const k = values[i]
+
+        update.size += getEstimatedValueSize(k)
+        update.values.push(k)
+
+        if (i === values.length - 1 || update.size >= this.preferredBlockSize) {
+          batch.push(update)
+          update = { size: 0, nodes: [], keys: [], values: [] }
+        }
+      }
+    }
 
     const blocks = new Array(batch.length)
 
@@ -368,33 +422,63 @@ module.exports = class WriteBatch {
       const seq = length + batch.length - i - 1
 
       const block = {
-        type: 0,
+        type: this.type,
         checkpoint: 0,
         batch: { start: batch.length - 1 - i, end: i },
         previous: null,
+        metadata: null,
         tree: null,
-        data: null,
-        cores: null
+        keys: null,
+        values: null,
+        cohorts: null
+      }
+
+      for (const k of update.values) {
+        if (block.values === null) block.values = []
+        const ptr = k.valuePointer
+
+        ptr.core = 0
+        ptr.context = context
+        ptr.seq = seq
+        ptr.offset = block.values.length
+        ptr.split = 0
+
+        block.values.push(k.value)
+        k.value = null // unlinked
       }
 
       for (const k of update.keys) {
-        if (block.data === null) block.data = []
+        if (block.keys === null) block.keys = []
 
         k.core = 0
         k.context = context
         k.seq = seq
-        k.offset = block.data.length
-        block.data.push(k)
+        k.offset = block.keys.length
+        k.changed = false
+
+        if (k.valuePointer) updateValuePointerContext(k.valuePointer, context)
+
+        block.keys.push(k)
       }
 
-      for (const n of update.node) {
+      for (const n of update.nodes) {
         if (block.tree === null) block.tree = []
 
         n.core = 0
         n.context = context
         n.seq = seq
         n.offset = block.tree.length
-        block.tree.push(n.value)
+        n.changed = false
+
+        const treeDelta = {
+          keys: n.value.keys.flush(this.deltaMax, this.deltaMin),
+          children: n.value.children.flush(this.deltaMax, this.deltaMin)
+        }
+
+        prepareCohorts(context, block, seq, treeDelta.keys, supportsCompression)
+        prepareCohorts(context, block, seq, treeDelta.children, false)
+
+        block.tree.push(treeDelta)
       }
 
       blocks[seq - length] = block
@@ -403,22 +487,19 @@ module.exports = class WriteBatch {
     const buffers = new Array(blocks.length)
 
     if (blocks.length > 0 && this.length > 0) {
-      const core = this.key
-        ? await context.getCoreOffsetByKey(this.key, this.tree.activeRequests)
-        : 0
+      const core = this.key ? await context.getCoreOffsetByKey(this.key, activeRequests) : 0
       blocks[blocks.length - 1].previous = { core, seq: this.length - 1 }
     }
 
     // TODO: make this transaction safe
     if (context.changed) {
-      context.changed = false
       context.checkpoint = context.core.length + blocks.length
-      blocks[blocks.length - 1].cores = context.cores
+      blocks[blocks.length - 1].metadata = context.flush()
     }
 
     for (let i = 0; i < blocks.length; i++) {
       blocks[i].checkpoint = context.checkpoint
-      buffers[i] = encodeBlock(blocks[i], this.blockFormat)
+      buffers[i] = encodeBlock(blocks[i])
     }
 
     if (this.closed) {
@@ -430,14 +511,15 @@ module.exports = class WriteBatch {
     for (let i = 0; i < batch.length; i++) {
       const update = batch[i]
 
-      for (let j = 0; j < update.node.length; j++) {
-        this.snapshot.bump(update.node)
+      for (let j = 0; j < update.nodes.length; j++) {
+        const node = update.nodes[j]
+        this.snapshot.bump(node)
       }
     }
   }
 }
 
-function toBlockFormat2(context, batch, ops) {
+async function toCompatType(context, batch, ops) {
   const map = new Map()
   let index = 0
 
@@ -450,10 +532,69 @@ function toBlockFormat2(context, batch, ops) {
     if (!op.put && !op.applied) continue
 
     const k = map.get(b4a.toString(op.key, 'hex'))
+
     const j = index++
-    if (j === batch.length) batch.push({ node: [], keys: [] })
-    batch[j].keys = [k || new DataPointer(context, 0, 0, 0, false, op.key, op.value)]
+    if (j === batch.length) batch.push({ size: 0, nodes: [], keys: [], values: [] })
+    batch[j].keys = [k || new KeyPointer(context, 0, 0, 0, false, op.key, op.value, null)]
   }
 
-  return batch
+  // compat doesnt support block 0
+  if (context.core.length > 0) return
+
+  const header = b4a.from('0a086879706572626565', 'hex')
+  await context.core.append(header)
+}
+
+function updateValuePointerContext(valuePointer, context) {
+  valuePointer.core = context.getCoreOffsetLocal(valuePointer.context, valuePointer.core)
+  valuePointer.context = context
+}
+
+// TODO: this isnt right anymore as we compress the delta post flush, but prob ok...
+function getEstimatedNodeSize(n) {
+  return (
+    n.keys.delta.length * ESTIMATED_POINTER_SIZE + n.children.delta.length * ESTIMATED_POINTER_SIZE
+  )
+}
+
+function getEstimatedKeySize(k) {
+  return k.key.byteLength + (k.valuePointer ? ESTIMATED_POINTER_SIZE : 0)
+}
+
+function getEstimatedValueSize(k) {
+  return k.value.byteLength
+}
+
+function prepareCohorts(context, block, seq, deltas, keys) {
+  for (const d of deltas) {
+    // same below but the delta might be a noop (ie add and the delete) - handle that
+    if (keys && d.changed && d.pointer && d.pointer.changed) d.pointer = null
+
+    if (d.pointer && d.pointer.context !== context) {
+      const p = d.pointer
+      p.core = context.getCoreOffsetLocal(p.context, p.core)
+      p.context = context
+    }
+
+    if (d.type !== OP_COHORT || !d.changed) continue
+    if (block.cohorts === null) block.cohorts = []
+
+    d.changed = false
+    d.pointer = new Pointer(context, 0, seq, block.cohorts.length)
+
+    for (const dd of d.deltas) {
+      if (keys && dd.changed && dd.pointer && dd.pointer.changed) dd.pointer = null
+      dd.changed = false
+      const p = dd.pointer
+      if (!p) continue
+      p.core = context.getCoreOffsetLocal(p.context, p.core)
+      p.context = context
+    }
+
+    block.cohorts.push(d.deltas)
+  }
+}
+
+function supportsCompression(type) {
+  return type !== TYPE_COMPAT && type !== 0
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hyperbee2",
-  "version": "1.2.0",
+  "version": "2.0.0",
   "description": "btree",
   "main": "index.js",
   "files": [
@@ -10,7 +10,8 @@
   ],
   "scripts": {
     "format": "prettier . --write",
-    "test": "prettier . --check && node test/all.js",
+    "lint": "lunte && prettier . --check",
+    "test": "node test/all.js",
     "test:bare": "bare test/all.js",
     "test:generate": "brittle -r test/all.js test/*.js"
   },
@@ -26,6 +27,7 @@
   "devDependencies": {
     "brittle": "^3.18.0",
     "corestore": "^7.4.5",
+    "lunte": "^1.4.0",
     "prettier": "^3.6.2",
     "prettier-config-holepunch": "^2.0.0"
   },