hyperbee2 1.1.2 → 2.0.0

package/lib/write.js

@@ -1,33 +1,58 @@
 const b4a = require('b4a')
 const c = require('compact-encoding')
-const { Block } = require('./encoding.js')
+const { OP_COHORT } = require('./compression.js')
+const { encodeBlock, TYPE_COMPAT, TYPE_LATEST } = require('./encoding.js')
 const {
+  Pointer,
+  KeyPointer,
+  ValuePointer,
   TreeNode,
   TreeNodePointer,
   MIN_KEYS,
-  UNCHANGED,
-  CHANGED,
+  INSERTED,
   NEEDS_SPLIT
 } = require('./tree.js')
 
+const PREFERRED_BLOCK_SIZE = 4096
+const INLINE_VALUE_SIZE = 4096
+const ESTIMATED_POINTER_SIZE = 12 // conversative, seq=8b, offset=2b, core=2b
+
 module.exports = class WriteBatch {
-  constructor(tree, { length = -1, key = null, autoUpdate = true } = {}) {
+  constructor(tree, opts = {}) {
+    const {
+      length = -1,
+      key = null,
+      autoUpdate = true,
+      compat = false,
+      type = compat ? TYPE_COMPAT : TYPE_LATEST,
+      deltaMax = supportsCompression(type) ? 16 : 0,
+      deltaMin = supportsCompression(type) ? 1 : Infinity,
+      inlineValueSize = INLINE_VALUE_SIZE,
+      preferredBlockSize = PREFERRED_BLOCK_SIZE
+    } = opts
+
     this.tree = tree
+    this.deltaMax = deltaMax
+    this.deltaMin = deltaMin
+    this.inlineValueSize = inlineValueSize
+    this.preferredBlockSize = preferredBlockSize
     this.snapshot = tree.snapshot()
     this.autoUpdate = autoUpdate
     this.length = length
     this.key = key
+    this.type = type
     this.closed = false
+    this.applied = 0
     this.root = null
     this.ops = []
   }
 
   tryPut(key, value) {
-    this.ops.push({ put: true, key, value })
+    this.ops.push({ put: true, applied: false, key, value })
   }
 
   tryDelete(key) {
-    this.ops.push({ put: false, key, value: null })
+    this.ops.push({ put: false, applied: false, key, value: null })
   }
 
   tryClear() {
@@ -51,7 +76,6 @@ module.exports = class WriteBatch {
 
     try {
       const ops = this.ops
-      this.ops = []
 
       const root = await this.tree.bootstrap()
 
@@ -60,20 +84,15 @@ module.exports = class WriteBatch {
 
       const changed = length === 0
       const seq = length === 0 ? 0 : length - 1
+      const value = changed ? new TreeNode([], []) : null
 
       this.length = length
-      this.root = new TreeNodePointer(
-        context,
-        0,
-        seq,
-        0,
-        changed,
-        changed ? new TreeNode([], []) : null
-      )
+      this.root = new TreeNodePointer(context, 0, seq, 0, changed, value)
 
       for (const op of ops) {
-        if (op.put) await this._put(op.key, op.value)
-        else await this._delete(op.key)
+        if (op.put) op.applied = await this._put(op.key, op.value)
+        else op.applied = await this._delete(op.key)
+        if (op.applied) this.applied++
       }
 
       await this._flush()
@@ -110,15 +129,16 @@ module.exports = class WriteBatch {
 
       while (s < e) {
         const mid = (s + e) >> 1
-        const m = v.keys[mid]
+        const m = v.keys.get(mid)
 
         c = b4a.compare(target, m.key)
 
         if (c === 0) {
-          if (b4a.equals(m.value, value)) return
+          const existing = await this.snapshot.inflateValue(m)
+          if (b4a.equals(existing, value)) return false
           v.setValue(this.tree.context, mid, value)
           for (let i = 0; i < stack.length; i++) stack[i].changed = true
-          return
+          return true
         }
 
         if (c < 0) e = mid
@@ -126,16 +146,21 @@ module.exports = class WriteBatch {
       }
 
       const i = c < 0 ? e : s
-      ptr = v.children[i]
+      ptr = v.children.get(i)
     }
 
     const v = ptr.value ? this.snapshot.bump(ptr) : await this.snapshot.inflate(ptr)
-    let status = v.put(this.tree.context, target, value, null)
-
-    if (status === UNCHANGED) return
+    let status = v.insertLeaf(this.tree.context, target, value)
+
+    if (status >= 0) {
+      // already exists, upsert if changed
+      const m = v.keys.get(status)
+      const existing = await this.snapshot.inflateValue(m)
+      if (b4a.equals(existing, value)) return false
+      v.setValue(this.tree.context, status, value)
+    }
 
     ptr.changed = true
-
     for (let i = 0; i < stack.length; i++) stack[i].changed = true
 
     while (status === NEEDS_SPLIT) {
@@ -145,16 +170,19 @@ module.exports = class WriteBatch {
 
       if (parent) {
         const p = parent.value ? this.snapshot.bump(parent) : await this.snapshot.inflate(parent)
-        status = p.put(this.tree.context, median.key, median.value, right)
+        status = p.insertNode(this.tree.context, median, right)
         ptr = parent
       } else {
         this.root = new TreeNodePointer(this.tree.context, 0, 0, 0, true, new TreeNode([], []))
         this.root.value.keys.push(median)
-        this.root.value.children.push(ptr, right)
+        this.root.value.children.push(ptr)
+        this.root.value.children.push(right)
         this.snapshot.bump(this.root)
-        status = UNCHANGED
+        status = INSERTED
       }
     }
+
+    return true
   }
 
   async _delete(key) {
@@ -172,32 +200,33 @@ module.exports = class WriteBatch {
 
       while (s < e) {
         const mid = (s + e) >> 1
-        c = b4a.compare(key, v.keys[mid].key)
+        c = b4a.compare(key, v.keys.get(mid).key)
 
         if (c === 0) {
           if (v.children.length) await this._setKeyToNearestLeaf(v, mid, stack)
           else v.removeKey(mid)
-
           // we mark these as changed late, so we don't rewrite them if it is a 404
           for (let i = 0; i < stack.length; i++) stack[i].changed = true
           this.root = await this._rebalance(stack)
-          return
+          return true
         }
 
         if (c < 0) e = mid
         else s = mid + 1
       }
 
-      if (!v.children.length) return
+      if (!v.children.length) return false
 
       const i = c < 0 ? e : s
-      ptr = v.children[i]
+      ptr = v.children.get(i)
     }
+
+    return false
   }
 
   async _setKeyToNearestLeaf(v, index, stack) {
-    let left = v.children[index]
-    let right = v.children[index + 1]
+    let left = v.children.get(index)
+    let right = v.children.get(index + 1)
 
     const [ls, rs] = await Promise.all([this._leafSize(left, false), this._leafSize(right, true)])
 
@@ -206,28 +235,28 @@ module.exports = class WriteBatch {
       stack.push(right)
       let r = right.value ? this.snapshot.bump(right) : await this.snapshot.inflate(right)
       while (r.children.length) {
-        right = r.children[0]
+        right = r.children.get(0)
         stack.push(right)
         r = right.value ? this.snapshot.bump(right) : await this.snapshot.inflate(right)
       }
-      v.keys[index] = r.keys.shift()
+      v.keys.set(index, r.keys.shift())
     } else {
       // if fewer leaves on the right
       stack.push(left)
       let l = left.value ? this.snapshot.bump(left) : await this.snapshot.inflate(left)
       while (l.children.length) {
-        left = l.children[l.children.length - 1]
+        left = l.children.get(l.children.length - 1)
         stack.push(left)
         l = left.value ? this.snapshot.bump(left) : await this.snapshot.inflate(left)
       }
-      v.keys[index] = l.keys.pop()
+      v.keys.set(index, l.keys.pop())
     }
   }
 
   async _leafSize(ptr, goLeft) {
     let v = ptr.value ? this.snapshot.bump(ptr) : await this.snapshot.inflate(ptr)
     while (v.children.length) {
-      ptr = v.children[goLeft ? 0 : v.children.length - 1]
+      ptr = v.children.get(goLeft ? 0 : v.children.length - 1)
       v = ptr.value ? this.snapshot.bump(ptr) : await this.snapshot.inflate(ptr)
     }
     return v.keys.length
@@ -235,6 +264,7 @@ module.exports = class WriteBatch {
 
   async _rebalance(stack) {
     const root = stack[0]
+    const minKeys = this.tree.context.minKeys
 
     while (stack.length > 1) {
       const ptr = stack.pop()
@@ -251,11 +281,11 @@ module.exports = class WriteBatch {
       let l = left && (left.value ? this.snapshot.bump(left) : await this.snapshot.inflate(left))
 
       // maybe borrow from left sibling?
-      if (l && l.keys.length > MIN_KEYS) {
+      if (l && l.keys.length > minKeys) {
         left.changed = true
-        v.keys.unshift(p.keys[index - 1])
+        v.keys.unshift(p.keys.get(index - 1))
         if (l.children.length) v.children.unshift(l.children.pop())
-        p.keys[index - 1] = l.keys.pop()
+        p.keys.set(index - 1, l.keys.pop())
         return root
       }
 
@@ -263,11 +293,11 @@ module.exports = class WriteBatch {
         right && (right.value ? this.snapshot.bump(right) : await this.snapshot.inflate(right))
 
       // maybe borrow from right sibling?
-      if (r && r.keys.length > MIN_KEYS) {
+      if (r && r.keys.length > minKeys) {
         right.changed = true
-        v.keys.push(p.keys[index])
+        v.keys.push(p.keys.get(index))
         if (r.children.length) v.children.push(r.children.shift())
-        p.keys[index] = r.keys.shift()
+        p.keys.set(index, r.keys.shift())
         return root
       }
 
@@ -282,7 +312,7 @@ module.exports = class WriteBatch {
       }
 
       left.changed = true
-      l.merge(r, p.keys[index])
+      l.merge(r, p.keys.get(index))
 
       parent.changed = true
       p.removeKey(index)
@@ -290,70 +320,101 @@ module.exports = class WriteBatch {
 
     const r = root.value ? this.snapshot.bump(root) : await this.snapshot.inflate(root)
     // check if the tree shrunk
-    if (!r.keys.length && r.children.length) return r.children[0]
+    if (!r.keys.length && r.children.length) return r.children.get(0)
     return root
   }
 
+  _shouldInlineValue(k) {
+    if (!k.value || supportsCompression(this.type)) return true
+    if (k.valuePointer) return true
+    return k.value.byteLength <= this.inlineValueSize
+  }
+
   async _flush() {
-    if (!this.root || !this.root.changed) return
+    if (!this.root || !this.root.changed) {
+      return
+    }
+
+    if (!this.root.value) await this.snapshot.inflate(this.root)
+
+    let update = { size: 0, nodes: [], keys: [], values: [] }
+    let minValue = -1
 
-    const update = { node: [], keys: [] }
     const batch = [update]
-    const stack = [{ update, node: this.root }]
+    const stack = [this.root]
+    const values = []
+
     const context = this.tree.context.getLocalContext()
+    const activeRequests = this.tree.activeRequests
 
-    await context.update(this.tree.activeRequests)
+    await context.update(activeRequests)
 
     while (stack.length > 0) {
-      const { update, node } = stack.pop()
+      const node = stack.pop()
 
-      node.changed = false
-      update.node.push(node)
+      if (this.type !== TYPE_COMPAT && update.size >= this.preferredBlockSize) {
+        update = { size: 0, nodes: [], keys: [], values: [] }
+        batch.push(update)
+      }
 
-      for (let i = 0; i < node.value.keys.length; i++) {
-        const k = node.value.keys[i]
+      update.nodes.push(node)
+      update.size += getEstimatedNodeSize(node.value)
 
-        if (!k.changed) {
-          k.core = await context.getCoreOffset(k.context, k.core, this.tree.activeRequests)
-          k.context = context
-          continue
+      const keys = node.value.keys
+      const children = node.value.children
+
+      for (let i = 0; i < keys.entries.length; i++) {
+        const k = keys.entries[i]
+        if (!k.changed) continue
+
+        if (!this._shouldInlineValue(k)) {
+          values.push(k)
+          k.valuePointer = new ValuePointer(context, 0, 0, 0, 0)
+
+          if (minValue === -1 || minValue < k.value.byteLength) {
+            minValue = k.value.byteLength
+          }
         }
 
-        k.changed = false
         update.keys.push(k)
+        update.size += getEstimatedKeySize(k)
       }
 
-      let first = true
+      for (let i = 0; i < children.entries.length; i++) {
+        const c = children.entries[i]
+        if (!c.value || !c.changed) continue
+        children.touch(i)
+        stack.push(c)
+      }
+    }
 
-      for (let i = 0; i < node.value.children.length; i++) {
-        const n = node.value.children[i]
+    if (this.type === TYPE_COMPAT) await toCompatType(context, batch, this.ops)
 
-        if (!n.changed) {
-          n.core = await context.getCoreOffset(n.context, n.core, this.tree.activeRequests)
-          n.context = context
-          continue
-        }
+    const length = context.core.length
+
+    // if noop and not genesis, bail early
+    if (this.applied === 0 && length > 0) return
+
+    if (minValue > -1 && minValue + update.size < this.preferredBlockSize) {
+      // TODO: repack the value into the block
+    }
+
+    if (values.length) {
+      update = { size: 0, nodes: [], keys: [], values: [] }
+
+      for (let i = 0; i < values.length; i++) {
+        const k = values[i]
 
-        if (first) {
-          stack.push({ update, node: n })
-          first = false
-        } else {
-          const update = { node: [], keys: [] }
+        update.size += getEstimatedValueSize(k)
+        update.values.push(k)
+
+        if (i === values.length - 1 || update.size >= this.preferredBlockSize) {
           batch.push(update)
-          stack.push({ update, node: n })
+          update = { size: 0, nodes: [], keys: [], values: [] }
         }
       }
     }
 
-    // if only the root was marked dirty and is === current bootstrap the batch is a noop - skip
-    if (update.node.length === 1 && update.keys.length === 0) {
-      const b = await this.tree.bootstrap()
-      const n = update.node[0]
-      if (b && b.context === n.context && b.core === n.core && b.seq === n.seq) return
-      if (!b && n.value.isEmpty()) return
-    }
-
-    const length = context.core.length
     const blocks = new Array(batch.length)
 
     for (let i = 0; i < batch.length; i++) {
@@ -361,33 +422,63 @@ module.exports = class WriteBatch {
       const seq = length + batch.length - i - 1
 
       const block = {
-        type: 0,
+        type: this.type,
         checkpoint: 0,
         batch: { start: batch.length - 1 - i, end: i },
         previous: null,
+        metadata: null,
         tree: null,
-        data: null,
-        cores: null
+        keys: null,
+        values: null,
+        cohorts: null
+      }
+
+      for (const k of update.values) {
+        if (block.values === null) block.values = []
+        const ptr = k.valuePointer
+
+        ptr.core = 0
+        ptr.context = context
+        ptr.seq = seq
+        ptr.offset = block.values.length
+        ptr.split = 0
+
+        block.values.push(k.value)
+        k.value = null // unlinked
       }
 
       for (const k of update.keys) {
-        if (block.data === null) block.data = []
+        if (block.keys === null) block.keys = []
 
         k.core = 0
         k.context = context
         k.seq = seq
-        k.offset = block.data.length
-        block.data.push(k)
+        k.offset = block.keys.length
+        k.changed = false
+
+        if (k.valuePointer) updateValuePointerContext(k.valuePointer, context)
+
+        block.keys.push(k)
       }
 
-      for (const n of update.node) {
+      for (const n of update.nodes) {
         if (block.tree === null) block.tree = []
 
         n.core = 0
         n.context = context
         n.seq = seq
         n.offset = block.tree.length
-        block.tree.push(n.value)
+        n.changed = false
+
+        const treeDelta = {
+          keys: n.value.keys.flush(this.deltaMax, this.deltaMin),
+          children: n.value.children.flush(this.deltaMax, this.deltaMin)
+        }
+
+        prepareCohorts(context, block, seq, treeDelta.keys, supportsCompression)
+        prepareCohorts(context, block, seq, treeDelta.children, false)
+
+        block.tree.push(treeDelta)
       }
 
       blocks[seq - length] = block
@@ -396,22 +487,19 @@ module.exports = class WriteBatch {
     const buffers = new Array(blocks.length)
 
     if (blocks.length > 0 && this.length > 0) {
-      const core = this.key
-        ? await context.getCoreOffsetByKey(this.key, this.tree.activeRequests)
-        : 0
+      const core = this.key ? await context.getCoreOffsetByKey(this.key, activeRequests) : 0
       blocks[blocks.length - 1].previous = { core, seq: this.length - 1 }
     }
 
     // TODO: make this transaction safe
     if (context.changed) {
-      context.changed = false
       context.checkpoint = context.core.length + blocks.length
-      blocks[blocks.length - 1].cores = context.cores
+      blocks[blocks.length - 1].metadata = context.flush()
     }
 
     for (let i = 0; i < blocks.length; i++) {
       blocks[i].checkpoint = context.checkpoint
-      buffers[i] = c.encode(Block, blocks[i])
+      buffers[i] = encodeBlock(blocks[i])
     }
 
     if (this.closed) {
@@ -423,9 +511,90 @@ module.exports = class WriteBatch {
     for (let i = 0; i < batch.length; i++) {
       const update = batch[i]
 
-      for (let j = 0; j < update.node.length; j++) {
-        this.snapshot.bump(update.node)
+      for (let j = 0; j < update.nodes.length; j++) {
+        const node = update.nodes[j]
+        this.snapshot.bump(node)
       }
     }
   }
 }
+
+async function toCompatType(context, batch, ops) {
+  const map = new Map()
+  let index = 0
+
+  for (const k of batch[0].keys) {
+    map.set(b4a.toString(k.key, 'hex'), k)
+  }
+
+  for (let i = ops.length - 1; i >= 0; i--) {
+    const op = ops[i]
+    if (!op.put && !op.applied) continue
+
+    const k = map.get(b4a.toString(op.key, 'hex'))
+
+    const j = index++
+    if (j === batch.length) batch.push({ size: 0, nodes: [], keys: [], values: [] })
+    batch[j].keys = [k || new KeyPointer(context, 0, 0, 0, false, op.key, op.value, null)]
+  }
+
+  // compat doesnt support block 0
+  if (context.core.length > 0) return
+
+  const header = b4a.from('0a086879706572626565', 'hex')
+  await context.core.append(header)
+}
+
+function updateValuePointerContext(valuePointer, context) {
+  valuePointer.core = context.getCoreOffsetLocal(valuePointer.context, valuePointer.core)
+  valuePointer.context = context
+}
+
+// TODO: this isnt right anymore as we compress the delta post flush, but prob ok...
+function getEstimatedNodeSize(n) {
+  return (
+    n.keys.delta.length * ESTIMATED_POINTER_SIZE + n.children.delta.length * ESTIMATED_POINTER_SIZE
+  )
+}
+
+function getEstimatedKeySize(k) {
+  return k.key.byteLength + (k.valuePointer ? ESTIMATED_POINTER_SIZE : 0)
+}
+
+function getEstimatedValueSize(k) {
+  return k.value.byteLength
+}
+
+function prepareCohorts(context, block, seq, deltas, keys) {
+  for (const d of deltas) {
+    // same below but the delta might be a noop (ie add and the delete) - handle that
+    if (keys && d.changed && d.pointer && d.pointer.changed) d.pointer = null
+
+    if (d.pointer && d.pointer.context !== context) {
+      const p = d.pointer
+      p.core = context.getCoreOffsetLocal(p.context, p.core)
+      p.context = context
+    }
+
+    if (d.type !== OP_COHORT || !d.changed) continue
+    if (block.cohorts === null) block.cohorts = []
+
+    d.changed = false
+    d.pointer = new Pointer(context, 0, seq, block.cohorts.length)
+
+    for (const dd of d.deltas) {
+      if (keys && dd.changed && dd.pointer && dd.pointer.changed) dd.pointer = null
+      dd.changed = false
+      const p = dd.pointer
+      if (!p) continue
+      p.core = context.getCoreOffsetLocal(p.context, p.core)
+      p.context = context
+    }
+
+    block.cohorts.push(d.deltas)
+  }
+}
+
+function supportsCompression(type) {
+  return type !== TYPE_COMPAT && type !== 0
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hyperbee2",
-  "version": "1.1.2",
+  "version": "2.0.0",
   "description": "btree",
   "main": "index.js",
   "files": [
@@ -10,7 +10,8 @@
   ],
   "scripts": {
     "format": "prettier . --write",
-    "test": "prettier . --check && node test/all.js",
+    "lint": "lunte && prettier . --check",
+    "test": "node test/all.js",
     "test:bare": "bare test/all.js",
     "test:generate": "brittle -r test/all.js test/*.js"
   },
@@ -26,6 +27,7 @@
   "devDependencies": {
     "brittle": "^3.18.0",
     "corestore": "^7.4.5",
+    "lunte": "^1.4.0",
     "prettier": "^3.6.2",
     "prettier-config-holepunch": "^2.0.0"
   },