hyperbee2 0.0.0

package/lib/key-value-stream.js

@@ -0,0 +1,134 @@
+const { Readable } = require('streamx')
+const b4a = require('b4a')
+
+module.exports = class KeyValueStream extends Readable {
+  constructor(tree, options = {}) {
+    super({ eagerOpen: true })
+
+    this.tree = tree
+    this.root = null
+
+    this._stack = []
+    this._activeRequests = options.activeRequests || this.tree.activeRequests
+    this._limit = options.limit === undefined ? -1 : options.limit
+    this._reverse = !!options.reverse
+    this._start = options.gte || options.gt || null
+    this._end = options.lte || options.lt || null
+    this._inclusive = !!(this._reverse ? options.lte : options.gte)
+    this._compare = (this._reverse ? options.gte : options.lte) ? 0 : -1
+  }
+
+  async _openp() {
+    if (!this.root) this.root = await this.tree.bootstrap(this._activeRequests)
+    if (!this.root) return
+
+    if (this._limit === 0) return
+
+    this._stack.push({ node: this.root, offset: 0 })
+
+    if (this._reverse ? !this._end : !this._start) return
+
+    const offset = this._inclusive ? 0 : 1
+
+    while (true) {
+      const top = this._stack[this._stack.length - 1]
+      const v = top.node.value
+        ? this.tree.bump(top.node)
+        : await this.tree.inflate(top.node, this._activeRequests)
+
+      for (let i = 0; i < v.keys.length; i++) {
+        const j = this._reverse ? v.keys.length - 1 - i : i
+
+        const c = this._reverse
+          ? b4a.compare(v.keys[j].key, this._end)
+          : b4a.compare(this._start, v.keys[j].key)
+
+        if (c < 0) break
+        top.offset = 2 * i + 1 + (c === 0 ? offset : 1)
+      }
+
+      const child = (top.offset & 1) === 0
+      const k = top.offset >> 1
+
+      if (!child || k >= v.children.length) break
+
+      const j = this._reverse ? v.children.length - 1 - k : k
+
+      this._stack.push({
+        node: v.children[j],
+        offset: 0
+      })
+
+      top.offset++
+    }
+  }
+
+  async _readp() {
+    while (this._stack.length && (this._limit === -1 || this._limit > 0)) {
+      const top = this._stack.pop()
+      const v = top.node.value
+        ? this.tree.bump(top.node)
+        : await this.tree.inflate(top.node, this._activeRequests)
+
+      const offset = top.offset++
+      const child = (offset & 1) === 0
+      const k = offset >> 1
+
+      if (child) {
+        this._stack.push(top)
+        if (k < v.children.length) {
+          const j = this._reverse ? v.children.length - 1 - k : k
+          this._stack.push({ node: v.children[j], offset: 0 })
+        }
+        continue
+      }
+
+      if (k < v.keys.length) {
+        const j = this._reverse ? v.keys.length - 1 - k : k
+
+        const data = top.node.value.keys[j]
+
+        const c = this._reverse
+          ? this._start
+            ? b4a.compare(this._start, data.key)
+            : -1
+          : this._end
+            ? b4a.compare(data.key, this._end)
+            : -1
+
+        if (c > this._compare) break
+
+        this._stack.push(top)
+        this.push(this._finalize(data))
+        if (this._limit !== -1) this._limit--
+        return
+      }
+    }
+
+    this.push(null)
+  }
+
+  _finalize(kv) {
+    return kv
+  }
+
+  async _open(cb) {
+    try {
+      await this._openp()
+    } catch (err) {
+      cb(err)
+      return
+    }
+    cb(null)
+  }
+
+  async _read(cb) {
+    try {
+      await this._readp()
+    } catch (err) {
+      cb(err)
+      return
+    }
+    cb(null)
+  }
+}

package/lib/tree.js

@@ -0,0 +1,130 @@
+const b4a = require('b4a')
+
+const T = 5
+const MIN_KEYS = T - 1
+const MAX_CHILDREN = MIN_KEYS * 2 + 1
+
+class DataPointer {
+  constructor(context, core, seq, offset, changed, key, value) {
+    this.context = context
+
+    this.core = core
+    this.seq = seq
+    this.offset = offset
+    this.changed = changed
+
+    this.key = key
+    this.value = value
+  }
+}
+
+class TreeNodePointer {
+  constructor(context, core, seq, offset, changed, value) {
+    this.context = context
+
+    this.core = core
+    this.seq = seq
+    this.offset = offset
+    this.changed = changed
+
+    this.value = value
+
+    this.next = null
+    this.prev = null
+  }
+}
+
+class TreeNode {
+  constructor(keys, children) {
+    this.keys = keys
+    this.children = children
+  }
+
+  put(context, key, value, child) {
+    let s = 0
+    let e = this.keys.length
+    let c
+
+    while (s < e) {
+      const mid = (s + e) >> 1
+      const k = this.keys[mid]
+
+      c = b4a.compare(key, k.key)
+
+      if (c === 0) {
+        this.keys[mid] = new DataPointer(context, 0, 0, 0, true, key, value)
+        return true
+      }
+
+      if (c < 0) e = mid
+      else s = mid + 1
+    }
+
+    const i = c < 0 ? e : s
+    this.keys.splice(i, 0, new DataPointer(context, 0, 0, 0, true, key, value))
+    if (child) this.children.splice(i + 1, 0, child)
+
+    return this.keys.length < MAX_CHILDREN
+  }
+
+  setValue(context, i, value) {
+    this.keys[i] = new DataPointer(context, 0, 0, 0, true, this.keys[i].key, value)
+  }
+
+  removeKey(i) {
+    this.keys.splice(i, 1)
+    if (this.children.length) {
+      this.children.splice(i + 1, 1)
+    }
+  }
+
+  siblings(parent) {
+    for (let i = 0; i < parent.children.length; i++) {
+      if (parent.children[i].value !== this) continue // TODO: move to a seq/offset check instead
+
+      const left = i ? parent.children[i - 1] : null
+      const right = i < parent.children.length - 1 ? parent.children[i + 1] : null
+      return { left, index: i, right }
+    }
+
+    // TODO: assert
+    throw new Error('Bad parent')
+  }
+
+  merge(node, median) {
+    this.keys.push(median)
+    for (let i = 0; i < node.keys.length; i++) this.keys.push(node.keys[i])
+    for (let i = 0; i < node.children.length; i++) this.children.push(node.children[i])
+  }
+
+  split(context) {
+    const len = this.keys.length >> 1
+    const right = new TreeNodePointer(context, 0, 0, 0, true, new TreeNode([], []))
+
+    while (right.value.keys.length < len) right.value.keys.push(this.keys.pop())
+    right.value.keys.reverse()
+
+    const median = this.keys.pop()
+
+    if (this.children.length) {
+      while (right.value.children.length < len + 1) right.value.children.push(this.children.pop())
+      right.value.children.reverse()
+    }
+
+    return {
+      left: this,
+      median,
+      right
+    }
+  }
+}
+
+exports.T = T
+exports.MIN_KEYS = MIN_KEYS
+exports.MAX_CHILDREN = MAX_CHILDREN
+
+exports.TreeNodePointer = TreeNodePointer
+exports.TreeNode = TreeNode
+exports.DataPointer = DataPointer
+
+exports.EMPTY = new TreeNodePointer(null, 0, 0, 0, false, null)

package/lib/write.js

@@ -0,0 +1,409 @@
+const b4a = require('b4a')
+const c = require('compact-encoding')
+const { TreeNode, TreeNodePointer, MIN_KEYS } = require('./tree.js')
+const { Block } = require('./encoding.js')
+
+module.exports = class WriteBatch {
+  constructor(tree, { length = -1, key = null, autoUpdate = true } = {}) {
+    this.tree = tree
+    this.snapshot = tree.snapshot()
+    this.autoUpdate = autoUpdate
+    this.length = length
+    this.key = key
+    this.closed = false
+    this.flushing = false
+    this.root = null
+    this.ops = []
+  }
+
+  tryPut(key, value) {
+    this.ops.push({ put: true, key, value })
+  }
+
+  tryDelete(key) {
+    this.ops.push({ put: false, key, value: null })
+  }
+
+  tryClear() {
+    this.ops = []
+    this.length = 0
+  }
+
+  async flush() {
+    if (this.flushing) throw new Error('Already flushed')
+    this.flushing = true
+
+    const ops = this.ops
+    this.ops = []
+
+    await this.tree.ready()
+
+    const length = this.length === -1 ? this.tree.core.length : this.length
+
+    const changed = length === 0
+    const seq = length === 0 ? 0 : length - 1
+
+    const context = this.key ? this.tree.context.getContextByKey(this.key) : this.tree.context
+
+    this.length = length
+    this.root = new TreeNodePointer(
+      context,
+      0,
+      seq,
+      0,
+      changed,
+      changed ? new TreeNode([], []) : null
+    )
+
+    for (const op of ops) {
+      if (op.put) await this._put(op.key, op.value)
+      else await this._delete(op.key)
+    }
+
+    await this._flush()
+    await this.snapshot.close()
+
+    if (this.autoUpdate) {
+      this.tree.update(this.root)
+    }
+  }
+
+  close() {
+    this.closed = true
+    return this.snapshot.close()
+  }
+
+  async _put(key, value) {
+    const stack = []
+    const target = key
+
+    let ptr = this.root
+
+    while (true) {
+      const v = ptr.value ? this.snapshot.bump(ptr) : await this.snapshot.inflate(ptr)
+      if (!v.children.length) break
+
+      stack.push(ptr)
+
+      let s = 0
+      let e = v.keys.length
+      let c = 0
+
+      while (s < e) {
+        const mid = (s + e) >> 1
+        const m = v.keys[mid]
+
+        c = b4a.compare(target, m.key)
+
+        if (c === 0) {
+          if (b4a.compare(m.value, value)) return
+          v.setValue(this.tree.context, mid, value)
+          for (let i = 0; i < stack.length; i++) stack[i].changed = true
+          return
+        }
+
+        if (c < 0) e = mid
+        else s = mid + 1
+      }
+
+      const i = c < 0 ? e : s
+      ptr = v.children[i]
+    }
+
+    const v = ptr.value ? this.snapshot.bump(ptr) : await this.snapshot.inflate(ptr)
+    let needsSplit = !v.put(this.tree.context, target, value, null)
+
+    ptr.changed = true
+
+    for (let i = 0; i < stack.length; i++) stack[i].changed = true
+
+    while (needsSplit) {
+      const v = ptr.value ? this.snapshot.bump(ptr) : await this.snapshot.inflate(ptr)
+      const parent = stack.pop()
+      const { median, right } = v.split(this.tree.context)
+
+      if (parent) {
+        const p = parent.value ? this.snapshot.bump(parent) : await this.snapshot.inflate(parent)
+        needsSplit = !p.put(this.tree.context, median.key, median.value, right)
+        ptr = parent
+      } else {
+        this.root = new TreeNodePointer(this.tree.context, 0, 0, 0, true, new TreeNode([], []))
+        this.root.value.keys.push(median)
+        this.root.value.children.push(ptr, right)
+        this.snapshot.bump(this.root)
+        needsSplit = false
+      }
+    }
+  }
+
+  async _delete(key) {
+    let ptr = this.root
+
+    const stack = []
+
+    while (true) {
+      const v = ptr.value ? this.snapshot.bump(ptr) : await this.snapshot.inflate(ptr)
+      stack.push(ptr)
+
+      let s = 0
+      let e = v.keys.length
+      let c = 0
+
+      while (s < e) {
+        const mid = (s + e) >> 1
+        c = b4a.compare(key, v.keys[mid].key)
+
+        if (c === 0) {
+          if (v.children.length) await this._setKeyToNearestLeaf(v, mid, stack)
+          else v.removeKey(mid)
+
+          // we mark these as changed late, so we don't rewrite them if it is a 404
+          for (let i = 0; i < stack.length; i++) stack[i].changed = true
+          this.root = await this._rebalance(stack)
+          return
+        }
+
+        if (c < 0) e = mid
+        else s = mid + 1
+      }
+
+      if (!v.children.length) return
+
+      const i = c < 0 ? e : s
+      ptr = v.children[i]
+    }
+  }
+
+  async _setKeyToNearestLeaf(v, index, stack) {
+    let left = v.children[index]
+    let right = v.children[index + 1]
+
+    const [ls, rs] = await Promise.all([this._leafSize(left, false), this._leafSize(right, true)])
+
+    if (ls < rs) {
+      // if fewer leaves on the left
+      stack.push(right)
+      let r = right.value ? this.snapshot.bump(right) : await this.snapshot.inflate(right)
+      while (r.children.length) {
+        right = r.children[0]
+        stack.push(right)
+        r = right.value ? this.snapshot.bump(right) : await this.snapshot.inflate(right)
+      }
+      v.keys[index] = r.keys.shift()
+    } else {
+      // if fewer leaves on the right
+      stack.push(left)
+      let l = left.value ? this.snapshot.bump(left) : await this.snapshot.inflate(left)
+      while (l.children.length) {
+        left = l.children[l.children.length - 1]
+        stack.push(left)
+        l = left.value ? this.snapshot.bump(left) : await this.snapshot.inflate(left)
+      }
+      v.keys[index] = l.keys.pop()
+    }
+  }
+
+  async _leafSize(ptr, goLeft) {
+    let v = ptr.value ? this.snapshot.bump(ptr) : await this.snapshot.inflate(ptr)
+    while (v.children.length) {
+      ptr = v.children[goLeft ? 0 : v.children.length - 1]
+      v = ptr.value ? this.snapshot.bump(ptr) : await this.snapshot.inflate(ptr)
+    }
+    return v.keys.length
+  }
+
+  async _rebalance(stack) {
+    const root = stack[0]
+
+    while (stack.length > 1) {
+      const ptr = stack.pop()
+      const parent = stack[stack.length - 1]
+
+      const v = ptr.value ? this.snapshot.bump(ptr) : await this.snapshot.inflate(ptr)
+
+      if (v.keys.length >= MIN_KEYS) return root
+
+      const p = parent.value ? this.snapshot.bump(parent) : await this.snapshot.inflate(parent)
+
+      let { left, index, right } = v.siblings(p)
+
+      let l = left && (left.value ? this.snapshot.bump(left) : await this.snapshot.inflate(left))
+
+      // maybe borrow from left sibling?
+      if (l && l.keys.length > MIN_KEYS) {
+        left.changed = true
+        v.keys.unshift(p.keys[index - 1])
+        if (l.children.length) v.children.unshift(l.children.pop())
+        p.keys[index - 1] = l.keys.pop()
+        return root
+      }
+
+      let r =
+        right && (right.value ? this.snapshot.bump(right) : await this.snapshot.inflate(right))
+
+      // maybe borrow from right sibling?
+      if (r && r.keys.length > MIN_KEYS) {
+        right.changed = true
+        v.keys.push(p.keys[index])
+        if (r.children.length) v.children.push(r.children.shift())
+        p.keys[index] = r.keys.shift()
+        return root
+      }
+
+      // merge node with another sibling
+      if (l) {
+        index--
+        r = v
+        right = ptr
+      } else {
+        l = v
+        left = ptr
+      }
+
+      left.changed = true
+      l.merge(r, p.keys[index])
+
+      parent.changed = true
+      p.removeKey(index)
+    }
+
+    const r = root.value ? this.snapshot.bump(root) : await this.snapshot.inflate(root)
+    // check if the tree shrunk
+    if (!r.keys.length && r.children.length) return r.children[0]
+    return root
+  }
+
+  async _flush() {
+    if (!this.root || !this.root.changed) return
+
+    const update = { node: [], keys: [] }
+    const batch = [update]
+    const stack = [{ update, node: this.root }]
+
+    await this.tree.context.update(this.tree.activeRequests)
+
+    while (stack.length > 0) {
+      const { update, node } = stack.pop()
+
+      node.changed = false
+      update.node.push(node)
+
+      for (let i = 0; i < node.value.keys.length; i++) {
+        const k = node.value.keys[i]
+
+        if (!k.changed) {
+          k.core = await this.tree.context.getCoreOffset(
+            k.context,
+            k.core,
+            this.tree.activeRequests
+          )
+          k.context = this.tree.context
+          continue
+        }
+
+        k.changed = false
+        update.keys.push(k)
+      }
+
+      let first = true
+
+      for (let i = 0; i < node.value.children.length; i++) {
+        const n = node.value.children[i]
+
+        if (!n.changed) {
+          n.core = await this.tree.context.getCoreOffset(
+            n.context,
+            n.core,
+            this.tree.activeRequests
+          )
+          n.context = this.tree.context
+          continue
+        }
+
+        if (first) {
+          stack.push({ update, node: n })
+          first = false
+        } else {
+          const update = { node: [], keys: [] }
+          batch.push(update)
+          stack.push({ update, node: n })
+        }
+      }
+    }
+
+    const length = this.tree.core.length
+    const blocks = new Array(batch.length)
+
+    for (let i = 0; i < batch.length; i++) {
+      const update = batch[i]
+      const seq = length + batch.length - i - 1
+
+      const block = {
+        type: 0,
+        checkpoint: 0,
+        batch: { start: batch.length - 1 - i, end: i },
+        previous: null,
+        tree: null,
+        data: null,
+        cores: null
+      }
+
+      for (const k of update.keys) {
+        if (block.data === null) block.data = []
+
+        k.core = 0
+        k.context = this.tree.context
+        k.seq = seq
+        k.offset = block.data.length
+        block.data.push(k)
+      }
+
+      for (const n of update.node) {
+        if (block.tree === null) block.tree = []
+
+        n.core = 0
+        n.context = this.tree.context
+        n.seq = seq
+        n.offset = block.tree.length
+        block.tree.push(n.value)
+      }
+
+      blocks[seq - length] = block
+    }
+
+    const buffers = new Array(blocks.length)
+
+    if (blocks.length > 0 && this.length > 0) {
+      const core = this.key
+        ? await this.tree.context.getCoreOffsetByKey(this.key, this.tree.activeRequests)
+        : 0
+      blocks[blocks.length - 1].previous = { core, seq: this.length - 1 }
+    }
+
+    // TODO: make this transaction safe
+    if (this.tree.context.changed) {
+      this.tree.context.changed = false
+      this.tree.context.checkpoint = this.tree.core.length + blocks.length
+      blocks[blocks.length - 1].cores = this.tree.context.cores
+    }
+
+    for (let i = 0; i < blocks.length; i++) {
+      blocks[i].checkpoint = this.tree.context.checkpoint
+      buffers[i] = c.encode(Block, blocks[i])
+    }
+
+    if (this.closed) {
+      throw new Error('Write batch is closed')
+    }
+
+    await this.tree.core.append(buffers)
+
+    for (let i = 0; i < batch.length; i++) {
+      const update = batch[i]
+
+      for (let j = 0; j < update.node.length; j++) {
+        this.snapshot.bump(update.node)
+      }
+    }
+  }
+}

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "hyperbee2",
+  "version": "0.0.0",
+  "description": "btree",
+  "main": "index.js",
+  "files": [
+    "index.js",
+    "lib/*.js",
+    "spec"
+  ],
+  "scripts": {
+    "format": "prettier . --write",
+    "test": "prettier . --check && node test/all.js",
+    "test:bare": "bare test/all.js",
+    "test:generate": "brittle -r test/all.js test/*.js"
+  },
+  "dependencies": {
+    "b4a": "^1.6.7",
+    "compact-encoding": "^2.16.1",
+    "hypercore": "^11.15.0",
+    "hyperschema": "^1.14.0",
+    "protocol-buffers-encodings": "^1.2.0",
+    "streamx": "^2.22.1"
+  },
+  "devDependencies": {
+    "brittle": "^3.18.0",
+    "corestore": "^7.4.5",
+    "prettier": "^3.6.2",
+    "prettier-config-holepunch": "^2.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/mafintosh/hyperbee2.git"
+  },
+  "author": "Mathias Buus (@mafintosh)",
+  "license": "Apache-2.0",
+  "bugs": {
+    "url": "https://github.com/mafintosh/hyperbee2/issues"
+  },
+  "homepage": "https://github.com/mafintosh/hyperbee2"
+}