hyperbee2 1.2.0 → 2.0.0

package/index.js

@@ -6,15 +6,24 @@ const { ChangesStream } = require('./lib/changes.js')
 const NodeCache = require('./lib/cache.js')
 const WriteBatch = require('./lib/write.js')
 const CoreContext = require('./lib/context.js')
-const { DataPointer, TreeNode, TreeNodePointer, EMPTY } = require('./lib/tree.js')
+const {
+  Pointer,
+  KeyPointer,
+  ValuePointer,
+  TreeNode,
+  TreeNodePointer,
+  EMPTY
+} = require('./lib/tree.js')
+const { DeltaOp, DeltaCohort, OP_COHORT } = require('./lib/compression.js')
 
 class Hyperbee {
   constructor(store, options = {}) {
     const {
+      t = 128, // legacy number for now, should be 128 now
       key = null,
       encryption = null,
       core = key ? store.get(key) : store.get({ key, name: 'bee', encryption }),
-      context = new CoreContext(store, core, core, encryption),
+      context = new CoreContext(store, core, core, encryption, t),
       maxCacheSize = 4096,
       cache = new NodeCache(maxCacheSize),
       root = null,
@@ -111,12 +120,12 @@ class Hyperbee {
     if (!this.view) await this.store.close()
   }
 
-  createReadStream(range) {
-    return new RangeStream(this, range)
+  createReadStream(options) {
+    return new RangeStream(this, options)
   }
 
-  createDiffStream(right, range) {
-    return new DiffStream(this, right, range)
+  createDiffStream(right, options) {
+    return new DiffStream(this, right, options)
   }
 
   createChangesStream(options) {
@@ -124,12 +133,21 @@ class Hyperbee {
   }
 
   async peek(range = {}) {
-    const rs = new KeyValueStream(this, { ...range, limit: 1 })
+    const rs = new RangeStream(this, { ...range, limit: 1 })
     let entry = null
     for await (const data of rs) entry = data
     return entry
   }
 
+  download(range = {}) {
+    const rs = new RangeStream(this, range)
+    rs.resume()
+    return new Promise((resolve, reject) => {
+      rs.once('error', reject)
+      rs.once('close', resolve)
+    })
+  }
+
   bump(ptr) {
     if (ptr.changed) return ptr.value
     this.cache.bump(ptr)
@@ -137,40 +155,75 @@ class Hyperbee {
     return ptr.value
   }
 
+  // TODO: unslab these and parallize
   async inflate(ptr, activeRequests = this.activeRequests) {
     if (ptr.value) {
       this.bump(ptr)
       return ptr.value
     }
 
-    const block = await ptr.context.getBlock(ptr.seq, ptr.core, activeRequests)
-    const context = await ptr.context.getContext(ptr.core, activeRequests)
+    const [block, context] = await Promise.all([
+      ptr.context.getBlock(ptr.seq, ptr.core, activeRequests),
+      ptr.context.getContext(ptr.core, activeRequests)
+    ])
+
     const tree = block.tree[ptr.offset]
 
     const keys = new Array(tree.keys.length)
     const children = new Array(tree.children.length)
 
     for (let i = 0; i < keys.length; i++) {
-      const k = tree.keys[i]
-      const blk =
-        k.seq === ptr.seq && k.core === 0 && ptr.core === 0
-          ? block
-          : await context.getBlock(k.seq, k.core, activeRequests)
-      const d = blk.data[k.offset]
-      keys[i] = new DataPointer(context, k.core, k.seq, k.offset, false, d.key, d.value)
+      const d = tree.keys[i]
+      keys[i] = inflateKey(context, d, ptr, block, activeRequests)
     }
 
     for (let i = 0; i < children.length; i++) {
-      const c = tree.children[i]
-      children[i] = new TreeNodePointer(context, c.core, c.seq, c.offset, false, null)
+      const d = tree.children[i]
+      children[i] = inflateChild(context, d, ptr, block, activeRequests)
     }
 
-    ptr.value = new TreeNode(keys, children)
+    ptr.value = new TreeNode(await Promise.all(keys), await Promise.all(children))
     this.bump(ptr)
 
     return ptr.value
   }
 
+  async finalizeKeyPointer(key, activeRequests = this.activeRequests) {
+    const value = key.value || (await this.inflateValue(key, activeRequests))
+
+    return {
+      core: key.context.getCore(key.core),
+      offset: key.offset,
+      seq: key.seq,
+      key: key.key,
+      value
+    }
+  }
+
+  async inflateValue(key, activeRequests = this.activeRequests) {
+    if (key.value) return key.value
+    if (!key.valuePointer) return null
+
+    const ptr = key.valuePointer
+
+    if (ptr.split === 0) {
+      const block = await ptr.context.getBlock(ptr.seq, ptr.core, activeRequests)
+      return block.values[ptr.offset]
+    }
+
+    const blockPromises = new Array(ptr.split + 1)
+    for (let i = 0; i < blockPromises.length; i++) {
+      blockPromises[i] = ptr.context.getBlock(ptr.seq - ptr.split + i, ptr.core, activeRequests)
+    }
+    const blocks = await Promise.all(blockPromises)
+    const splitValue = new Array(blockPromises.length)
+    for (let i = 0; i < splitValue.length - 1; i++) {
+      splitValue[i] = blocks[i].values[0]
+    }
+    splitValue[splitValue.length - 1] = blocks[blocks.length - 1].buffer[ptr.offset]
+    return b4a.concat(splitValue)
+  }
+
   async bootstrap(activeRequests = this.activeRequests) {
     if (!this.root) await this.ready()
     if (this.unbatch) await this._rollback(activeRequests)
@@ -223,11 +276,11 @@ class Hyperbee {
 
       while (s < e) {
         const mid = (s + e) >> 1
-        const m = v.keys[mid]
+        const m = v.keys.get(mid)
 
         c = b4a.compare(key, m.key)
 
-        if (c === 0) return m
+        if (c === 0) return this.finalizeKeyPointer(m, activeRequests)
 
         if (c < 0) e = mid
         else s = mid + 1
@@ -236,7 +289,7 @@ class Hyperbee {
       if (!v.children.length) return null
 
       const i = c < 0 ? e : s
-      ptr = v.children[i]
+      ptr = v.children.get(i)
     }
   }
 }
@@ -244,3 +297,84 @@ class Hyperbee {
 module.exports = Hyperbee
 
 function noop() {}
+
+function inflateKey(context, d, ptr, block, activeRequests) {
+  if (d.type === OP_COHORT) return inflateKeyCohort(context, d, ptr, block, activeRequests)
+  return inflateKeyDelta(context, d, ptr, block, activeRequests)
+}
+
+async function inflateKeyDelta(context, d, ptr, block, activeRequests) {
+  const k = d.pointer
+
+  if (!k) return new DeltaOp(false, d.type, d.index, null)
+
+  const blk =
+    k.seq === ptr.seq && k.core === 0 && ptr.core === 0
+      ? block
+      : await context.getBlock(k.seq, k.core, activeRequests)
+
+  const bk = blk.keys[k.offset]
+
+  let vp = null
+
+  if (bk.valuePointer) {
+    const p = bk.valuePointer
+    const ctx = await context.getContext(k.core, activeRequests)
+    vp = new ValuePointer(ctx, p.core, p.seq, p.offset, p.split)
+  }
+
+  const kp = new KeyPointer(context, k.core, k.seq, k.offset, false, bk.key, bk.value, vp)
+  return new DeltaOp(false, d.type, d.index, kp)
+}
+
+async function inflateKeyCohort(context, d, ptr, block, activeRequests) {
+  const co = d.pointer
+
+  const blk =
+    co.seq === ptr.seq && co.core === 0 && ptr.core === 0
+      ? block
+      : await context.getBlock(co.seq, co.core, activeRequests)
+
+  const cohort = blk.cohorts[co.offset]
+  const promises = new Array(cohort.length)
+
+  for (let i = 0; i < cohort.length; i++) {
+    const p = cohort[i]
+    const k = inflateKeyDelta(context, p, co, blk, activeRequests)
+    promises[i] = k
+  }
+
+  const p = new Pointer(context, co.core, co.seq, co.offset)
+  return new DeltaCohort(false, p, await Promise.all(promises))
+}
+
+function inflateChild(context, d, ptr, block, activeRequests) {
+  if (d.type === OP_COHORT) return inflateChildCohort(context, d, ptr, block, activeRequests)
+  return Promise.resolve(inflateChildDelta(context, d, ptr, block, activeRequests))
+}
+
+function inflateChildDelta(context, d, ptr, block, activeRequests) {
+  const p = d.pointer
+  const c = p && new TreeNodePointer(context, p.core, p.seq, p.offset, false, null)
+  return new DeltaOp(false, d.type, d.index, c)
+}
+
+async function inflateChildCohort(context, d, ptr, block, activeRequests) {
+  const co = d.pointer
+
+  const blk =
+    co.seq === ptr.seq && co.core === 0 && ptr.core === 0
+      ? block
+      : await context.getBlock(co.seq, co.core, activeRequests)
+
+  const cohort = blk.cohorts[co.offset]
+  const deltas = new Array(cohort.length)
+
+  for (let i = 0; i < cohort.length; i++) {
+    const c = cohort[i]
+    deltas[i] = inflateChildDelta(context, c, co, blk, activeRequests)
+  }
+
+  const p = new Pointer(context, co.core, co.seq, co.offset)
+  return new DeltaCohort(false, p, deltas)
+}

package/lib/cache.js

@@ -9,6 +9,13 @@ module.exports = class NodeCache {
     return this.latest ? this.latest.next : null
   }
 
+  empty() {
+    const maxSize = this.maxSize
+    this.maxSize = 0
+    this.gc()
+    this.maxSize = maxSize
+  }
+
   gc() {
     while (this.size > this.maxSize) {
       const old = this.oldest()

package/lib/compat.js

@@ -2,6 +2,7 @@
 
 const encodings = require('protocol-buffers-encodings')
 const b4a = require('b4a')
+const { OP_COHORT, OP_INSERT } = require('./compression')
 const varint = encodings.varint
 const skip = encodings.skip
 
@@ -12,26 +13,43 @@ function decodeCompat(buffer, seq) {
   const node = buffer.length > 1 ? decodeNode(buffer) : null
   const index = node ? decodeYoloIndex(node.index) : { levels: [] }
   const morphed = {
-    type: 0,
+    type: 1,
     checkpoint: 0,
     batch: { start: 0, end: 0 },
     previous: seq > 1 ? { core: 0, seq: seq - 1 } : null,
+    metadata: null,
     tree: [],
-    data: node ? [{ key: node.key, value: node.value }] : [],
-    cores: null
+    keys: node ? [{ key: node.key, value: node.value, valuePointer: null }] : [],
+    values: [],
+    cohorts: []
   }
 
   for (const lvl of index.levels) {
     const t = { keys: [], children: [] }
 
-    for (let i = 0; i < lvl.keys.length; i++) {
-      const seq = lvl.keys[i]
-      t.keys.push({ core: 0, seq, offset: 0 })
+    if (lvl.keys.length) {
+      const cohort = []
+      const offset = morphed.cohorts.push(cohort) - 1
+
+      for (let i = 0; i < lvl.keys.length; i++) {
+        const seq = lvl.keys[i]
+        cohort.push({ type: OP_INSERT, index: i, pointer: { core: 0, seq, offset: 0 } })
+      }
+
+      t.keys.push({ type: OP_COHORT, index: 0, pointer: { core: 0, seq, offset } })
     }
-    for (let i = 0; i < lvl.children.length; i += 2) {
-      const seq = lvl.children[i]
-      const offset = lvl.children[i + 1]
-      t.children.push({ core: 0, seq, offset })
+
+    if (lvl.children.length) {
+      const cohort = []
+      const offset = morphed.cohorts.push(cohort) - 1
+
+      for (let i = 0; i < lvl.children.length; i += 2) {
+        const seq = lvl.children[i]
+        const offset = lvl.children[i + 1]
+        cohort.push({ type: OP_INSERT, index: i / 2, pointer: { core: 0, seq, offset } })
+      }
+
+      t.children.push({ type: OP_COHORT, index: 0, pointer: { core: 0, seq, offset } })
     }
 
     morphed.tree.push(t)
@@ -46,8 +64,16 @@ function encodeCompat(node) {
   if (node.tree) {
     for (const t of node.tree) {
       const lvl = { keys: [], children: [] }
-      for (const k of t.keys) lvl.keys.push(k.seq)
-      for (const c of t.children) lvl.children.push(c.seq, c.offset)
+      for (const k of t.keys) {
+        for (const kk of node.cohorts[k.pointer.offset]) {
+          lvl.keys.push(kk.pointer.seq)
+        }
+      }
+      for (const c of t.children) {
+        for (const cc of node.cohorts[c.pointer.offset]) {
+          lvl.children.push(cc.pointer.seq, cc.pointer.offset)
+        }
+      }
       index.levels.push(lvl)
     }
   }
@@ -56,8 +82,8 @@ function encodeCompat(node) {
   encodeYoloIndex(index, bufIndex, 0)
 
   const n = {
-    key: node.data[0].key,
-    value: node.data[0].value,
+    key: node.keys[0].key,
+    value: node.keys[0].value,
     index: bufIndex
   }
 
@@ -65,28 +91,6 @@ function encodeCompat(node) {
   return encodeNode(n, buf, 0)
 }
 
-function deflateIndex(index) {
-  const levels = index.map((l) => {
-    const keys = []
-    const children = []
-
-    for (let i = 0; i < l.value.keys.length; i++) {
-      keys.push(l.value.keys[i].seq)
-    }
-
-    for (let i = 0; i < l.value.children.length; i++) {
-      children.push(l.value.children[i].seq, l.value.children[i].offset)
-    }
-
-    return { keys, children }
-  })
-
-  const idx = { levels }
-  const buf = b4a.allocUnsafe(encodingLengthYoloIndex(idx))
-
-  return encodeYoloIndex(idx, buf, 0)
-}
-
 function decodeLevel(buf, offset, end) {
   if (!offset) offset = 0
   if (!end) end = buf.length
@@ -141,7 +145,7 @@ function encodingLengthLevel(obj) {
     let packedLen = 0
     for (let i = 0; i < obj.keys.length; i++) {
       if (!obj.keys[i]) continue
-      let len = encodings.varint.encodingLength(obj.keys[i])
+      const len = encodings.varint.encodingLength(obj.keys[i])
       packedLen += len
     }
     if (packedLen) {
@@ -152,7 +156,7 @@ function encodingLengthLevel(obj) {
     let packedLen = 0
     for (let i = 0; i < obj.children.length; i++) {
       if (!obj.children[i]) continue
-      let len = encodings.varint.encodingLength(obj.children[i])
+      const len = encodings.varint.encodingLength(obj.children[i])
       packedLen += len
     }
     if (packedLen) {
@@ -164,7 +168,7 @@ function encodingLengthLevel(obj) {
 
 function encodeLevel(obj, buf, offset) {
   if (!offset) offset = 0
-  let oldOffset = offset
+  const oldOffset = offset
   if (obj.keys) {
     let packedLen = 0
     for (let i = 0; i < obj.keys.length; i++) {
@@ -239,7 +243,7 @@ function encodingLengthYoloIndex(obj) {
   if (obj.levels) {
     for (let i = 0; i < obj.levels.length; i++) {
       if (!obj.levels[i]) continue
-      let len = encodingLengthLevel(obj.levels[i])
+      const len = encodingLengthLevel(obj.levels[i])
       length += varint.encodingLength(len)
       length += 1 + len
     }
@@ -249,7 +253,7 @@ function encodingLengthYoloIndex(obj) {
 
 function encodeYoloIndex(obj, buf, offset) {
   if (!offset) offset = 0
-  let oldOffset = offset
+  const oldOffset = offset
   if (obj.levels) {
     for (let i = 0; i < obj.levels.length; i++) {
       if (!obj.levels[i]) continue
@@ -318,7 +322,7 @@ function encodingLengthNode(obj) {
   len = encodings.bytes.encodingLength(obj.key)
   length += 1 + len
   if (obj.value) {
-    let len = encodings.bytes.encodingLength(obj.value)
+    const len = encodings.bytes.encodingLength(obj.value)
     length += 1 + len
   }
   return length
@@ -326,7 +330,7 @@ function encodingLengthNode(obj) {
 
 function encodeNode(obj, buf, offset) {
   if (!offset) offset = 0
-  let oldOffset = offset
+  const oldOffset = offset
   buf[offset++] = 10
   encodings.bytes.encode(obj.index, buf, offset)
   offset += encodings.bytes.encode.bytes

package/lib/compression.js

@@ -0,0 +1,163 @@
+const OP_SET = 0
+const OP_INSERT = 1
+const OP_DEL = 2
+const OP_COHORT = 3
+
+class DeltaOp {
+  constructor(changed, type, index, pointer) {
+    this.type = type
+    this.index = index
+    this.pointer = pointer
+    this.changed = changed
+  }
+}
+
+class DeltaCohort extends DeltaOp {
+  constructor(changed, pointer, deltas) {
+    super(changed, OP_COHORT, 0, pointer)
+    this.deltas = deltas
+  }
+}
+
+class CompressedArray {
+  constructor(delta) {
+    this.entries = []
+    this.delta = delta
+
+    for (const d of delta) {
+      if (d.type === OP_COHORT) {
+        for (const dd of d.deltas) {
+          apply(this.entries, dd.type, dd.index, dd.pointer)
+        }
+      } else {
+        apply(this.entries, d.type, d.index, d.pointer)
+      }
+    }
+  }
+
+  get length() {
+    return this.entries.length
+  }
+
+  touch(index) {
+    const pointer = this.entries[index]
+    if (pointer.changedBy === this) return
+    this.set(index, pointer)
+  }
+
+  get(index) {
+    return this.entries[index]
+  }
+
+  push(pointer) {
+    this.insert(this.entries.length, pointer)
+  }
+
+  unshift(pointer) {
+    this.insert(0, pointer)
+  }
+
+  pop() {
+    if (this.entries.length === 0) return
+    const head = this.entries[this.entries.length - 1]
+    this.delete(this.entries.length - 1)
+    return head
+  }
+
+  shift() {
+    if (this.entries.length === 0) return
+    const tail = this.entries[0]
+    this.delete(0)
+    return tail
+  }
+
+  _touch(pointer) {
+    if (pointer) pointer.changedBy = this
+  }
+
+  insert(index, pointer) {
+    if (!insert(this.entries, index, pointer)) return
+    this._touch(pointer)
+    this.delta.push(new DeltaOp(true, OP_INSERT, index, pointer))
+  }
+
+  delete(index) {
+    if (!del(this.entries, index)) return
+    this._touch(null)
+    this.delta.push(new DeltaOp(true, OP_DEL, index, null))
+  }
+
+  set(index, pointer) {
+    if (!set(this.entries, index, pointer)) return
+    this._touch(pointer)
+    this.delta.push(new DeltaOp(true, OP_SET, index, pointer))
+  }
+
+  flush(max, min) {
+    if (this.delta.length <= max) return this.delta
+
+    const direct = []
+    while (this.delta.length && this.delta[this.delta.length - 1].type !== OP_COHORT) {
+      direct.push(this.delta.pop())
+    }
+    direct.reverse()
+
+    if (direct.length > min && direct.length < this.entries.length) {
+      const co = new DeltaCohort(true, null, [])
+      for (const d of direct) {
+        co.deltas.push(d)
+      }
+      this.delta.push(co)
+    } else {
+      const co = new DeltaCohort(true, null, [])
+      for (let i = 0; i < this.entries.length; i++) {
+        const d = new DeltaOp(true, OP_INSERT, i, this.entries[i])
+        co.deltas.push(d)
+      }
+      this.delta = [co]
+    }
+
+    return this.delta
+  }
+}
+
+exports.CompressedArray = CompressedArray
+exports.DeltaOp = DeltaOp
+exports.DeltaCohort = DeltaCohort
+
+exports.OP_SET = OP_SET
+exports.OP_INSERT = OP_INSERT
+exports.OP_DEL = OP_DEL
+exports.OP_COHORT = OP_COHORT
+
+function del(entries, index) {
+  if (index >= entries.length) return false
+  entries.splice(index, 1)
+  return true
+}
+
+function insert(entries, index, pointer) {
+  if (index >= entries.length + 1) return false
+  entries.splice(index, 0, pointer)
+  return true
+}
+
+function set(entries, index, pointer) {
+  if (index >= entries.length) return false
+  // if (entries[index] === pointer) return false
+  entries[index] = pointer
+  return true
+}
+
+function apply(entries, type, index, pointer) {
+  if (type === OP_INSERT) {
+    return insert(entries, index, pointer)
+  }
+  if (type === OP_DEL) {
+    return del(entries, index)
+  }
+  if (type === OP_SET) {
+    return set(entries, index, pointer)
+  }
+  return false
+}