hyperbee2 1.1.2 → 2.0.0

package/index.js

@@ -1,19 +1,29 @@
 const b4a = require('b4a')
 const Hypercore = require('hypercore')
-const KeyValueStream = require('./lib/key-value-stream.js')
-const ChangesStream = require('./lib/changes-stream.js')
+const { RangeStream } = require('./lib/ranges.js')
+const { DiffStream } = require('./lib/diff.js')
+const { ChangesStream } = require('./lib/changes.js')
 const NodeCache = require('./lib/cache.js')
 const WriteBatch = require('./lib/write.js')
 const CoreContext = require('./lib/context.js')
-const { DataPointer, TreeNode, TreeNodePointer, EMPTY } = require('./lib/tree.js')
+const {
+  Pointer,
+  KeyPointer,
+  ValuePointer,
+  TreeNode,
+  TreeNodePointer,
+  EMPTY
+} = require('./lib/tree.js')
+const { DeltaOp, DeltaCohort, OP_COHORT } = require('./lib/compression.js')
 
 class Hyperbee {
   constructor(store, options = {}) {
     const {
+      t = 128, // legacy number for now, should be 128 now
       key = null,
       encryption = null,
       core = key ? store.get(key) : store.get({ key, name: 'bee', encryption }),
-      context = new CoreContext(store, core, core, encryption),
+      context = new CoreContext(store, core, core, encryption, t),
       maxCacheSize = 4096,
       cache = new NodeCache(maxCacheSize),
       root = null,
@@ -110,8 +120,12 @@ class Hyperbee {
     if (!this.view) await this.store.close()
   }
 
-  createReadStream(range) {
-    return new KeyValueStream(this, range)
+  createReadStream(options) {
+    return new RangeStream(this, options)
+  }
+
+  createDiffStream(right, options) {
+    return new DiffStream(this, right, options)
   }
 
   createChangesStream(options) {
@@ -119,12 +133,21 @@ class Hyperbee {
   }
 
   async peek(range = {}) {
-    const rs = new KeyValueStream(this, { ...range, limit: 1 })
+    const rs = new RangeStream(this, { ...range, limit: 1 })
     let entry = null
     for await (const data of rs) entry = data
     return entry
   }
 
+  download(range = {}) {
+    const rs = new RangeStream(this, range)
+    rs.resume()
+    return new Promise((resolve, reject) => {
+      rs.once('error', reject)
+      rs.once('close', resolve)
+    })
+  }
+
   bump(ptr) {
     if (ptr.changed) return ptr.value
     this.cache.bump(ptr)
@@ -132,40 +155,75 @@ class Hyperbee {
     return ptr.value
   }
 
+  // TODO: unslab these and parallize
   async inflate(ptr, activeRequests = this.activeRequests) {
     if (ptr.value) {
       this.bump(ptr)
       return ptr.value
     }
 
-    const block = await ptr.context.getBlock(ptr.seq, ptr.core, activeRequests)
-    const context = await ptr.context.getContext(ptr.core, activeRequests)
+    const [block, context] = await Promise.all([
+      ptr.context.getBlock(ptr.seq, ptr.core, activeRequests),
+      ptr.context.getContext(ptr.core, activeRequests)
+    ])
+
     const tree = block.tree[ptr.offset]
 
     const keys = new Array(tree.keys.length)
     const children = new Array(tree.children.length)
 
     for (let i = 0; i < keys.length; i++) {
-      const k = tree.keys[i]
-      const blk =
-        k.seq === ptr.seq && k.core === 0 && ptr.core === 0
-          ? block
-          : await context.getBlock(k.seq, k.core, activeRequests)
-      const d = blk.data[k.offset]
-      keys[i] = new DataPointer(context, k.core, k.seq, k.offset, false, d.key, d.value)
+      const d = tree.keys[i]
+      keys[i] = inflateKey(context, d, ptr, block, activeRequests)
     }
 
     for (let i = 0; i < children.length; i++) {
-      const c = tree.children[i]
-      children[i] = new TreeNodePointer(context, c.core, c.seq, c.offset, false, null)
+      const d = tree.children[i]
+      children[i] = inflateChild(context, d, ptr, block, activeRequests)
     }
 
-    ptr.value = new TreeNode(keys, children)
+    ptr.value = new TreeNode(await Promise.all(keys), await Promise.all(children))
     this.bump(ptr)
 
     return ptr.value
   }
 
+  async finalizeKeyPointer(key, activeRequests = this.activeRequests) {
+    const value = key.value || (await this.inflateValue(key, activeRequests))
+
+    return {
+      core: key.context.getCore(key.core),
+      offset: key.offset,
+      seq: key.seq,
+      key: key.key,
+      value
+    }
+  }
+
+  async inflateValue(key, activeRequests = this.activeRequests) {
+    if (key.value) return key.value
+    if (!key.valuePointer) return null
+
+    const ptr = key.valuePointer
+
+    if (ptr.split === 0) {
+      const block = await ptr.context.getBlock(ptr.seq, ptr.core, activeRequests)
+      return block.values[ptr.offset]
+    }
+
+    const blockPromises = new Array(ptr.split + 1)
+    for (let i = 0; i < blockPromises.length; i++) {
+      blockPromises[i] = ptr.context.getBlock(ptr.seq - ptr.split + i, ptr.core, activeRequests)
+    }
+    const blocks = await Promise.all(blockPromises)
+    const splitValue = new Array(blockPromises.length)
+    for (let i = 0; i < splitValue.length - 1; i++) {
+      splitValue[i] = blocks[i].values[0]
+    }
+    splitValue[splitValue.length - 1] = blocks[blocks.length - 1].buffer[ptr.offset]
+    return b4a.concat(splitValue)
+  }
+
   async bootstrap(activeRequests = this.activeRequests) {
     if (!this.root) await this.ready()
     if (this.unbatch) await this._rollback(activeRequests)
@@ -218,11 +276,11 @@ class Hyperbee {
 
       while (s < e) {
         const mid = (s + e) >> 1
-        const m = v.keys[mid]
+        const m = v.keys.get(mid)
 
         c = b4a.compare(key, m.key)
 
-        if (c === 0) return m
+        if (c === 0) return this.finalizeKeyPointer(m, activeRequests)
 
         if (c < 0) e = mid
         else s = mid + 1
@@ -231,7 +289,7 @@ class Hyperbee {
       if (!v.children.length) return null
 
       const i = c < 0 ? e : s
-      ptr = v.children[i]
+      ptr = v.children.get(i)
     }
   }
 }
@@ -239,3 +297,84 @@ class Hyperbee {
 module.exports = Hyperbee
 
 function noop() {}
+
+function inflateKey(context, d, ptr, block, activeRequests) {
+  if (d.type === OP_COHORT) return inflateKeyCohort(context, d, ptr, block, activeRequests)
+  return inflateKeyDelta(context, d, ptr, block, activeRequests)
+}
+
+async function inflateKeyDelta(context, d, ptr, block, activeRequests) {
+  const k = d.pointer
+
+  if (!k) return new DeltaOp(false, d.type, d.index, null)
+
+  const blk =
+    k.seq === ptr.seq && k.core === 0 && ptr.core === 0
+      ? block
+      : await context.getBlock(k.seq, k.core, activeRequests)
+
+  const bk = blk.keys[k.offset]
+
+  let vp = null
+
+  if (bk.valuePointer) {
+    const p = bk.valuePointer
+    const ctx = await context.getContext(k.core, activeRequests)
+    vp = new ValuePointer(ctx, p.core, p.seq, p.offset, p.split)
+  }
+
+  const kp = new KeyPointer(context, k.core, k.seq, k.offset, false, bk.key, bk.value, vp)
+  return new DeltaOp(false, d.type, d.index, kp)
+}
+
+async function inflateKeyCohort(context, d, ptr, block, activeRequests) {
+  const co = d.pointer
+
+  const blk =
+    co.seq === ptr.seq && co.core === 0 && ptr.core === 0
+      ? block
+      : await context.getBlock(co.seq, co.core, activeRequests)
+
+  const cohort = blk.cohorts[co.offset]
+  const promises = new Array(cohort.length)
+
+  for (let i = 0; i < cohort.length; i++) {
+    const p = cohort[i]
+    const k = inflateKeyDelta(context, p, co, blk, activeRequests)
+    promises[i] = k
+  }
+
+  const p = new Pointer(context, co.core, co.seq, co.offset)
+  return new DeltaCohort(false, p, await Promise.all(promises))
+}
+
+function inflateChild(context, d, ptr, block, activeRequests) {
+  if (d.type === OP_COHORT) return inflateChildCohort(context, d, ptr, block, activeRequests)
+  return Promise.resolve(inflateChildDelta(context, d, ptr, block, activeRequests))
+}
+
+function inflateChildDelta(context, d, ptr, block, activeRequests) {
+  const p = d.pointer
+  const c = p && new TreeNodePointer(context, p.core, p.seq, p.offset, false, null)
+  return new DeltaOp(false, d.type, d.index, c)
+}
+
+async function inflateChildCohort(context, d, ptr, block, activeRequests) {
+  const co = d.pointer
+
+  const blk =
+    co.seq === ptr.seq && co.core === 0 && ptr.core === 0
+      ? block
+      : await context.getBlock(co.seq, co.core, activeRequests)
+
+  const cohort = blk.cohorts[co.offset]
+  const deltas = new Array(cohort.length)
+
+  for (let i = 0; i < cohort.length; i++) {
+    const c = cohort[i]
+    deltas[i] = inflateChildDelta(context, c, co, blk, activeRequests)
+  }
+
+  const p = new Pointer(context, co.core, co.seq, co.offset)
+  return new DeltaCohort(false, p, deltas)
+}

package/lib/cache.js

@@ -9,6 +9,13 @@ module.exports = class NodeCache {
     return this.latest ? this.latest.next : null
   }
 
+  empty() {
+    const maxSize = this.maxSize
+    this.maxSize = 0
+    this.gc()
+    this.maxSize = maxSize
+  }
+
   gc() {
     while (this.size > this.maxSize) {
       const old = this.oldest()

package/lib/{changes-stream.js → changes.js}

@@ -1,8 +1,9 @@
 const { Readable } = require('streamx')
 
-module.exports = class ChangesStream extends Readable {
+class ChangesStream extends Readable {
   constructor(tree, options = {}) {
-    super({ eagerOpen: true })
+    const { highWaterMark } = options
+    super({ eagerOpen: true, highWaterMark })
 
     const { head = null, activeRequests = tree.activeRequests } = options
 
@@ -76,3 +77,5 @@ module.exports = class ChangesStream extends Readable {
     cb(null)
   }
 }
+
+exports.ChangesStream = ChangesStream

package/lib/compat.js

@@ -1,35 +1,55 @@
 // ported from the old protocol buffer impl in old bee
 
 const encodings = require('protocol-buffers-encodings')
+const b4a = require('b4a')
+const { OP_COHORT, OP_INSERT } = require('./compression')
 const varint = encodings.varint
 const skip = encodings.skip
 
-module.exports = decodeCompat
+exports.decode = decodeCompat
+exports.encode = encodeCompat
 
 function decodeCompat(buffer, seq) {
-  const node = decodeNode(buffer)
-  const index = decodeYoloIndex(node.index)
+  const node = buffer.length > 1 ? decodeNode(buffer) : null
+  const index = node ? decodeYoloIndex(node.index) : { levels: [] }
   const morphed = {
-    type: 0,
+    type: 1,
     checkpoint: 0,
     batch: { start: 0, end: 0 },
     previous: seq > 1 ? { core: 0, seq: seq - 1 } : null,
+    metadata: null,
     tree: [],
-    data: [{ key: node.key, value: node.value }],
-    cores: null
+    keys: node ? [{ key: node.key, value: node.value, valuePointer: null }] : [],
+    values: [],
+    cohorts: []
   }
 
   for (const lvl of index.levels) {
     const t = { keys: [], children: [] }
 
-    for (let i = 0; i < lvl.keys.length; i++) {
-      const seq = lvl.keys[i]
-      t.keys.push({ core: 0, seq, offset: 0 })
+    if (lvl.keys.length) {
+      const cohort = []
+      const offset = morphed.cohorts.push(cohort) - 1
+
+      for (let i = 0; i < lvl.keys.length; i++) {
+        const seq = lvl.keys[i]
+        cohort.push({ type: OP_INSERT, index: i, pointer: { core: 0, seq, offset: 0 } })
+      }
+
+      t.keys.push({ type: OP_COHORT, index: 0, pointer: { core: 0, seq, offset } })
     }
-    for (let i = 0; i < lvl.children.length; i += 2) {
-      const seq = lvl.children[i]
-      const offset = lvl.children[i + 1]
-      t.children.push({ core: 0, seq, offset })
+
+    if (lvl.children.length) {
+      const cohort = []
+      const offset = morphed.cohorts.push(cohort) - 1
+
+      for (let i = 0; i < lvl.children.length; i += 2) {
+        const seq = lvl.children[i]
+        const offset = lvl.children[i + 1]
+        cohort.push({ type: OP_INSERT, index: i / 2, pointer: { core: 0, seq, offset } })
+      }
+
+      t.children.push({ type: OP_COHORT, index: 0, pointer: { core: 0, seq, offset } })
     }
 
     morphed.tree.push(t)
@@ -38,6 +58,39 @@ function decodeCompat(buffer, seq) {
   return morphed
 }
 
+function encodeCompat(node) {
+  const index = { levels: [] }
+
+  if (node.tree) {
+    for (const t of node.tree) {
+      const lvl = { keys: [], children: [] }
+      for (const k of t.keys) {
+        for (const kk of node.cohorts[k.pointer.offset]) {
+          lvl.keys.push(kk.pointer.seq)
+        }
+      }
+      for (const c of t.children) {
+        for (const cc of node.cohorts[c.pointer.offset]) {
+          lvl.children.push(cc.pointer.seq, cc.pointer.offset)
+        }
+      }
+      index.levels.push(lvl)
+    }
+  }
+
+  const bufIndex = b4a.allocUnsafe(encodingLengthYoloIndex(index))
+  encodeYoloIndex(index, bufIndex, 0)
+
+  const n = {
+    key: node.keys[0].key,
+    value: node.keys[0].value,
+    index: bufIndex
+  }
+
+  const buf = b4a.allocUnsafe(encodingLengthNode(n))
+  return encodeNode(n, buf, 0)
+}
+
 function decodeLevel(buf, offset, end) {
   if (!offset) offset = 0
   if (!end) end = buf.length
@@ -86,6 +139,74 @@ function decodeLevel(buf, offset, end) {
   }
 }
 
+function encodingLengthLevel(obj) {
+  let length = 0
+  if (obj.keys) {
+    let packedLen = 0
+    for (let i = 0; i < obj.keys.length; i++) {
+      if (!obj.keys[i]) continue
+      const len = encodings.varint.encodingLength(obj.keys[i])
+      packedLen += len
+    }
+    if (packedLen) {
+      length += 1 + packedLen + varint.encodingLength(packedLen)
+    }
+  }
+  if (obj.children) {
+    let packedLen = 0
+    for (let i = 0; i < obj.children.length; i++) {
+      if (!obj.children[i]) continue
+      const len = encodings.varint.encodingLength(obj.children[i])
+      packedLen += len
+    }
+    if (packedLen) {
+      length += 1 + packedLen + varint.encodingLength(packedLen)
+    }
+  }
+  return length
+}
+
+function encodeLevel(obj, buf, offset) {
+  if (!offset) offset = 0
+  const oldOffset = offset
+  if (obj.keys) {
+    let packedLen = 0
+    for (let i = 0; i < obj.keys.length; i++) {
+      if (!obj.keys[i]) continue
+      packedLen += encodings.varint.encodingLength(obj.keys[i])
+    }
+    if (packedLen) {
+      buf[offset++] = 10
+      varint.encode(packedLen, buf, offset)
+      offset += varint.encode.bytes
+    }
+    for (let i = 0; i < obj.keys.length; i++) {
+      if (!obj.keys[i]) continue
+      encodings.varint.encode(obj.keys[i], buf, offset)
+      offset += encodings.varint.encode.bytes
+    }
+  }
+  if (obj.children) {
+    let packedLen = 0
+    for (let i = 0; i < obj.children.length; i++) {
+      if (!obj.children[i]) continue
+      packedLen += encodings.varint.encodingLength(obj.children[i])
+    }
+    if (packedLen) {
+      buf[offset++] = 18
+      varint.encode(packedLen, buf, offset)
+      offset += varint.encode.bytes
+    }
+    for (let i = 0; i < obj.children.length; i++) {
+      if (!obj.children[i]) continue
+      encodings.varint.encode(obj.children[i], buf, offset)
+      offset += encodings.varint.encode.bytes
+    }
+  }
+  encodeLevel.bytes = offset - oldOffset
+  return buf
+}
+
 function decodeYoloIndex(buf, offset, end) {
   if (!offset) offset = 0
   if (!end) end = buf.length
@@ -117,6 +238,37 @@ function decodeYoloIndex(buf, offset, end) {
   }
 }
 
+function encodingLengthYoloIndex(obj) {
+  let length = 0
+  if (obj.levels) {
+    for (let i = 0; i < obj.levels.length; i++) {
+      if (!obj.levels[i]) continue
+      const len = encodingLengthLevel(obj.levels[i])
+      length += varint.encodingLength(len)
+      length += 1 + len
+    }
+  }
+  return length
+}
+
+function encodeYoloIndex(obj, buf, offset) {
+  if (!offset) offset = 0
+  const oldOffset = offset
+  if (obj.levels) {
+    for (let i = 0; i < obj.levels.length; i++) {
+      if (!obj.levels[i]) continue
+      buf[offset++] = 10
+      const len = encodingLengthLevel(obj.levels[i])
+      varint.encode(len, buf, offset)
+      offset += varint.encode.bytes
+      encodeLevel(obj.levels[i], buf, offset)
+      offset += encodeLevel.bytes
+    }
+  }
+  encodeYoloIndex.bytes = offset - oldOffset
+  return buf
+}
+
 function decodeNode(buf, offset, end) {
   if (!offset) offset = 0
   if (!end) end = buf.length
@@ -162,3 +314,34 @@ function decodeNode(buf, offset, end) {
     }
   }
 }
+
+function encodingLengthNode(obj) {
+  let length = 0
+  let len = encodings.bytes.encodingLength(obj.index)
+  length += 1 + len
+  len = encodings.bytes.encodingLength(obj.key)
+  length += 1 + len
+  if (obj.value) {
+    const len = encodings.bytes.encodingLength(obj.value)
+    length += 1 + len
+  }
+  return length
+}
+
+function encodeNode(obj, buf, offset) {
+  if (!offset) offset = 0
+  const oldOffset = offset
+  buf[offset++] = 10
+  encodings.bytes.encode(obj.index, buf, offset)
+  offset += encodings.bytes.encode.bytes
+  buf[offset++] = 18
+  encodings.bytes.encode(obj.key, buf, offset)
+  offset += encodings.bytes.encode.bytes
+  if (obj.value) {
+    buf[offset++] = 26
+    encodings.bytes.encode(obj.value, buf, offset)
+    offset += encodings.bytes.encode.bytes
+  }
+  encodeNode.bytes = offset - oldOffset
+  return buf
+}