hyperbee2 1.1.2 → 2.0.0

package/lib/compression.js

@@ -0,0 +1,163 @@
+const OP_SET = 0
+const OP_INSERT = 1
+const OP_DEL = 2
+const OP_COHORT = 3
+
+class DeltaOp {
+  constructor(changed, type, index, pointer) {
+    this.type = type
+    this.index = index
+    this.pointer = pointer
+    this.changed = changed
+  }
+}
+
+class DeltaCohort extends DeltaOp {
+  constructor(changed, pointer, deltas) {
+    super(changed, OP_COHORT, 0, pointer)
+    this.deltas = deltas
+  }
+}
+
+class CompressedArray {
+  constructor(delta) {
+    this.entries = []
+    this.delta = delta
+
+    for (const d of delta) {
+      if (d.type === OP_COHORT) {
+        for (const dd of d.deltas) {
+          apply(this.entries, dd.type, dd.index, dd.pointer)
+        }
+      } else {
+        apply(this.entries, d.type, d.index, d.pointer)
+      }
+    }
+  }
+
+  get length() {
+    return this.entries.length
+  }
+
+  touch(index) {
+    const pointer = this.entries[index]
+    if (pointer.changedBy === this) return
+    this.set(index, pointer)
+  }
+
+  get(index) {
+    return this.entries[index]
+  }
+
+  push(pointer) {
+    this.insert(this.entries.length, pointer)
+  }
+
+  unshift(pointer) {
+    this.insert(0, pointer)
+  }
+
+  pop() {
+    if (this.entries.length === 0) return
+    const head = this.entries[this.entries.length - 1]
+    this.delete(this.entries.length - 1)
+    return head
+  }
+
+  shift() {
+    if (this.entries.length === 0) return
+    const tail = this.entries[0]
+    this.delete(0)
+    return tail
+  }
+
+  _touch(pointer) {
+    if (pointer) pointer.changedBy = this
+  }
+
+  insert(index, pointer) {
+    if (!insert(this.entries, index, pointer)) return
+    this._touch(pointer)
+    this.delta.push(new DeltaOp(true, OP_INSERT, index, pointer))
+  }
+
+  delete(index) {
+    if (!del(this.entries, index)) return
+    this._touch(null)
+    this.delta.push(new DeltaOp(true, OP_DEL, index, null))
+  }
+
+  set(index, pointer) {
+    if (!set(this.entries, index, pointer)) return
+    this._touch(pointer)
+    this.delta.push(new DeltaOp(true, OP_SET, index, pointer))
+  }
+
+  flush(max, min) {
+    if (this.delta.length <= max) return this.delta
+
+    const direct = []
+    while (this.delta.length && this.delta[this.delta.length - 1].type !== OP_COHORT) {
+      direct.push(this.delta.pop())
+    }
+    direct.reverse()
+
+    if (direct.length > min && direct.length < this.entries.length) {
+      const co = new DeltaCohort(true, null, [])
+      for (const d of direct) {
+        co.deltas.push(d)
+      }
+      this.delta.push(co)
+    } else {
+      const co = new DeltaCohort(true, null, [])
+      for (let i = 0; i < this.entries.length; i++) {
+        const d = new DeltaOp(true, OP_INSERT, i, this.entries[i])
+        co.deltas.push(d)
+      }
+      this.delta = [co]
+    }
+
+    return this.delta
+  }
+}
+
+exports.CompressedArray = CompressedArray
+exports.DeltaOp = DeltaOp
+exports.DeltaCohort = DeltaCohort
+
+exports.OP_SET = OP_SET
+exports.OP_INSERT = OP_INSERT
+exports.OP_DEL = OP_DEL
+exports.OP_COHORT = OP_COHORT
+
+function del(entries, index) {
+  if (index >= entries.length) return false
+  entries.splice(index, 1)
+  return true
+}
+
+function insert(entries, index, pointer) {
+  if (index >= entries.length + 1) return false
+  entries.splice(index, 0, pointer)
+  return true
+}
+
+function set(entries, index, pointer) {
+  if (index >= entries.length) return false
+  // if (entries[index] === pointer) return false
+  entries[index] = pointer
+  return true
+}
+
+function apply(entries, type, index, pointer) {
+  if (type === OP_INSERT) {
+    return insert(entries, index, pointer)
+  }
+  if (type === OP_DEL) {
+    return del(entries, index)
+  }
+  if (type === OP_SET) {
+    return set(entries, index, pointer)
+  }
+  return false
+}

package/lib/context.js

@@ -3,10 +3,13 @@ const ScopeLock = require('scope-lock')
 const { decodeBlock } = require('./encoding.js')
 
 class CoreContext {
-  constructor(store, local, core, encryption, lock = new ScopeLock(), other = new Map()) {
+  constructor(store, local, core, encryption, t, lock = new ScopeLock(), other = new Map()) {
     this.store = store
     this.local = local
     this.core = core
+    this.t = t
+    this.minKeys = t - 1
+    this.maxKeys = 2 * t - 1
     this.encryption = encryption
     this.lock = lock
     this.other = other
@@ -17,6 +20,11 @@ class CoreContext {
     this.changed = false
   }
 
+  // TODO: remove, left here for easier debugging for now
+  // [Symbol.for('nodejs.util.inspect.custom')]() {
+  //   return '[CoreContext]'
+  // }
+
   async update(activeRequests) {
     await this.core.ready()
 
@@ -34,7 +42,7 @@ class CoreContext {
 
       if (length < this.length) return
 
-      this.cores = block.cores || []
+      this.cores = block.metadata ? block.metadata.cores : []
       while (this.opened.length < this.cores.length) this.opened.push(null)
     }
 
@@ -44,8 +52,9 @@ class CoreContext {
     this.length = length
   }
 
-  async getCoreOffset(context, core, activeRequests) {
-    if (core !== 0 && core - 1 >= context.cores.length) await context.update(activeRequests)
+  getCoreOffsetLocal(context, core) {
+    if (context === this) return core
+
     const key = core === 0 ? context.core.key : context.cores[core - 1].key
 
     if (b4a.equals(key, this.core.key)) return 0
@@ -60,9 +69,16 @@ class CoreContext {
 
     this.changed = true
     this.cores.push({ key, fork: 0, length: 0, treeHash: null })
+    this.opened.push(null)
+
     return this.cores.length
   }
 
+  async getCoreOffset(context, core, activeRequests) {
+    if (core !== 0 && core - 1 >= context.cores.length) await context.update(activeRequests)
+    return this.getCoreOffsetLocal(context, core)
+  }
+
   async getCoreOffsetByKey(key, activeRequests) {
     await this.core.ready()
 
@@ -86,6 +102,7 @@ class CoreContext {
 
     this.changed = true
     this.cores.push({ key, fork: 0, length: 0, treeHash: null })
+    this.opened.push(null)
     return this.cores.length
   }
 
@@ -99,8 +116,17 @@ class CoreContext {
     return this.opened[index - 1]
   }
 
+  getCoreKey(index) {
+    if (index === 0) return this.core.key
+    if (index > this.cores.length) throw new Error('Bad core index: ' + index)
+    if (this.opened[index - 1] !== null && this.opened[index - 1].key) {
+      return this.opened[index - 1].key
+    }
+    return this.cores[index - 1].key
+  }
+
   async getBlock(seq, core, activeRequests) {
-    if (core !== 0) await this.update(activeRequests)
+    if (core !== 0 && core - 1 >= this.cores.length) await this.update(activeRequests)
     const hc = this.getCore(core)
     const buffer = await hc.get(seq, { activeRequests })
     const block = decodeBlock(buffer, seq)
@@ -117,8 +143,7 @@ class CoreContext {
     const hex = b4a.toString(key, 'hex')
     if (this.other.has(hex)) return this.other.get(hex)
 
-    const hc = this.store.get({ key, encryption: this.encryption })
-    const ctx = new CoreContext(this.store, this.local, hc, this.encryption, this.lock, this.other)
+    const ctx = this._createContext(this.store.get({ key, encryption: this.encryption }))
     this.other.set(hex, ctx)
     return ctx
   }
@@ -131,11 +156,27 @@ class CoreContext {
     const hex = b4a.toString(this.cores[core - 1].key, 'hex')
     if (this.other.has(hex)) return this.other.get(hex)
 
-    const hc = this.getCore(core)
-    const ctx = new CoreContext(this.store, this.local, hc, this.encryption, this.lock, this.other)
+    const ctx = this._createContext(this.getCore(core))
     this.other.set(hex, ctx)
     return ctx
   }
+
+  flush() {
+    this.changed = false
+    return {
+      cores: this.cores
+    }
+  }
+
+  _createContext(core) {
+    const store = this.store
+    const local = this.local
+    const encryption = this.encryption
+    const t = this.t
+    const lock = this.lock
+    const other = this.other
+    return new CoreContext(store, local, core, encryption, t, lock, other)
+  }
 }
 
 module.exports = CoreContext

package/lib/diff.js

@@ -0,0 +1,142 @@
+const { Readable } = require('streamx')
+const b4a = require('b4a')
+const { RangeIterator } = require('./ranges.js')
+
+class DiffIterator {
+  constructor(tree, left, right, { limit = -1, activeRequests = [] } = {}) {
+    this.tree = tree
+
+    this.left = left
+    this.right = right
+
+    this.left.limit = this.right.limit = -1
+
+    this.activeRequests = activeRequests
+    this.limit = limit
+
+    this.nextLeft = null
+    this.nextRight = null
+  }
+
+  async open() {
+    await Promise.all([this.left.open(), this.right.open()])
+  }
+
+  async _shiftRight() {
+    const right = await this.tree.finalizeKeyPointer(this.nextRight, this.activeRequests)
+    this.nextRight = null
+    if (this.limit > 0) this.limit--
+    return { left: null, right }
+  }
+
+  async _shiftLeft() {
+    const left = await this.tree.finalizeKeyPointer(this.nextLeft, this.activeRequests)
+    this.nextLeft = null
+    if (this.limit > 0) this.limit--
+    return { left, right: null }
+  }
+
+  _fastForward() {
+    while (this.left.stack.length && this.right.stack.length) {
+      const a = this.left.stack[this.left.stack.length - 1]
+      const b = this.right.stack[this.right.stack.length - 1]
+
+      if (a.offset !== b.offset) return
+      if (!isSame(a.node, b.node)) return
+
+      this.left.stack.pop()
+      this.right.stack.pop()
+    }
+  }
+
+  async next() {
+    while (this.limit === -1 || this.limit > 0) {
+      const leftPromise = this.nextLeft ? null : this.left.nextKey()
+      const rightPromise = this.nextRight ? null : this.right.nextKey()
+
+      const [l, r] = await Promise.all([leftPromise, rightPromise])
+
+      if (leftPromise) this.nextLeft = l
+      if (rightPromise) this.nextRight = r
+
+      if (!this.nextLeft && !this.nextRight) return null
+      if (!this.nextLeft) return this._shiftRight()
+      if (!this.nextRight) return this._shiftLeft()
+
+      const cmp = b4a.compare(this.nextLeft.key, this.nextRight.key)
+
+      if (cmp === 0) {
+        const leftKey = this.nextLeft
+        const rightKey = this.nextRight
+
+        this.nextRight = this.nextLeft = null
+
+        if (isSame(leftKey, rightKey)) {
+          this._fastForward()
+          continue
+        }
+
+        const [left, right] = await Promise.all([
+          this.tree.finalizeKeyPointer(leftKey, this.activeRequests),
+          this.tree.finalizeKeyPointer(rightKey, this.activeRequests)
+        ])
+
+        if (this.limit > 0) this.limit--
+        return { left, right }
+      }
+
+      if (cmp < 0) return this._shiftLeft()
+      return this._shiftRight()
+    }
+
+    return null
+  }
+}
+
+class DiffStream extends Readable {
+  constructor(left, right, options = {}) {
+    const { highWaterMark } = options
+    super({ eagerOpen: true, highWaterMark })
+
+    this.left = left
+    this.right = right
+    this.iterator = new DiffIterator(
+      left,
+      new RangeIterator(left, options),
+      new RangeIterator(right, options),
+      options
+    )
+  }
+
+  async _open(cb) {
+    try {
+      await this.iterator.open()
+    } catch (err) {
+      cb(err)
+      return
+    }
+    cb(null)
+  }
+
+  async _read(cb) {
+    try {
+      this.push(await this.iterator.next())
+    } catch (err) {
+      cb(err)
+      return
+    }
+    cb(null)
+  }
+}
+
+exports.DiffIterator = DiffIterator
+exports.DiffStream = DiffStream
+
+function isSame(a, b) {
+  if (a.seq !== b.seq || a.offset !== b.offset) return false
+
+  const k1 = a.context.getCoreKey(a.core)
+  const k2 = b.context.getCoreKey(b.core)
+
+  return k1 === k2 || b4a.equals(k1, k2) ? true : false
+}

package/lib/encoding.js

@@ -1,14 +1,144 @@
 const c = require('compact-encoding')
 const { getEncoding } = require('../spec/hyperschema')
-const decodeCompatBlock = require('./compat.js')
+const { OP_COHORT, OP_INSERT } = require('./compression.js')
+const compat = require('./compat.js')
 
-const Block = getEncoding('@bee/block')
+const Block0 = getEncoding('@bee/block-0')
+const Block1 = getEncoding('@bee/block-1')
 
-exports.Block = Block
+const TYPE_COMPAT = 10 // compat with bee1, all blocks starts with 0x10
+
+exports.TYPE_COMPAT = TYPE_COMPAT
+exports.TYPE_LATEST = 1
+
+exports.Block = Block1
+exports.encodeBlock = encodeBlock
 exports.decodeBlock = decodeBlock
 
+function encodeBlock(block) {
+  if (block.type === 1) {
+    return c.encode(Block1, block)
+  }
+
+  if (block.type === 0) {
+    return encodeBlock0(block)
+  }
+
+  if (block.type === TYPE_COMPAT) {
+    return compat.encode(block)
+  }
+
+  throw new Error('Unknown block type: ' + block.type)
+}
+
 function decodeBlock(buffer, seq) {
-  const isCompat = buffer.length > 0 && buffer[0] === 0x0a
-  const block = isCompat ? decodeCompatBlock(buffer, seq) : c.decode(Block, buffer)
-  return block
+  const state = { start: 0, end: buffer.byteLength, buffer }
+  const type = state.end > 0 ? c.uint.decode(state) : 0
+
+  state.start = 0
+
+  if (type === 1) {
+    return Block1.decode(state)
+  }
+
+  if (type === 0) {
+    return decodeBlock0(state, seq)
+  }
+
+  if (type === TYPE_COMPAT) {
+    return compat.decode(buffer, seq)
+  }
+
+  throw new Error('Unknown block type: ' + type)
+}
+
+function dataToKey(d) {
+  return { ...d, valuePointer: null }
+}
+
+function keyToData(d) {
+  return { key: d.key, value: d.value }
+}
+
+function decodeBlock0(state, seq) {
+  const cohorts = []
+  const blk = Block0.decode(state)
+  const tree = []
+
+  for (const t of blk.tree) {
+    const next = {
+      keys: toCohort(seq, t.keys, cohorts),
+      children: toCohort(seq, t.children, cohorts)
+    }
+
+    tree.push(next)
+  }
+
+  return {
+    type: 0,
+    checkpoint: blk.checkpoint,
+    batch: blk.batch,
+    previous: blk.previous,
+    metadata: { cores: blk.cores },
+    tree,
+    keys: blk.data.map(dataToKey),
+    values: null,
+    cohorts
+  }
+}
+
+function encodeBlock0(blk) {
+  const tree = []
+
+  for (const t of blk.tree) {
+    tree.push({
+      keys: fromCohort(t.keys, blk.cohorts),
+      children: fromCohort(t.children, blk.cohorts)
+    })
+  }
+
+  return c.encode(Block0, {
+    type: 0,
+    checkpoint: blk.checkpoint,
+    batch: blk.batch,
+    previous: blk.previous,
+    tree,
+    data: blk.keys.map(keyToData),
+    cores: blk.metadata ? blk.metadata.cores : null
+  })
+}
+
+function fromCohort(deltas, cohorts) {
+  if (deltas.length === 0) return []
+  const cohort = cohorts[deltas[0].pointer.offset]
+  const pointers = []
+  for (const d of cohort) pointers.push(d.pointer)
+  return pointers
+}
+
+function toCohort(seq, pointers, cohorts) {
+  if (!pointers.length) return []
+
+  const cohort = []
+  const offset = cohorts.push(cohort) - 1
+
+  for (let i = 0; i < pointers.length; i++) {
+    cohort.push({
+      type: OP_INSERT,
+      index: i,
+      pointer: pointers[i]
+    })
+  }
+
+  return [
+    {
+      type: OP_COHORT,
+      index: 0,
+      pointer: {
+        core: 0,
+        seq,
+        offset
+      }
+    }
+  ]
 }