hyperbee2 0.0.0

package/README.md

@@ -0,0 +1,19 @@
+# hyperbee2
+
+```
+npm install hyperbee2
+```
+
+## Usage
+
+```js
+const Hyperbee2 = require('hyperbee2')
+
+const db = new Hyperbee2(store)
+
+// see tests for more
+```
+
+## License
+
+Apache-2.0

package/index.js

@@ -0,0 +1,240 @@
+const b4a = require('b4a')
+const Hypercore = require('hypercore')
+const KeyValueStream = require('./lib/key-value-stream.js')
+const ChangesStream = require('./lib/changes-stream.js')
+const NodeCache = require('./lib/cache.js')
+const WriteBatch = require('./lib/write.js')
+const CoreContext = require('./lib/context.js')
+const { DataPointer, TreeNode, TreeNodePointer, EMPTY } = require('./lib/tree.js')
+
+class Hyperbee {
+  constructor(store, options = {}) {
+    const {
+      key = null,
+      core = key ? store.get(key) : store.get({ key, name: 'bee' }),
+      context = new CoreContext(store, core),
+      maxCacheSize = 4096,
+      cache = new NodeCache(maxCacheSize),
+      root = null,
+      activeRequests = [],
+      view = false,
+      writable = true,
+      unbatch = 0
+    } = options
+
+    this.store = store
+    this.root = root
+    this.cache = cache
+    this.context = context
+    this.activeRequests = activeRequests
+    this.view = view
+    this.writable = writable
+    this.unbatch = unbatch
+
+    this.ready().catch(noop)
+  }
+
+  head() {
+    if (!this.root) return null
+    if (this.root === EMPTY) return { length: 0, key: this.context.core.key }
+    return { length: this.root.seq + 1, key: this.root.context.core.key }
+  }
+
+  get core() {
+    return this.context.core
+  }
+
+  get opening() {
+    return this.store.opening
+  }
+
+  get opened() {
+    return this.store.opened
+  }
+
+  get closing() {
+    return this.store.closing
+  }
+
+  get closed() {
+    return this.store.closed
+  }
+
+  replicate(...opts) {
+    return this.store.replicate(...opts)
+  }
+
+  _makeView(root, writable, unbatch) {
+    return new Hyperbee(this.store, {
+      context: this.context,
+      cache: this.cache,
+      root,
+      view: true,
+      writable,
+      unbatch
+    })
+  }
+
+  checkout({ length = this.core.length, key = null, writable = false } = {}) {
+    const context = key ? this.context.getContextByKey(key) : this.context
+    const root = length === 0 ? EMPTY : new TreeNodePointer(context, 0, length - 1, 0, false, null)
+    return this._makeView(root, writable, 0)
+  }
+
+  snapshot() {
+    return this._makeView(this.root, false, 0)
+  }
+
+  undo(n) {
+    return this._makeView(this.root, true, n)
+  }
+
+  write(opts) {
+    if (!this.writable) throw new Error('Not writable')
+    return new WriteBatch(this, opts)
+  }
+
+  async ready() {
+    if (!this.core.opened) await this.core.ready()
+    if (this.root) return
+
+    this.root =
+      this.context.core.length === 0
+        ? EMPTY
+        : new TreeNodePointer(this.context, 0, this.core.length - 1, 0, false, null)
+  }
+
+  async close() {
+    if (this.activeRequests.length) Hypercore.clearRequests(this.activeRequests)
+    if (!this.view) await this.store.close()
+  }
+
+  createReadStream(range) {
+    return new KeyValueStream(this, range)
+  }
+
+  createChangesStream(options) {
+    return new ChangesStream(this, options)
+  }
+
+  async peek(range = {}) {
+    const rs = new KeyValueStream(this, { ...range, limit: 1 })
+    let entry = null
+    for await (const data of rs) entry = data
+    return entry
+  }
+
+  bump(ptr) {
+    if (ptr.changed) return ptr.value
+    this.cache.bump(ptr)
+    this.cache.gc()
+    return ptr.value
+  }
+
+  async inflate(ptr, activeRequests = this.activeRequests) {
+    if (ptr.value) {
+      this.bump(ptr)
+      return ptr.value
+    }
+
+    const block = await ptr.context.getBlock(ptr.seq, ptr.core, activeRequests)
+    const context = await ptr.context.getContext(ptr.core, activeRequests)
+    const tree = block.tree[ptr.offset]
+
+    const keys = new Array(tree.keys.length)
+    const children = new Array(tree.children.length)
+
+    for (let i = 0; i < keys.length; i++) {
+      const k = tree.keys[i]
+      const blk =
+        k.seq === ptr.seq && k.core === ptr.core
+          ? block
+          : await context.getBlock(k.seq, k.core, activeRequests)
+      const d = blk.data[k.offset]
+      keys[i] = new DataPointer(context, k.core, k.seq, k.offset, false, d.key, d.value)
+    }
+
+    for (let i = 0; i < children.length; i++) {
+      const c = tree.children[i]
+      children[i] = new TreeNodePointer(context, c.core, c.seq, c.offset, false, null)
+    }
+
+    ptr.value = new TreeNode(keys, children)
+    this.bump(ptr)
+
+    return ptr.value
+  }
+
+  async bootstrap(activeRequests = this.activeRequests) {
+    if (!this.root) await this.ready()
+    if (this.unbatch) await this._rollback(activeRequests)
+    return this.root === EMPTY ? null : this.root
+  }
+
+  async _rollback(activeRequests) {
+    const expected = this.unbatch
+
+    let n = expected
+    let length = this.root === EMPTY ? 0 : this.root.seq + 1
+    let context = this.context
+
+    while (n > 0 && length > 0 && expected === this.unbatch) {
+      const seq = length - 1
+      const blk = await context.getBlock(seq, 0, activeRequests)
+
+      if (!blk.previous) {
+        length = 0
+        break
+      }
+
+      context = await context.getContext(blk.previous.core, activeRequests)
+      length = blk.previous.seq + 1
+      n--
+    }
+
+    if (expected === this.unbatch) {
+      this.context = context
+      this.root = length === 0 ? EMPTY : new TreeNodePointer(context, 0, length - 1, 0, false, null)
+      this.unbatch = 0
+    }
+  }
+
+  update(root = null) {
+    this.root = root
+    this.unbatch = 0
+  }
+
+  async get(key, { activeRequests = this.activeRequests } = {}) {
+    let ptr = await this.bootstrap(activeRequests)
+    if (!ptr) return null
+
+    while (true) {
+      const v = ptr.value ? this.bump(ptr) : await this.inflate(ptr, activeRequests)
+
+      let s = 0
+      let e = v.keys.length
+      let c = 0
+
+      while (s < e) {
+        const mid = (s + e) >> 1
+        const m = v.keys[mid]
+
+        c = b4a.compare(key, m.key)
+
+        if (c === 0) return m
+
+        if (c < 0) e = mid
+        else s = mid + 1
+      }
+
+      if (!v.children.length) return null
+
+      const i = c < 0 ? e : s
+      ptr = v.children[i]
+    }
+  }
+}
+
+module.exports = Hyperbee
+
+function noop() {}

package/lib/cache.js

@@ -0,0 +1,65 @@
+module.exports = class NodeCache {
+  constructor(maxSize) {
+    this.size = 0
+    this.maxSize = maxSize
+    this.latest = null
+  }
+
+  oldest() {
+    return this.latest ? this.latest.next : null
+  }
+
+  gc() {
+    while (this.size > this.maxSize) {
+      const old = this.oldest()
+      if (old.changed) break
+      this.remove(old)
+      old.value = null
+    }
+  }
+
+  bump(node) {
+    if (node === this.latest) return
+
+    if (node.prev) this.remove(node)
+    this.size++
+
+    if (!this.latest) {
+      node.prev = node.next = node
+      this.latest = node
+    } else {
+      node.prev = this.latest
+      node.next = this.latest.next
+      this.latest.next.prev = node
+      this.latest.next = node
+      this.latest = node
+    }
+  }
+
+  remove(node) {
+    if (node.prev) {
+      this.size--
+
+      if (node === this.latest) {
+        this.latest = node.next === node ? null : node.next
+      }
+
+      node.prev.next = node.next
+      node.next.prev = node.prev
+    }
+
+    node.prev = node.next = null
+  }
+
+  *[Symbol.iterator]() {
+    const node = this.latest
+    if (node === null) return
+
+    let next = node
+
+    do {
+      yield next
+      next = next.next
+    } while (node !== next)
+  }
+}

package/lib/changes-stream.js

@@ -0,0 +1,78 @@
+const { Readable } = require('streamx')
+
+module.exports = class ChangesStream extends Readable {
+  constructor(tree, options = {}) {
+    super({ eagerOpen: true })
+
+    const { head = null, activeRequests = tree.activeRequests } = options
+
+    this.tree = tree
+
+    this._head = head
+    this._context = null
+    this._activeRequests = activeRequests
+  }
+
+  async _openp() {
+    await this.tree.bootstrap(this._activeRequests)
+    if (this._head === null) this._head = this.tree.head()
+    if (this._head !== null) this._context = this.tree.context.getContextByKey(this._head.key)
+  }
+
+  async _readp() {
+    if (!this._context || this._head.length === 0) {
+      this.push(null)
+      return
+    }
+
+    const data = {
+      head: this._head,
+      batch: []
+    }
+
+    const seq = this._head.length - 1
+    const blk = await this._context.getBlock(seq, 0, this._activeRequests)
+    const batchStart = seq - blk.batch.start
+    const remaining = new Array(blk.batch.start)
+
+    for (let i = 0; i < remaining.length; i++) {
+      remaining[i] = this._context.getBlock(batchStart + i, 0, this._activeRequests)
+    }
+
+    for (const blk of await Promise.all(remaining)) data.batch.push(blk)
+    data.batch.push(blk)
+
+    this.push(data)
+
+    if (!blk.previous) {
+      this._head = null
+      this._context = null
+      return
+    }
+
+    this._context = await this._context.getContext(blk.previous.core, this._activeRequests)
+    this._head = { key: this._context.core.key, length: blk.previous.seq + 1 }
+  }
+
+  async _open(cb) {
+    try {
+      await this._openp()
+    } catch (err) {
+      cb(err)
+      return
+    }
+
+    cb(null)
+  }
+
+  async _read(cb) {
+    try {
+      await this._readp()
+    } catch (err) {
+      cb(err)
+      return
+    }
+
+    cb(null)
+  }
+}

package/lib/compat.js

@@ -0,0 +1,164 @@
+// ported from the old protocol buffer impl in old bee
+
+const encodings = require('protocol-buffers-encodings')
+const varint = encodings.varint
+const skip = encodings.skip
+
+module.exports = decodeCompat
+
+function decodeCompat(buffer, seq) {
+  const node = decodeNode(buffer)
+  const index = decodeYoloIndex(node.index)
+  const morphed = {
+    type: 0,
+    checkpoint: 0,
+    batch: { start: 0, end: 0 },
+    previous: seq > 1 ? { core: 0, seq: seq - 1 } : null,
+    tree: [],
+    data: [{ key: node.key, value: node.value }],
+    cores: null
+  }
+
+  for (const lvl of index.levels) {
+    const t = { keys: [], children: [] }
+
+    for (let i = 0; i < lvl.keys.length; i++) {
+      const seq = lvl.keys[i]
+      t.keys.push({ core: 0, seq, offset: 0 })
+    }
+    for (let i = 0; i < lvl.children.length; i += 2) {
+      const seq = lvl.children[i]
+      const offset = lvl.children[i + 1]
+      t.children.push({ core: 0, seq, offset })
+    }
+
+    morphed.tree.push(t)
+  }
+
+  return morphed
+}
+
+function decodeLevel(buf, offset, end) {
+  if (!offset) offset = 0
+  if (!end) end = buf.length
+  if (!(end <= buf.length && offset <= buf.length)) throw new Error('Decoded message is not valid')
+
+  const oldOffset = offset
+
+  const obj = {
+    keys: [],
+    children: []
+  }
+
+  while (true) {
+    if (end <= offset) {
+      decodeLevel.bytes = offset - oldOffset
+      return obj
+    }
+    const prefix = varint.decode(buf, offset)
+    offset += varint.decode.bytes
+    const tag = prefix >> 3
+    switch (tag) {
+      case 1: {
+        let packedEnd = varint.decode(buf, offset)
+        offset += varint.decode.bytes
+        packedEnd += offset
+        while (offset < packedEnd) {
+          obj.keys.push(encodings.varint.decode(buf, offset))
+          offset += encodings.varint.decode.bytes
+        }
+        break
+      }
+      case 2: {
+        let packedEnd = varint.decode(buf, offset)
+        offset += varint.decode.bytes
+        packedEnd += offset
+        while (offset < packedEnd) {
+          obj.children.push(encodings.varint.decode(buf, offset))
+          offset += encodings.varint.decode.bytes
+        }
+        break
+      }
+      default: {
+        offset = skip(prefix & 7, buf, offset)
+      }
+    }
+  }
+}
+
+function decodeYoloIndex(buf, offset, end) {
+  if (!offset) offset = 0
+  if (!end) end = buf.length
+  if (!(end <= buf.length && offset <= buf.length)) throw new Error('Decoded message is not valid')
+  const oldOffset = offset
+  const obj = {
+    levels: []
+  }
+  while (true) {
+    if (end <= offset) {
+      decodeYoloIndex.bytes = offset - oldOffset
+      return obj
+    }
+    const prefix = varint.decode(buf, offset)
+    offset += varint.decode.bytes
+    const tag = prefix >> 3
+    switch (tag) {
+      case 1: {
+        const len = varint.decode(buf, offset)
+        offset += varint.decode.bytes
+        obj.levels.push(decodeLevel(buf, offset, offset + len))
+        offset += decodeLevel.bytes
+        break
+      }
+      default: {
+        offset = skip(prefix & 7, buf, offset)
+      }
+    }
+  }
+}
+
+function decodeNode(buf, offset, end) {
+  if (!offset) offset = 0
+  if (!end) end = buf.length
+  if (!(end <= buf.length && offset <= buf.length)) throw new Error('Decoded message is not valid')
+  const oldOffset = offset
+  const obj = {
+    index: null,
+    key: null,
+    value: null
+  }
+  let found0 = false
+  let found1 = false
+  while (true) {
+    if (end <= offset) {
+      if (!found0 || !found1) throw new Error('Decoded message is not valid')
+      decodeNode.bytes = offset - oldOffset
+      return obj
+    }
+    const prefix = varint.decode(buf, offset)
+    offset += varint.decode.bytes
+    const tag = prefix >> 3
+    switch (tag) {
+      case 1: {
+        obj.index = encodings.bytes.decode(buf, offset)
+        offset += encodings.bytes.decode.bytes
+        found0 = true
+        break
+      }
+      case 2: {
+        obj.key = encodings.bytes.decode(buf, offset)
+        offset += encodings.bytes.decode.bytes
+        found1 = true
+        break
+      }
+      case 3: {
+        obj.value = encodings.bytes.decode(buf, offset)
+        offset += encodings.bytes.decode.bytes
+        break
+      }
+      default: {
+        offset = skip(prefix & 7, buf, offset)
+      }
+    }
+  }
+}

package/lib/context.js

@@ -0,0 +1,128 @@
+const b4a = require('b4a')
+const { decodeBlock } = require('./encoding.js')
+
+class CoreContext {
+  constructor(store, core, other = new Map()) {
+    this.store = store
+    this.core = core
+    this.other = other
+    this.length = 0
+    this.checkpoint = 0
+    this.opened = []
+    this.cores = []
+    this.changed = false
+  }
+
+  async update(activeRequests) {
+    await this.core.ready()
+
+    if (this.length === this.core.length || this.core.length === 0) return
+
+    const length = this.core.length
+    const seq = length - 1
+    const buffer = await this.core.get(seq, { activeRequests })
+    const block = decodeBlock(buffer, seq)
+    const checkpoint = block.checkpoint
+
+    if (checkpoint > 0) {
+      const buffer = await this.core.get(checkpoint - 1)
+      const block = decodeBlock(buffer, checkpoint - 1)
+
+      if (length < this.length) return
+
+      this.cores = block.cores || []
+      while (this.opened.length < this.cores.length) this.opened.push(null)
+    }
+
+    if (length < this.length) return
+
+    this.checkpoint = checkpoint
+    this.length = length
+  }
+
+  async getCoreOffset(context, core, activeRequests) {
+    if (core !== 0 && core - 1 >= context.cores.length) await context.update(activeRequests)
+    const key = core === 0 ? context.core.key : context.cores[core - 1]
+
+    if (b4a.equals(key, this.core.key)) return 0
+
+    // TODO: prop use a map...
+    for (let i = 0; i < this.cores.length; i++) {
+      const k = this.cores[i]
+      if (b4a.equals(k, key)) {
+        return i + 1
+      }
+    }
+
+    this.changed = true
+    this.cores.push(key)
+    return this.cores.length
+  }
+
+  async getCoreOffsetByKey(key, activeRequests) {
+    await this.core.ready()
+
+    if (b4a.equals(key, this.core.key)) return 0
+
+    for (let i = 0; i < this.cores.length; i++) {
+      const k = this.cores[i]
+      if (b4a.equals(k, key)) {
+        return i + 1
+      }
+    }
+
+    await this.update(activeRequests)
+
+    for (let i = 0; i < this.cores.length; i++) {
+      const k = this.cores[i]
+      if (b4a.equals(k, key)) {
+        return i + 1
+      }
+    }
+
+    this.changed = true
+    this.cores.push(key)
+    return this.cores.length
+  }
+
+  getCore(index) {
+    if (index === 0) return this.core
+    if (index > this.cores.length) throw new Error('Bad core index: ' + index)
+    if (this.opened[index - 1] === null)
+      this.opened[index - 1] = this.store.get(this.cores[index - 1])
+    return this.opened[index - 1]
+  }
+
+  async getBlock(seq, core, activeRequests) {
+    if (core !== 0) await this.update(activeRequests)
+    const hc = this.getCore(core)
+    const buffer = await hc.get(seq, { activeRequests })
+    const block = decodeBlock(buffer, seq)
+    return block
+  }
+
+  getContextByKey(key) {
+    const hex = b4a.toString(key, 'hex')
+    if (this.other.has(hex)) return this.other.get(hex)
+
+    const ctx = new CoreContext(this.store, this.store.get(key))
+    this.other.set(hex, ctx)
+    return ctx
+  }
+
+  async getContext(core, activeRequests) {
+    if (core === 0) return this
+    if (core > this.cores.length) await this.update(activeRequests)
+    if (core > this.cores.length) throw new Error('Bad core index: ' + core)
+
+    const hex = b4a.toString(this.cores[core - 1], 'hex')
+    if (this.other.has(hex)) return this.other.get(hex)
+
+    const hc = this.getCore(core)
+    const ctx = new CoreContext(this.store, hc)
+    this.other.set(hex, ctx)
+    return ctx
+  }
+}
+
+module.exports = CoreContext

package/lib/encoding.js

@@ -0,0 +1,14 @@
+const c = require('compact-encoding')
+const { getEncoding } = require('../spec/hyperschema')
+const decodeCompatBlock = require('./compat.js')
+
+const Block = getEncoding('@bee/block')
+
+exports.Block = Block
+exports.decodeBlock = decodeBlock
+
+function decodeBlock(buffer, seq) {
+  const isCompat = buffer.length > 0 && buffer[0] === 0x0a
+  const block = isCompat ? decodeCompatBlock(buffer, seq) : c.decode(Block, buffer)
+  return block
+}