hypercore-storage 0.0.40 → 1.0.0

package/lib/block-dependency-stream.js

@@ -0,0 +1,107 @@
+const { Readable, getStreamError } = require('streamx')
+const { core } = require('./keys')
+
+module.exports = class BlockStream extends Readable {
+  constructor (core, db, updates, start, end, reverse) {
+    super()
+
+    this.core = core
+    this.db = db
+    this.updates = updates
+    this.end = end
+    this.reverse = reverse === true
+
+    this._drained = true
+    this._consumed = 0
+    this._stream = null
+    this._oncloseBound = this._onclose.bind(this)
+    this._maybeDrainBound = this._maybeDrain.bind(this)
+
+    this._update()
+  }
+
+  _update () {
+    if (this._consumed > this.core.dependencies.length) return
+
+    const index = this.reverse ? this.core.dependencies.length - this._consumed : this._consumed
+    const offset = index === 0 ? 0 : this.core.dependencies[index - 1].length
+
+    let end = 0
+    let ptr = 0
+
+    if (this._consumed < this.core.dependencies.length) {
+      const dep = this.core.dependencies[this._consumed]
+      end = this.end === -1 ? dep.length : Math.min(this.end, dep.length)
+      ptr = dep.dataPointer
+    } else {
+      end = this.end === -1 ? Infinity : this.end
+      ptr = this.core.dataPointer
+    }
+
+    this._consumed++
+
+    if (end === offset) return
+
+    this._makeStream(core.block(ptr, offset), core.block(ptr, end))
+  }
+
+  _predestroy () {
+    if (this._stream !== null) this._stream.destroy()
+  }
+
+  _read (cb) {
+    this._drained = this._onreadable()
+    cb(null)
+  }
+
+  _maybeDrain () {
+    if (this._drained === true) return
+    this._drained = this._onreadable()
+  }
+
+  _onreadable () {
+    if (this._stream === null) {
+      this.push(null)
+      return true
+    }
+
+    let data = this._stream.read()
+
+    if (data === null) return false
+
+    do {
+      this.push(data)
+      data = this._stream.read()
+    } while (data !== null)
+
+    return true
+  }
+
+  _onclose () {
+    if (this.destroying) return
+
+    const err = getStreamError(this._stream)
+
+    if (err !== null) {
+      this.destroy(err)
+      return
+    }
+
+    // empty the current stream
+    if (this._onreadable() === true) this._drained = true
+
+    this._stream = this._start = this._end = null
+
+    this._update()
+    this._maybeDrain()
+  }
+
+  _makeStream (start, end) {
+    this._stream = this.updates.iterator(this.db, start, end, this.reverse)
+    this._stream.on('readable', this._maybeDrainBound)
+    this._stream.on('error', noop)
+    this._stream.on('close', this._oncloseBound)
+  }
+}
+
+function noop () {}

package/lib/keys.js

@@ -0,0 +1,242 @@
+const { UINT, STRING } = require('index-encoder')
+const c = require('compact-encoding')
+const b4a = require('b4a')
+
+const TL_HEAD = 0
+const TL_CORE_BY_DKEY = 1
+const TL_CORE_BY_ALIAS = 2
+const TL_CORE = 3
+const TL_DATA = 4
+
+const TL_END = TL_DATA + 1
+
+const CORE_AUTH = 0
+const CORE_SESSIONS = 1
+
+const DATA_HEAD = 0
+const DATA_DEPENDENCY = 1
+const DATA_HINTS = 2
+const DATA_BLOCK = 3
+const DATA_TREE = 4
+const DATA_BITFIELD = 5
+const DATA_USER_DATA = 6
+
+const slab = { buffer: b4a.allocUnsafe(65536), start: 0, end: 0 }
+
+const store = {}
+const core = {}
+
+store.clear = function () {
+  const state = alloc()
+  let start = state.start
+  UINT.encode(state, 0)
+  const a = state.buffer.subarray(start, state.start)
+  start = state.start
+  UINT.encode(state, TL_END)
+  const b = state.buffer.subarray(start, state.start)
+  return [a, b]
+}
+
+store.head = function () {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_HEAD)
+  return state.buffer.subarray(start, state.start)
+}
+
+store.core = function (discoveryKey) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_CORE_BY_DKEY)
+  c.fixed32.encode(state, discoveryKey)
+  return state.buffer.subarray(start, state.start)
+}
+
+store.coreStart = function () {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_CORE_BY_DKEY)
+  return state.buffer.subarray(start, state.start)
+}
+
+store.coreEnd = function () {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_CORE_BY_DKEY + 1)
+  return state.buffer.subarray(start, state.start)
+}
+
+store.coreByAlias = function ({ namespace, name }) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_CORE_BY_ALIAS)
+  c.fixed32.encode(state, namespace)
+  STRING.encode(state, name)
+  return state.buffer.subarray(start, state.start)
+}
+
+store.coreByAliasStart = function (namespace) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_CORE_BY_ALIAS)
+  if (namespace) c.fixed32.encode(state, namespace)
+  return state.buffer.subarray(start, state.start)
+}
+
+store.coreByAliasEnd = function (namespace) {
+  const state = alloc()
+  const start = state.start
+
+  if (namespace) {
+    UINT.encode(state, TL_CORE_BY_ALIAS)
+    c.fixed32.encode(state, namespace)
+    state.buffer[state.start++] = 0xff
+  } else {
+    UINT.encode(state, TL_CORE_BY_ALIAS + 1)
+  }
+
+  return state.buffer.subarray(start, state.start)
+}
+
+store.alias = function (buffer) {
+  const state = { buffer, start: 0, end: buffer.byteLength }
+  UINT.decode(state) // ns
+  const namespace = c.fixed32.decode(state)
+  const name = STRING.decode(state)
+  return { namespace, name }
+}
+
+store.discoveryKey = function (buffer) {
+  const state = { buffer, start: 0, end: buffer.byteLength }
+  UINT.decode(state) // ns
+  return c.fixed32.decode(state)
+}
+
+core.auth = function (ptr) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_CORE)
+  UINT.encode(state, ptr)
+  UINT.encode(state, CORE_AUTH)
+  return state.buffer.subarray(start, state.start)
+}
+
+core.sessions = function (ptr) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_CORE)
+  UINT.encode(state, ptr)
+  UINT.encode(state, CORE_SESSIONS)
+  return state.buffer.subarray(start, state.start)
+}
+
+core.head = function (ptr) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_DATA)
+  UINT.encode(state, ptr)
+  UINT.encode(state, DATA_HEAD)
+  return state.buffer.subarray(start, state.start)
+}
+
+core.dependency = function (ptr) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_DATA)
+  UINT.encode(state, ptr)
+  UINT.encode(state, DATA_DEPENDENCY)
+  return state.buffer.subarray(start, state.start)
+}
+
+core.hints = function (ptr) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_DATA)
+  UINT.encode(state, ptr)
+  UINT.encode(state, DATA_HINTS)
+  return state.buffer.subarray(start, state.start)
+}
+
+core.block = function (ptr, index) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_DATA)
+  UINT.encode(state, ptr)
+  UINT.encode(state, DATA_BLOCK)
+  UINT.encode(state, index)
+  return state.buffer.subarray(start, state.start)
+}
+
+core.tree = function (ptr, index) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_DATA)
+  UINT.encode(state, ptr)
+  UINT.encode(state, DATA_TREE)
+  UINT.encode(state, index)
+  return state.buffer.subarray(start, state.start)
+}
+
+core.bitfield = function (ptr, index, type) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_DATA)
+  UINT.encode(state, ptr)
+  UINT.encode(state, DATA_BITFIELD)
+  UINT.encode(state, index)
+  UINT.encode(state, type)
+  return state.buffer.subarray(start, state.start)
+}
+
+core.userData = function (ptr, key) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_DATA)
+  UINT.encode(state, ptr)
+  UINT.encode(state, DATA_USER_DATA)
+  STRING.encode(state, key)
+  return state.buffer.subarray(start, state.start)
+}
+
+core.userDataEnd = function (ptr) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_DATA)
+  UINT.encode(state, ptr)
+  UINT.encode(state, DATA_USER_DATA + 1)
+  return state.buffer.subarray(start, state.start)
+}
+
+core.blockIndex = function (buffer) {
+  const state = { buffer, start: 0, end: buffer.byteLength }
+  UINT.decode(state) // ns
+  UINT.decode(state) // ptr
+  UINT.decode(state) // type
+  return UINT.decode(state)
+}
+
+core.bitfieldIndexAndType = function (buffer) {
+  const state = { buffer, start: 0, end: buffer.byteLength }
+  UINT.decode(state) // ns
+  UINT.decode(state) // ptr
+  UINT.decode(state) // type
+  return [UINT.decode(state), UINT.decode(state)]
+}
+
+core.userDataKey = function (buffer) {
+  const state = { buffer, start: 0, end: buffer.byteLength }
+  UINT.decode(state) // ns
+  UINT.decode(state) // ptr
+  UINT.decode(state) // type
+  return STRING.decode(state)
+}
+
+module.exports = { store, core }
+
+function alloc () {
+  if (slab.buffer.byteLength - slab.start < 4096) {
+    slab.buffer = b4a.allocUnsafe(slab.buffer.byteLength)
+    slab.start = 0
+  }
+  return slab
+}

package/lib/streams.js

@@ -0,0 +1,112 @@
+const BlockDependencyStream = require('./block-dependency-stream.js')
+const { core, store } = require('./keys.js')
+const schema = require('../spec/hyperschema')
+
+const CORESTORE_CORE = schema.getEncoding('@corestore/core')
+const CORE_TREE_NODE = schema.getEncoding('@core/tree-node')
+
+module.exports = {
+  createBlockStream,
+  createBitfieldStream,
+  createUserDataStream,
+  createCoreStream,
+  createAliasStream,
+  createTreeNodeStream
+}
+
+function createCoreStream (db, view) {
+  const start = store.coreStart()
+  const end = store.coreEnd()
+
+  const ite = view.iterator(db, start, end, false)
+
+  ite._readableState.map = mapCore
+  return ite
+}
+
+function createAliasStream (db, view, namespace) {
+  const start = store.coreByAliasStart(namespace)
+  const end = store.coreByAliasEnd(namespace)
+
+  const ite = view.iterator(db, start, end, false)
+
+  ite._readableState.map = mapAlias
+  return ite
+}
+
+function createBlockIterator (ptr, db, view, start, end, reverse) {
+  if (ptr.dependencies.length > 0) {
+    return new BlockDependencyStream(ptr, db, view, start, end, reverse)
+  }
+
+  const s = core.block(ptr.dataPointer, start)
+  const e = core.block(ptr.dataPointer, end === -1 ? Infinity : end)
+  return view.iterator(db, s, e, reverse)
+}
+
+function createBlockStream (ptr, db, view, { gt = -1, gte = gt + 1, lte = -1, lt = lte === -1 ? -1 : lte + 1, reverse = false } = {}) {
+  const ite = createBlockIterator(ptr, db, view, gte, lt, reverse)
+
+  ite._readableState.map = mapBlock
+  return ite
+}
+
+function createBitfieldStream (ptr, db, view, { gt = -1, gte = gt + 1, lte = -1, lt = lte === -1 ? -1 : lte + 1, reverse = false } = {}) {
+  const s = core.bitfield(ptr.dataPointer, gte, 0)
+  const e = core.bitfield(ptr.dataPointer, lt === -1 ? Infinity : lt, 0)
+  const ite = view.iterator(db, s, e, false)
+
+  ite._readableState.map = mapBitfield
+  return ite
+}
+
+// NOTE: this does not do dependency lookups atm
+function createTreeNodeStream (ptr, db, view, { gt = -1, gte = gt + 1, lte = -1, lt = lte === -1 ? -1 : lte + 1, reverse = false } = {}) {
+  const s = core.tree(ptr.dataPointer, gte, 0)
+  const e = core.tree(ptr.dataPointer, lt === -1 ? Infinity : lt, 0)
+  const ite = view.iterator(db, s, e, false)
+
+  ite._readableState.map = mapTreeNode
+  return ite
+}
+
+function createUserDataStream (ptr, db, view, { gt = null, gte = '', lte = null, lt = null, reverse = false } = {}) {
+  if (gt !== null || lte !== null) throw new Error('gt and lte not yet supported for user data streams')
+
+  const s = core.userData(ptr.dataPointer, gte)
+  const e = lt === null ? core.userDataEnd(ptr.dataPointer) : core.userData(ptr.dataPointer, lt)
+  const ite = view.iterator(db, s, e, false)
+
+  ite._readableState.map = mapUserData
+  return ite
+}
+
+function mapBitfield (data) {
+  const [index, type] = core.bitfieldIndexAndType(data.key)
+  if (type !== 0) return null // ignore for now
+  return { index, page: data.value }
+}
+
+function mapUserData (data) {
+  const key = core.userDataKey(data.key)
+  return { key, value: data.value }
+}
+
+function mapCore (data) {
+  const discoveryKey = store.discoveryKey(data.key)
+  const core = CORESTORE_CORE.decode({ start: 0, end: data.value.byteLength, buffer: data.value })
+  return { discoveryKey, core }
+}
+
+function mapAlias (data) {
+  const alias = store.alias(data.key)
+  return { alias, discoveryKey: data.value }
+}
+
+function mapBlock (data) {
+  return { index: core.blockIndex(data.key), value: data.value }
+}
+
+function mapTreeNode (data) {
+  return CORE_TREE_NODE.decode({ start: 0, end: data.value.byteLength, buffer: data.value })
+}