hypercore-storage 0.0.41 → 1.0.1

package/lib/tx.js

@@ -0,0 +1,286 @@
+const schema = require('../spec/hyperschema')
+const { store, core } = require('./keys.js')
+const View = require('./view.js')
+const b4a = require('b4a')
+const flat = require('flat-tree')
+
+const CORESTORE_HEAD = schema.getEncoding('@corestore/head')
+const CORESTORE_CORE = schema.getEncoding('@corestore/core')
+
+const CORE_AUTH = schema.getEncoding('@core/auth')
+const CORE_SESSIONS = schema.getEncoding('@core/sessions')
+const CORE_HEAD = schema.getEncoding('@core/head')
+const CORE_TREE_NODE = schema.getEncoding('@core/tree-node')
+const CORE_DEPENDENCY = schema.getEncoding('@core/dependency')
+const CORE_HINTS = schema.getEncoding('@core/hints')
+
+class CoreTX {
+  constructor (core, db, view, changes) {
+    this.core = core
+    this.db = db
+    this.view = view
+    this.changes = changes
+  }
+
+  setAuth (auth) {
+    this.changes.push([core.auth(this.core.corePointer), encode(CORE_AUTH, auth), null])
+  }
+
+  setSessions (sessions) {
+    this.changes.push([core.sessions(this.core.corePointer), encode(CORE_SESSIONS, sessions), null])
+  }
+
+  setHead (head) {
+    this.changes.push([core.head(this.core.dataPointer), encode(CORE_HEAD, head), null])
+  }
+
+  setDependency (dep) {
+    this.changes.push([core.dependency(this.core.dataPointer), encode(CORE_DEPENDENCY, dep), null])
+  }
+
+  setHints (hints) {
+    this.changes.push([core.hints(this.core.dataPointer), encode(CORE_HINTS, hints), null])
+  }
+
+  putBlock (index, data) {
+    this.changes.push([core.block(this.core.dataPointer, index), data, null])
+  }
+
+  deleteBlock (index) {
+    this.changes.push([core.block(this.core.dataPointer, index), null, null])
+  }
+
+  deleteBlockRange (start, end) {
+    this.changes.push([
+      core.block(this.core.dataPointer, start),
+      null,
+      core.block(this.core.dataPointer, end === -1 ? Infinity : end)
+    ])
+  }
+
+  putBitfieldPage (index, data) {
+    this.changes.push([core.bitfield(this.core.dataPointer, index, 0), data, null])
+  }
+
+  deleteBitfieldPage (index) {
+    this.changes.push([core.bitfield(this.core.dataPointer, index, 0), null, null])
+  }
+
+  deleteBitfieldPageRange (start, end) {
+    this.changes.push([
+      core.bitfield(this.core.dataPointer, start, 0),
+      null,
+      core.bitfield(this.core.dataPointer, end === -1 ? Infinity : end, 0)
+    ])
+  }
+
+  putTreeNode (node) {
+    this.changes.push([core.tree(this.core.dataPointer, node.index), encode(CORE_TREE_NODE, node), null])
+  }
+
+  deleteTreeNode (index) {
+    this.changes.push([core.tree(this.core.dataPointer, index), null, null])
+  }
+
+  deleteTreeNodeRange (start, end) {
+    this.changes.push([
+      core.tree(this.core.dataPointer, start),
+      null,
+      core.tree(this.core.dataPointer, end === -1 ? Infinity : end)
+    ])
+  }
+
+  putUserData (key, value) {
+    const buffer = typeof value === 'string' ? b4a.from(value) : value
+    this.changes.push([core.userData(this.core.dataPointer, key), buffer, null])
+  }
+
+  deleteUserData (key) {
+    this.changes.push([core.userData(this.core.dataPointer, key), null, null])
+  }
+
+  flush () {
+    const changes = this.changes
+    if (changes === null) return Promise.resolve(!this.view)
+
+    this.changes = null
+
+    if (this.view) {
+      this.view.apply(changes)
+      return Promise.resolve(false)
+    }
+
+    return View.flush(changes, this.db)
+  }
+}
+
+class CoreRX {
+  constructor (core, db, view) {
+    this.core = core
+    this.read = db.read({ autoDestroy: true })
+    this.view = view
+
+    view.readStart()
+  }
+
+  async getAuth () {
+    return await decode(CORE_AUTH, await this.view.get(this.read, core.auth(this.core.corePointer)))
+  }
+
+  async getSessions () {
+    return await decode(CORE_SESSIONS, await this.view.get(this.read, core.sessions(this.core.corePointer)))
+  }
+
+  async getHead () {
+    return await decode(CORE_HEAD, await this.view.get(this.read, core.head(this.core.dataPointer)))
+  }
+
+  async getDependency () {
+    return await decode(CORE_DEPENDENCY, await this.view.get(this.read, core.dependency(this.core.dataPointer)))
+  }
+
+  async getHints () {
+    return await decode(CORE_HINTS, await this.view.get(this.read, core.hints(this.core.dataPointer)))
+  }
+
+  getBlock (index) {
+    const dep = findBlockDependency(this.core.dependencies, index)
+    const data = dep === null ? this.core.dataPointer : dep.dataPointer
+    return this.view.get(this.read, core.block(data, index))
+  }
+
+  getBitfieldPage (index) {
+    return this.view.get(this.read, core.bitfield(this.core.dataPointer, index, 0))
+  }
+
+  async getTreeNode (index) {
+    const dep = findTreeDependency(this.core.dependencies, index)
+    const data = dep === null ? this.core.dataPointer : dep.dataPointer
+    return decode(CORE_TREE_NODE, await this.view.get(this.read, core.tree(data, index)))
+  }
+
+  async hasTreeNode (index) {
+    return (await this.getTreeNode(index)) !== null
+  }
+
+  getUserData (key) {
+    return this.view.get(this.read, core.userData(this.core.dataPointer, key))
+  }
+
+  tryFlush () {
+    this.read.tryFlush()
+    this._free()
+  }
+
+  destroy () {
+    this.read.destroy()
+    this._free()
+  }
+
+  _free () {
+    if (this.view === null) return
+    this.view.readStop()
+    this.view = null
+  }
+}
+
+class CorestoreTX {
+  constructor (view) {
+    this.view = view
+    this.changes = []
+  }
+
+  setHead (head) {
+    this.changes.push([store.head(), encode(CORESTORE_HEAD, head), null])
+  }
+
+  putCore (discoveryKey, ptr) {
+    this.changes.push([store.core(discoveryKey), encode(CORESTORE_CORE, ptr), null])
+  }
+
+  putCoreByAlias (alias, discoveryKey) {
+    this.changes.push([store.coreByAlias(alias), discoveryKey, null])
+  }
+
+  clear () {
+    const [start, end] = store.clear()
+    this.changes.push([start, null, end])
+  }
+
+  apply () {
+    if (this.changes === null) return
+    this.view.apply(this.changes)
+    this.changes = null
+  }
+}
+
+class CorestoreRX {
+  constructor (db, view) {
+    this.read = db.read({ autoDestroy: true })
+    this.view = view
+
+    view.readStart()
+  }
+
+  async getHead () {
+    return decode(CORESTORE_HEAD, await this.view.get(this.read, store.head()))
+  }
+
+  async getCore (discoveryKey) {
+    return decode(CORESTORE_CORE, await this.view.get(this.read, store.core(discoveryKey)))
+  }
+
+  getCoreByAlias (alias) {
+    return this.view.get(this.read, store.coreByAlias(alias))
+  }
+
+  tryFlush () {
+    this.read.tryFlush()
+    this._free()
+  }
+
+  destroy () {
+    this.read.destroy()
+    this._free()
+  }
+
+  _free () {
+    if (this.view === null) return
+    this.view.readStop()
+    this.view = null
+  }
+}
+
+module.exports = { CorestoreTX, CorestoreRX, CoreTX, CoreRX }
+
+function findBlockDependency (dependencies, index) {
+  for (let i = 0; i < dependencies.length; i++) {
+    const dep = dependencies[i]
+    if (index < dep.length) return dep
+  }
+
+  return null
+}
+
+function findTreeDependency (dependencies, index) {
+  for (let i = 0; i < dependencies.length; i++) {
+    const dep = dependencies[i]
+    if (flat.rightSpan(index) <= (dep.length - 1) * 2) return dep
+  }
+
+  return null
+}
+
+function decode (enc, buffer) {
+  if (buffer === null) return null
+  return enc.decode({ start: 0, end: buffer.byteLength, buffer })
+}
+
+function encode (enc, m) {
+  // TODO: use fancy slab for small messages
+  const state = { start: 0, end: 0, buffer: null }
+  enc.preencode(state, m)
+  state.buffer = b4a.allocUnsafe(state.end)
+  enc.encode(state, m)
+  return state.buffer
+}

package/lib/view.js

@@ -0,0 +1,345 @@
+const { Readable, getStreamError } = require('streamx')
+const b4a = require('b4a')
+
+class OverlayStream extends Readable {
+  constructor (stream, start, end, reverse, changes, cleared) {
+    super()
+
+    this.start = start
+    this.end = end
+    this.reverse = reverse
+    this.changes = changes
+    this.cleared = cleared
+    this.change = 0
+    this.range = 0
+
+    this._stream = stream
+    this._drained = false
+
+    this._stream.on('readable', this._drainMaybe.bind(this))
+    this._stream.on('error', noop)
+    this._stream.on('close', this._onclose.bind(this))
+  }
+
+  _drainMaybe () {
+    if (this._drained === true) return
+    this._drained = this._onreadable()
+  }
+
+  _onclose () {
+    if (this.destroying) return
+
+    const err = getStreamError(this._stream)
+
+    if (err !== null) {
+      this.destroy(err)
+      return
+    }
+
+    while (this.change < this.changes.length) {
+      const c = this.changes[this.change++]
+      const key = c[0]
+      const value = c[1]
+
+      if (value !== null && this._inRange(key)) this.push({ key, value })
+    }
+
+    this.push(null)
+    this._stream = null
+  }
+
+  _onreadable () {
+    let data = this._stream.read()
+
+    if (data === null) return false
+
+    do {
+      this._push(data)
+      data = this._stream.read()
+    } while (data !== null)
+
+    return true
+  }
+
+  _read (cb) {
+    this._drained = this._onreadable()
+    cb(null)
+  }
+
+  _predestroy () {
+    this.stream.destroy()
+  }
+
+  _push (entry) {
+    const key = entry.key
+
+    while (this.range < this.cleared.length) {
+      const c = this.cleared[this.range]
+
+      // we moved past the range
+      if (this.reverse ? b4a.compare(key, c[0]) < 0 : b4a.compare(c[2], key) <= 0) {
+        this.range++
+        continue
+      }
+
+      // we didnt move past and are in, drop
+      if (b4a.compare(c[0], key) <= 0 && b4a.compare(key, c[2]) < 0) {
+        return true
+      }
+
+      break
+    }
+
+    while (this.change < this.changes.length) {
+      const c = this.changes[this.change]
+      const key = c[0]
+      const value = typeof c[1] === 'string' ? b4a.from(c[1]) : c[1]
+      const cmp = b4a.compare(key, entry.key)
+
+      // same value, if not deleted, return new one
+      if (cmp === 0) {
+        this.change++
+        return value === null || this._inRange(key) === false ? true : this.push({ key, value })
+      }
+
+      // we moved past the change, push it
+      if (this.reverse ? cmp > 0 : cmp < 0) {
+        this.change++
+        if (value !== null && this._inRange(key) === true) this.push({ key, value })
+        continue
+      }
+
+      return this.push(entry)
+    }
+
+    return this.push(entry)
+  }
+
+  _inRange (key) {
+    return b4a.compare(this.start, key) <= 0 && b4a.compare(key, this.end) < 0
+  }
+}
+
+class Overlay {
+  constructor () {
+    this.indexed = 0
+    this.changes = null
+    this.cleared = null
+    this.reverse = false
+  }
+
+  update (view, reverse) {
+    if (view.indexed === this.indexed) return
+
+    const changes = view.map === null ? [] : [...view.map.values()]
+    const cleared = view.cleared === null ? [] : view.cleared.slice(0)
+
+    const cmp = reverse ? cmpChangeReverse : cmpChange
+
+    changes.sort(cmp)
+    cleared.sort(cmp)
+
+    this.indexed = view.indexed
+    this.changes = changes
+    this.cleared = cleared
+    this.reverse = reverse
+  }
+
+  createStream (stream, start, end, reverse) {
+    return new OverlayStream(
+      stream,
+      start,
+      end,
+      reverse,
+      this.reverse === reverse ? this.changes : reverseArray(this.changes),
+      this.reverse === reverse ? this.cleared : reverseArray(this.cleared)
+    )
+  }
+}
+
+class View {
+  constructor () {
+    this.map = null
+    this.indexed = 0
+    this.changes = null
+    this.cleared = null
+    this.overlay = null
+    this.snap = null
+    this.readers = 0
+  }
+
+  snapshot () {
+    if (this._attached()) return this.snap.snapshot()
+
+    const snap = new View()
+
+    snap.map = this.map
+    snap.indexed = this.indexed
+    snap.changes = this.changes
+    snap.cleared = this.cleared
+
+    if (this._frozen()) return snap
+
+    this.readers++
+    snap.snap = this
+
+    return snap
+  }
+
+  readStart () {
+    if (this.snap !== null) this.readers++
+  }
+
+  readStop () {
+    if (this.snap !== null && --this.readers === 0) this.snap.readers--
+  }
+
+  size () {
+    return this.changes === null ? 0 : this.changes.length
+  }
+
+  updated () {
+    return this.changes === null
+  }
+
+  get (read, key) {
+    return this.changes === null ? read.get(key) : this._indexAndGet(read, key)
+  }
+
+  reset () {
+    this.indexed = 0
+    this.snap = this.map = this.changes = this.cleared = null
+  }
+
+  iterator (db, start, end, reverse) {
+    const stream = db.iterator({ gte: start, lt: end, reverse })
+    if (this.changes === null) return stream
+
+    this._index()
+
+    if (this.overlay === null) this.overlay = new Overlay()
+    this.overlay.update(this, reverse)
+    return this.overlay.createStream(stream, start, end, reverse)
+  }
+
+  _indexAndGet (read, key) {
+    this._index()
+    const change = this.map.get(b4a.toString(key, 'hex'))
+
+    if (change === undefined) {
+      return this.cleared === null
+        ? read.get(key)
+        : this._readAndMaybeDrop(read, key)
+    }
+
+    return Promise.resolve(change[1])
+  }
+
+  async _readAndMaybeDrop (read, key) {
+    const value = await read.get(key)
+    if (value === null) return null
+
+    for (let i = 0; i < this.cleared.length; i++) {
+      const c = this.cleared[i]
+      // check if in range
+      if (b4a.compare(c[0], key) <= 0 && b4a.compare(key, c[2]) < 0) return null
+    }
+
+    return value
+  }
+
+  _attached () {
+    return this.snap !== null && this.changes === this.snap.changes
+  }
+
+  _frozen () {
+    return this.changes === null || (this.snap !== null && this.changes !== this.snap.changes)
+  }
+
+  _index () {
+    // if we are a snap and we are still attached (ie no mutations), simply copy the refs
+    if (this._attached()) {
+      this.snap._index()
+      this.map = this.snap.map
+      this.cleared = this.snap.cleared
+      this.indexed = this.snap.indexed
+      return
+    }
+
+    if (this.changes.length === this.indexed) return
+    if (this.map === null) this.map = new Map()
+
+    while (this.indexed < this.changes.length) {
+      const c = this.changes[this.indexed++]
+
+      if (c[2] === null) this.map.set(b4a.toString(c[0], 'hex'), c)
+      else this._indexRange(c)
+    }
+  }
+
+  _indexRange (range) {
+    const s = b4a.toString(range[0], 'hex')
+    const e = b4a.toString(range[2], 'hex')
+
+    for (const [key, c] of this.map) {
+      if (s <= key && key < e) this.map.set(key, [c[0], null, null])
+    }
+
+    if (this.cleared === null) this.cleared = []
+    this.cleared.push(range)
+  }
+
+  apply (changes) {
+    if (this.snap !== null) throw new Error('Illegal to push changes to a snapshot')
+
+    if (this.readers !== 0 && this.changes !== null) {
+      this.changes = this.changes.slice(0)
+      this.cleared = this.cleared === null ? null : this.cleared.slice(0)
+      this.map = this.map === null ? null : new Map([...this.map])
+    }
+
+    if (this.changes === null) {
+      this.changes = changes
+      return
+    }
+
+    for (let i = 0; i < changes.length; i++) {
+      this.changes.push(changes[i])
+    }
+  }
+
+  static async flush (changes, db) {
+    if (changes === null) return true
+
+    const w = db.write({ autoDestroy: true })
+
+    for (const [start, value, end] of changes) {
+      if (end !== null) w.tryDeleteRange(start, end)
+      else if (value !== null) w.tryPut(start, value)
+      else w.tryDelete(start)
+    }
+
+    await w.flush()
+
+    return true
+  }
+}
+
+module.exports = View
+
+function cmpChange (a, b) {
+  const c = b4a.compare(a[0], b[0])
+  return c === 0 ? b4a.compare(a[2], b[2]) : c
+}
+
+function cmpChangeReverse (a, b) {
+  return cmpChange(b, a)
+}
+
+function noop () {}
+
+function reverseArray (list) {
+  const r = new Array(list.length)
+  for (let i = 0; i < list.length; i++) r[r.length - 1 - i] = list[i]
+  return r
+}