hypercore-storage 0.0.21 → 0.0.23

package/index.js

@@ -5,8 +5,10 @@ const RW = require('read-write-mutexify')
 const b4a = require('b4a')
 const flat = require('flat-tree')
 const assert = require('nanoassert')
+
 const m = require('./lib/messages')
 const DependencyStream = require('./lib/dependency-stream')
+const MemoryOverlay = require('./lib/memory-overlay')
 
 const INF = b4a.from([0xff])
 
@@ -181,6 +183,10 @@ class ReadBatch {
     return this._get(encodeCoreIndex(this.storage.corePointer, CORE.ENCRYPTION_KEY), null)
   }
 
+  getDataDependency () {
+    return this._get(encodeDataIndex(this.storage.dataPointer, DATA.DEPENDENCY), m.DataDependency)
+  }
+
   getDataInfo (info) {
     return this._get(encodeDataIndex(this.storage.dataPointer, DATA.INFO), m.DataInfo)
   }
@@ -426,23 +432,32 @@ class HypercoreStorage {
     return this.dbSnapshot !== null
   }
 
-  async registerBatch (name, length, overwrite) {
-    // todo: make sure opened
+  dependencyLength () {
+    return this.dependencies.length
+      ? this.dependencies[this.dependencies.length - 1].length
+      : -1
+  }
+
+  async openBatch (name) {
     const existing = await this.db.get(encodeBatch(this.corePointer, CORE.BATCHES, name))
-    const storage = new HypercoreStorage(this.root, this.discoveryKey, this.corePointer, this.dataPointer, null)
+    if (!existing) return null
 
-    if (existing && !overwrite) {
-      const dataPointer = c.decode(m.DataPointer, existing)
-      storage.dataPointer = dataPointer
-      storage.dependencies = await addDependencies(this.db, storage.dataPointer, length)
-      return storage
-    }
+    const storage = new HypercoreStorage(this.root, this.discoveryKey, this.corePointer, this.dataPointer, this.dbSnapshot)
+    const dataPointer = c.decode(m.DataPointer, existing)
 
+    storage.dataPointer = dataPointer
+    storage.dependencies = await addDependencies(this.db, storage.dataPointer, -1)
+
+    return storage
+  }
+
+  async registerBatch (name, head) {
     await this.mutex.write.lock()
 
+    const storage = new HypercoreStorage(this.root, this.discoveryKey, this.corePointer, this.dataPointer, null)
+
     try {
       const info = await getStorageInfo(this.db)
-
       const write = this.db.write()
 
       storage.dataPointer = info.free++
@@ -453,18 +468,23 @@ class HypercoreStorage {
 
       initialiseCoreData(batch)
 
-      batch.setDataDependency({ data: this.dataPointer, length })
+      batch.setDataDependency({ data: this.dataPointer, length: head.length })
       batch.setBatchPointer(name, storage.dataPointer)
+      if (head.rootHash) batch.setCoreHead(head) // if no root hash its the empty core - no head yet
 
       await write.flush()
 
-      storage.dependencies = await addDependencies(this.db, storage.dataPointer, length)
+      storage.dependencies = await addDependencies(this.db, storage.dataPointer, head.length)
       return storage
     } finally {
       this.mutex.write.unlock()
     }
   }
 
+  createMemoryOverlay () {
+    return new MemoryOverlay(this)
+  }
+
   snapshot () {
     assert(this.closed === false)
     return new HypercoreStorage(this.root, this.discoveryKey, this.corePointer, this.dataPointer, this.db.snapshot())
@@ -515,7 +535,7 @@ class HypercoreStorage {
     return s
   }
 
-  async peakLastTreeNode () {
+  async peekLastTreeNode () {
     assert(this.closed === false)
 
     const last = await this.db.peek(encodeIndexRange(this.dataPointer, DATA.TREE, this.dbSnapshot, { reverse: true }))
@@ -523,7 +543,7 @@ class HypercoreStorage {
     return c.decode(m.TreeNode, last.value)
   }
 
-  async peakLastBitfieldPage () {
+  async peekLastBitfieldPage () {
     assert(this.closed === false)
 
     const last = await this.db.peek(encodeIndexRange(this.dataPointer, DATA.BITFIELD, this.dbSnapshot, { reverse: true }))
@@ -709,7 +729,7 @@ async function addDependencies (db, dataPointer, treeLength) {
   let dep = await db.get(encodeDataIndex(dataPointer, DATA.DEPENDENCY))
   while (dep) {
     const { data, length } = c.decode(m.DataDependency, dep)
-    if (length <= treeLength) dependencies.push({ data, length })
+    if (treeLength === -1 || length <= treeLength) dependencies.push({ data, length })
 
     dep = await db.get(encodeDataIndex(data, DATA.DEPENDENCY))
   }

package/lib/memory-overlay.js

@@ -0,0 +1,285 @@
+const { ASSERTION } = require('hypercore-errors')
+
+const TipList = require('./tip-list')
+
+class MemoryOverlay {
+  constructor (storage) {
+    this.storage = storage
+    this.head = null
+    this.auth = null
+    this.localKeyPair = null
+    this.encryptionKey = null
+    this.dataDependency = null
+    this.dataInfo = null
+    this.userData = null
+    this.blocks = null
+    this.treeNodes = null
+    this.bitfields = null
+
+    this.snapshotted = false
+  }
+
+  async registerBatch (name, length, overwrite) {
+    todo()
+  }
+
+  snapshot () {
+    todo()
+  }
+
+  get dependencies () {
+    return this.storage.dependencies
+  }
+
+  dependencyLength () {
+    if (this.dataDependency) return this.dataDependency.length
+    return this.storage.dependencyLength()
+  }
+
+  createReadBatch () {
+    return new MemoryOverlayReadBatch(this, this.storage.createReadBatch())
+  }
+
+  createWriteBatch () {
+    return new MemoryOverlayWriteBatch(this)
+  }
+
+  createBlockStream () {
+    todo()
+  }
+
+  createUserDataStream () {
+    todo()
+  }
+
+  createTreeNodeStream () {
+    todo()
+  }
+
+  createBitfieldPageStream () {
+    todo()
+  }
+
+  async peekLastTreeNode () {
+    todo()
+  }
+
+  async peekLastBitfieldPage () {
+    const mem = this.bitfields !== null ? this.bitfields.get(this.bitfields.length() - 1) : { valid: false }
+
+    const page = mem.valid ? mem.value : null
+    const index = page ? this.bitfields.length() - 1 : -1
+
+    const disk = await this.storage.peekLastBitfieldPage()
+
+    return (page && (!disk || index > disk.index)) ? { index, page } : disk
+  }
+
+  close () {
+    return Promise.resolve()
+  }
+
+  merge (overlay) {
+    if (overlay.head !== null) this.head = overlay.head
+    if (overlay.auth !== null) this.auth = overlay.auth
+    if (overlay.localKeyPair !== null) this.localKeyPair = overlay.localKeyPair
+    if (overlay.encryptionKey !== null) this.encryptionKey = overlay.encryptionKey
+    if (overlay.dataDependency !== null) this.dataDependency = overlay.dataDependency
+    if (overlay.dataInfo !== null) this.dataInfo = overlay.dataInfo
+    if (overlay.userData !== null) this.userData = mergeMap(this.userData, overlay.userData)
+    if (overlay.blocks !== null) this.blocks = mergeTip(this.blocks, overlay.blocks)
+    if (overlay.treeNodes !== null) this.treeNodes = mergeMap(this.treeNodes, overlay.treeNodes)
+    if (overlay.bitfields !== null) this.bitfields = mergeTip(this.bitfields, overlay.bitfields)
+  }
+}
+
+module.exports = MemoryOverlay
+
+class MemoryOverlayReadBatch {
+  constructor (overlay, read) {
+    this.read = read
+    this.overlay = overlay
+  }
+
+  async getCoreHead () {
+    return this.overlay.head !== null ? this.overlay.head : this.read.getCoreHead()
+  }
+
+  async getCoreAuth () {
+    return this.overlay.auth !== null ? this.overlay.auth : this.read.getCoreAuth()
+  }
+
+  async getDataDependency () {
+    return this.overlay.dataDependency !== null ? this.overlay.dataDependency : this.read.getDataDependency()
+  }
+
+  async getLocalKeyPair () {
+    return this.overlay.localKeyPair !== null ? this.overlay.localKeyPair : this.read.getLocalKeyPair()
+  }
+
+  async getEncryptionKey () {
+    return this.overlay.encryptionKey !== null ? this.overlay.encryptionKey : this.read.getEncryptionKey()
+  }
+
+  async getDataInfo () {
+    return this.overlay.dataInfo !== null ? this.overlay.dataInfo : this.read.getDataInfo()
+  }
+
+  async getUserData (key) {
+    return this.overlay.userData !== null && this.overlay.userData.has(key)
+      ? this.overlay.userData.get(key)
+      : this.read.getUserData(key)
+  }
+
+  async hasBlock (index) {
+    if (this.overlay.blocks !== null && index >= this.overlay.blocks.offset) {
+      const blk = this.overlay.blocks.get(index)
+      if (blk.valid) return true
+    }
+    return this.read.hasBlock(index)
+  }
+
+  async getBlock (index, error) {
+    if (this.overlay.blocks !== null && index >= this.overlay.blocks.offset) {
+      const blk = this.overlay.blocks.get(index)
+      if (blk.valid) return blk.value
+    }
+    return this.read.getBlock(index, error)
+  }
+
+  async hasTreeNode (index) {
+    return this.overlay.treeNodes !== null
+      ? this.overlay.treeNodes.has(index)
+      : this.read.hasTreeNode(index)
+  }
+
+  async getTreeNode (index, error) {
+    return this.overlay.treeNodes !== null && this.overlay.treeNodes.has(index)
+      ? this.overlay.treeNodes.get(index)
+      : this.read.getTreeNode(index, error)
+  }
+
+  async getBitfieldPage (index) {
+    if (this.overlay.bitfields !== null && index >= this.overlay.bitfields.offset) {
+      const page = this.overlay.bitfields.get(index)
+      if (page.valid) return page.value
+    }
+    return this.read.getBitfieldPage(index)
+  }
+
+  destroy () {
+    this.read.destroy()
+  }
+
+  flush () {
+    return this.read.flush()
+  }
+
+  tryFlush () {
+    this.read.tryFlush()
+  }
+}
+
+class MemoryOverlayWriteBatch {
+  constructor (storage) {
+    this.storage = storage
+    this.overlay = new MemoryOverlay()
+  }
+
+  setCoreHead (head) {
+    this.overlay.head = head
+  }
+
+  setCoreAuth (auth) {
+    this.overlay.auth = auth
+  }
+
+  setBatchPointer (name, pointer) {
+    todo()
+  }
+
+  setDataDependency (dependency) {
+    this.overlay.dataDependency = dependency
+  }
+
+  setLocalKeyPair (keyPair) {
+    this.overlay.localKeyPair = keyPair
+  }
+
+  setEncryptionKey (encryptionKey) {
+    this.overlay.encryptionKey = encryptionKey
+  }
+
+  setDataInfo (info) {
+    this.overlay.dataInfo = info
+  }
+
+  setUserData (key, value) {
+    if (this.overlay.userData === null) this.overlay.userData = new Map()
+    this.overlay.userData.set(key, value)
+  }
+
+  putBlock (index, data) {
+    if (this.overlay.blocks === null) this.overlay.blocks = new TipList()
+    this.overlay.blocks.put(index, data)
+  }
+
+  deleteBlock (index) {
+    todo()
+  }
+
+  deleteBlockRange (start, end) {
+    if (this.overlay.blocks === null) this.overlay.blocks = new TipList()
+    this.overlay.blocks.delete(start, end)
+  }
+
+  putTreeNode (node) {
+    if (this.overlay.treeNodes === null) this.overlay.treeNodes = new Map()
+    this.overlay.treeNodes.set(node.index, node)
+  }
+
+  deleteTreeNode (index) {
+    todo()
+  }
+
+  deleteTreeNodeRange (start, end) {
+    todo()
+  }
+
+  putBitfieldPage (index, page) {
+    if (this.overlay.bitfields === null) this.overlay.bitfields = new TipList()
+    this.overlay.bitfields.put(index, page)
+  }
+
+  deleteBitfieldPage (index) {
+    todo()
+  }
+
+  deleteBitfieldPageRange (start, end) {
+    if (this.overlay.bitfields === null) this.overlay.bitfields = new TipList()
+    this.overlay.bitfields.delete(start, end)
+  }
+
+  destroy () {}
+
+  flush () {
+    this.storage.merge(this.overlay)
+    return Promise.resolve()
+  }
+}
+
+function mergeMap (a, b) {
+  if (a === null) return b
+  for (const [key, value] of b) a.set(key, value)
+  return a
+}
+
+function mergeTip (a, b) {
+  if (a === null) return b
+  a.merge(b)
+  return a
+}
+
+function todo () {
+  throw ASSERTION('Not supported yet, but will be')
+}

package/lib/tip-list.js

@@ -0,0 +1,93 @@
+const { ASSERTION } = require('hypercore-errors')
+
+module.exports = class TipList {
+  constructor () {
+    this.offset = 0
+    this.removed = 0
+    this.data = []
+  }
+
+  length () {
+    return this.offset + this.data.length
+  }
+
+  end () {
+    if (this.removed) return this.removed
+    return this.offset + this.data.length
+  }
+
+  delete (start, end) {
+    if (end !== -1) {
+      if (end < this.length() || (this.length() < start && this.length() !== 0)) {
+        throw ASSERTION('Invalid delete on tip list')
+      }
+    }
+
+    if (this.end() === 0) this.offset = start
+    if (this.removed < end || end === -1) this.removed = end
+
+    if (start < this.offset) {
+      this.offset = start
+      this.data = [] // clear everything
+      return
+    }
+
+    while (this.length() > start) this.data.pop()
+  }
+
+  put (index, value) {
+    if (this.end() === 0) {
+      this.offset = index
+      this.data.push(value)
+      return
+    }
+
+    if (!this.removed && this.end() === index) {
+      this.data.push(value)
+      return
+    }
+
+    throw ASSERTION('Invalid put on tip list')
+  }
+
+  get (index) {
+    if (index < this.offset) {
+      return {
+        valid: false,
+        value: null
+      }
+    }
+
+    if (index - this.offset < this.data.length) {
+      return {
+        valid: true,
+        value: this.data[index - this.offset]
+      }
+    }
+
+    return {
+      valid: index < this.end(),
+      value: null
+    }
+  }
+
+  * [Symbol.iterator] () {
+    for (let i = 0; i < this.data.length; i++) {
+      yield [i + this.offset, this.data[i]]
+    }
+  }
+
+  merge (tip) {
+    const invalidDeletion = tip.removed && tip.end() !== -1 && tip.end() < this.end()
+    const invalidTip = (tip.removed !== -1 && tip.end() < this.offset) || this.end() < tip.offset
+
+    if (invalidTip || invalidDeletion) throw ASSERTION('Cannot merge tip list')
+
+    while (this.end() !== tip.offset && tip.offset >= this.offset && tip.end() >= this.end()) this.data.pop()
+    while (tip.removed && this.end() > tip.length() && this.data.length) this.data.pop()
+
+    for (const data of tip.data) this.data.push(data)
+
+    return this
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore-storage",
-  "version": "0.0.21",
+  "version": "0.0.23",
   "main": "index.js",
   "files": [
     "index.js",
@@ -16,6 +16,7 @@
     "b4a": "^1.6.6",
     "compact-encoding": "^2.15.0",
     "flat-tree": "^1.10.0",
+    "hypercore-errors": "^1.3.0",
     "index-encoder": "^3.0.1",
     "nanoassert": "^2.0.0",
     "read-write-mutexify": "^2.1.0",