hypercore-storage 0.0.37 → 0.0.39

package/index.js

@@ -162,7 +162,7 @@ class WriteBatch {
 
   flush () {
     if (this.atom) return this.atom.flush()
-    return this.write.flush()
+    return flushAndDestroy(this.write)
   }
 }
 
@@ -259,11 +259,11 @@ class ReadBatch {
   }
 
   flush () {
-    return this.read.flush()
+    return flushAndDestroy(this.read)
   }
 
   tryFlush () {
-    this.read.tryFlush()
+    tryFlushAndDestroy(this.read)
   }
 }
 
@@ -378,10 +378,12 @@ class Atom {
     try {
       await batch.flush()
     } catch (err) {
+      batch.destroy()
       reject(err)
       return
     }
 
+    batch.destroy()
     resolve()
   }
 
@@ -411,7 +413,7 @@ class Atom {
 
 module.exports = class CoreStorage {
   constructor (dir) {
-    this.db = new RocksDB(dir)
+    this.db = typeof dir === 'object' ? dir : new RocksDB(dir)
     this.mutex = new RW()
   }
 
@@ -433,7 +435,7 @@ module.exports = class CoreStorage {
     try {
       const b = this.db.write()
       b.tryPut(b4a.from([TL.LOCAL_SEED]), seed)
-      await b.flush()
+      await flushAndDestroy(b)
 
       return true
     } finally {
@@ -488,7 +490,7 @@ module.exports = class CoreStorage {
   async clear () {
     const b = this.db.write()
     b.tryDeleteRange(b4a.from([TL.STORAGE_INFO]), INF)
-    await b.flush()
+    await flushAndDestroy(b)
   }
 
   async has (discoveryKey) {
@@ -556,6 +558,7 @@ class HypercoreStorage {
     this.root = root
     this.db = root.db
     this.dbSnapshot = snapshot
+    this.dbRead = snapshot || this.db
     this.mutex = root.mutex
 
     this.discoveryKey = discoveryKey
@@ -617,7 +620,8 @@ class HypercoreStorage {
       batch.setBatchPointer(name, storage.dataPointer)
       if (head.rootHash) batch.setCoreHead(head) // if no root hash its the empty core - no head yet
 
-      await write.flush()
+      if (atom) await atom.flush()
+      else await flushAndDestroy(write)
 
       storage.dependencies = await addDependencies(this.db, storage.dataPointer, head.length)
       return storage
@@ -644,7 +648,7 @@ class HypercoreStorage {
 
   snapshot () {
     assert(this.destroyed === false)
-    const s = new HypercoreStorage(this.root, this.discoveryKey, this.corePointer, this.dataPointer, this.db.snapshot())
+    const s = new HypercoreStorage(this.root, this.discoveryKey, this.corePointer, this.dataPointer, this.dbRead.snapshot())
 
     for (const { data, length } of this.dependencies) s.dependencies.push({ data, length })
 
@@ -677,8 +681,7 @@ class HypercoreStorage {
   createReadBatch (opts) {
     assert(this.destroyed === false)
 
-    const snapshot = this.dbSnapshot
-    return new ReadBatch(this, this.db.read({ snapshot }))
+    return new ReadBatch(this, this.dbRead.read())
   }
 
   createWriteBatch (atom) {
@@ -689,6 +692,24 @@ class HypercoreStorage {
     return new WriteBatch(this, this.db.write(), null)
   }
 
+  // helper for atomic flows
+  async createWriteBatchAndLock (atomizer, lock) {
+    assert(this.destroyed === false)
+    if (atomizer) atomizer.enter()
+
+    try {
+      await lock.lock()
+    } catch (err) {
+      if (atomizer) atomizer.exit()
+      throw err
+    }
+
+    const batch = this.createWriteBatch(atomizer)
+    if (atomizer) atomizer.exit()
+
+    return batch
+  }
+
   createBlockStream (opts = {}) {
     assert(this.destroyed === false)
     return createStream(this, createBlockStream, opts)
@@ -697,8 +718,8 @@ class HypercoreStorage {
   createUserDataStream (opts = {}) {
     assert(this.destroyed === false)
 
-    const r = encodeUserDataRange(this.dataPointer, DATA.USER_DATA, this.dbSnapshot, opts)
-    const s = this.db.iterator(r)
+    const r = encodeUserDataRange(this.dataPointer, DATA.USER_DATA, opts)
+    const s = this.dbRead.iterator(r)
     s._readableState.map = mapStreamUserData
     return s
   }
@@ -706,8 +727,8 @@ class HypercoreStorage {
   createTreeNodeStream (opts = {}) {
     assert(this.destroyed === false)
 
-    const r = encodeIndexRange(this.dataPointer, DATA.TREE, this.dbSnapshot, opts)
-    const s = this.db.iterator(r)
+    const r = encodeIndexRange(this.dataPointer, DATA.TREE, opts)
+    const s = this.dbRead.iterator(r)
     s._readableState.map = mapStreamTreeNode
     return s
   }
@@ -715,8 +736,8 @@ class HypercoreStorage {
   createBitfieldPageStream (opts = {}) {
     assert(this.destroyed === false)
 
-    const r = encodeIndexRange(this.dataPointer, DATA.BITFIELD, this.dbSnapshot, opts)
-    const s = this.db.iterator(r)
+    const r = encodeIndexRange(this.dataPointer, DATA.BITFIELD, opts)
+    const s = this.dbRead.iterator(r)
     s._readableState.map = mapStreamBitfieldPage
     return s
   }
@@ -724,7 +745,7 @@ class HypercoreStorage {
   async peekLastTreeNode () {
     assert(this.destroyed === false)
 
-    const last = await this.db.peek(encodeIndexRange(this.dataPointer, DATA.TREE, this.dbSnapshot, { reverse: true }))
+    const last = await this.dbRead.peek(encodeIndexRange(this.dataPointer, DATA.TREE, { reverse: true }))
     if (last === null) return null
     return c.decode(m.TreeNode, last.value)
   }
@@ -732,7 +753,7 @@ class HypercoreStorage {
   async peekLastBitfieldPage () {
     assert(this.destroyed === false)
 
-    const last = await this.db.peek(encodeIndexRange(this.dataPointer, DATA.BITFIELD, this.dbSnapshot, { reverse: true }))
+    const last = await this.dbRead.peek(encodeIndexRange(this.dataPointer, DATA.BITFIELD, { reverse: true }))
     if (last === null) return null
     return mapStreamBitfieldPage(last)
   }
@@ -741,19 +762,19 @@ class HypercoreStorage {
     if (this.destroyed) return
     this.destroyed = true
 
-    if (this.dbSnapshot) this.dbSnapshot.destroy()
+    if (this.dbSnapshot) this.dbSnapshot.close().catch(noop)
     this.dbSnapshot = null
   }
 }
 
 function createStream (storage, createStreamType, opts) {
   return storage.dependencies.length === 0
-    ? createStreamType(storage.db, storage.dbSnapshot, storage.dataPointer, opts)
+    ? createStreamType(storage.db, storage.dataPointer, opts)
     : new DependencyStream(storage, createStreamType, opts)
 }
 
-function createBlockStream (db, snap, data, opts) {
-  const r = encodeIndexRange(data, DATA.BLOCK, snap, opts)
+function createBlockStream (db, data, opts) {
+  const r = encodeIndexRange(data, DATA.BLOCK, opts)
   const s = db.iterator(r)
   s._readableState.map = mapStreamBlock
   return s
@@ -827,8 +848,8 @@ function ensureSlab (size) {
   return SLAB
 }
 
-function encodeIndexRange (pointer, type, snapshot, opts) {
-  const bounded = { snapshot, gt: null, gte: null, lte: null, lt: null, reverse: !!opts.reverse, limit: toLimit(opts.limit) }
+function encodeIndexRange (pointer, type, opts) {
+  const bounded = { gt: null, gte: null, lte: null, lt: null, reverse: !!opts.reverse, limit: toLimit(opts.limit) }
 
   if (opts.gt || opts.gt === 0) bounded.gt = encodeDataIndex(pointer, type, opts.gt)
   else if (opts.gte) bounded.gte = encodeDataIndex(pointer, type, opts.gte)
@@ -841,8 +862,8 @@ function encodeIndexRange (pointer, type, snapshot, opts) {
   return bounded
 }
 
-function encodeUserDataRange (pointer, type, snapshot, opts) {
-  const bounded = { snapshot, gt: null, gte: null, lte: null, lt: null, reverse: !!opts.reverse, limit: toLimit(opts.limit) }
+function encodeUserDataRange (pointer, type, opts) {
+  const bounded = { gt: null, gte: null, lte: null, lt: null, reverse: !!opts.reverse, limit: toLimit(opts.limit) }
 
   if (opts.gt || opts.gt === 0) bounded.gt = encodeUserDataIndex(pointer, type, opts.gt)
   else if (opts.gte) bounded.gte = encodeUserDataIndex(pointer, type, opts.gte)
@@ -965,3 +986,24 @@ function initialiseCoreData (db, { userData } = {}) {
     }
   }
 }
+
+function noop () {}
+
+async function tryFlushAndDestroy (batch) {
+  try {
+    await batch.flush()
+  } catch {}
+
+  batch.destroy()
+}
+
+async function flushAndDestroy (batch) {
+  try {
+    await batch.flush()
+  } catch (err) {
+    batch.destroy()
+    throw err
+  }
+
+  batch.destroy()
+}

package/lib/dependency-stream.js

@@ -93,7 +93,7 @@ module.exports = class DependencyStream extends Readable {
   _nextStream () {
     const { data, gte, lt } = this._streams[this._next++]
 
-    const stream = this.createStream(this.storage.db, this.storage.dbSnapshot, data, {
+    const stream = this.createStream(this.storage.dbRead, data, {
       reverse: this._reverse,
       limit: this._limit,
       gte,

package/lib/memory-overlay.js

@@ -117,7 +117,7 @@ class MemoryOverlay {
     this.snapshotted = false
     this.snapshotShared = null
     this.activeSnapshots = 0
-    this.copyOnWrite = false
+    this.copied = false
   }
 
   async registerBatch (name, length, overwrite) {
@@ -128,7 +128,6 @@ class MemoryOverlay {
     if (this.snapshotShared !== null) return this.snapshotShared.snapshot()
 
     this.activeSnapshots++
-    this.copyOnWrite = true
 
     const snap = new MemoryOverlay(this.storage.snapshot())
 
@@ -150,7 +149,8 @@ class MemoryOverlay {
   }
 
   _copy () {
-    this.copyOnWrite = false
+    if (this.copied) return
+    this.copied = true
 
     // clone state
     if (this.head !== null) this.head = Object.assign(this.head)
@@ -179,7 +179,7 @@ class MemoryOverlay {
   }
 
   createWriteBatch () {
-    if (this.copyOnWrite) this._copy()
+    if (this.activeSnapshots > 0) this._copy()
     return new MemoryOverlayWriteBatch(this)
   }
 
@@ -222,7 +222,7 @@ class MemoryOverlay {
   destroy () {
     if (this.snapshotted) this.storage.destroy()
     if (this.snapshotShared === null) return
-    if (--this.snapshotShared.activeSnapshots === 0) this.snapshotShared.copyOnWrite = false
+    this.snapshotShared.activeSnapshots--
     this.snapshotShared = null
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore-storage",
-  "version": "0.0.37",
+  "version": "0.0.39",
   "main": "index.js",
   "files": [
     "index.js",
@@ -22,7 +22,7 @@
     "queue-tick": "^1.0.1",
     "read-write-mutexify": "^2.1.0",
     "resolve-reject-promise": "^1.0.0",
-    "rocksdb-native": "^2.2.0",
+    "rocksdb-native": "^3.0.0",
     "streamx": "^2.20.1"
   },
   "devDependencies": {