hypercore-storage 0.0.36 → 0.0.38

package/index.js

@@ -162,7 +162,7 @@ class WriteBatch {
 
   flush () {
     if (this.atom) return this.atom.flush()
-    return this.write.flush()
+    return flushAndDestroy(this.write)
   }
 }
 
@@ -259,15 +259,32 @@ class ReadBatch {
   }
 
   flush () {
-    return this.read.flush()
+    return flushAndDestroy(this.read)
   }
 
   tryFlush () {
-    this.read.tryFlush()
+    tryFlushAndDestroy(this.read)
   }
 }
 
-class Atomizer {
+class Lock {
+  constructor (mutex) {
+    this.mutex = mutex
+    this.refs = 0
+    this.promise = null
+  }
+
+  acquire () {
+    if (this.refs++ === 0) this.promise = this.mutex.lock()
+    return this
+  }
+
+  release () {
+    if (--this.refs === 0) return this.mutex.unlock()
+  }
+}
+
+class Atom {
   constructor (db) {
     this.db = db
     this.batch = null
@@ -276,6 +293,11 @@ class Atomizer {
     this.flushing = null
     this.resolve = null
     this.reject = null
+
+    this._executing = null
+    this._waiting = []
+    this._queue = []
+    this._enqueue = (lock, resolve, reject) => this._queue.push({ lock, resolve, reject })
   }
 
   enter () {
@@ -286,6 +308,43 @@ class Atomizer {
     if (--this.refs === 0) this._commit()
   }
 
+  acquire (mutex) {
+    const lock = this._lock(mutex)
+    return new Promise(this._enqueue.bind(this, lock))
+  }
+
+  _lock (mutex) {
+    for (const lock of this._waiting) {
+      if (lock.mutex === mutex) return lock.acquire()
+    }
+
+    const lock = new Lock(mutex)
+    this._waiting.push(lock)
+
+    if (this._executing === null) this._executing = this._execute()
+
+    return lock.acquire()
+  }
+
+  async _execute () {
+    this.enter()
+    for (const { promise } of this._waiting) await promise
+
+    const queue = this._queue
+
+    this._waiting = []
+    this._queue = []
+    this._executing = null
+
+    for (const { lock, resolve, reject } of queue) {
+      if (this.destroyed) reject(new Error('Atomizer destroyed'))
+      else resolve(lock)
+    }
+
+    this._ensureTick() // allow caller to enter
+    this.exit()
+  }
+
   createBatch () {
     if (this.refs === 0) this._ensureTick()
     this.enter()
@@ -299,6 +358,8 @@ class Atomizer {
   }
 
   async _commit () {
+    if (this.batch === null) return
+
     const batch = this.batch
     const resolve = this.resolve
     const reject = this.reject
@@ -317,10 +378,12 @@ class Atomizer {
     try {
       await batch.flush()
     } catch (err) {
+      batch.destroy()
       reject(err)
       return
     }
 
+    batch.destroy()
     resolve()
   }
 
@@ -350,7 +413,7 @@ class Atomizer {
 
 module.exports = class CoreStorage {
   constructor (dir) {
-    this.db = new RocksDB(dir)
+    this.db = typeof dir === 'object' ? dir : new RocksDB(dir)
     this.mutex = new RW()
   }
 
@@ -372,7 +435,7 @@ module.exports = class CoreStorage {
     try {
       const b = this.db.write()
       b.tryPut(b4a.from([TL.LOCAL_SEED]), seed)
-      await b.flush()
+      await flushAndDestroy(b)
 
       return true
     } finally {
@@ -420,14 +483,14 @@ module.exports = class CoreStorage {
     return this.db.close()
   }
 
-  atomizer () {
-    return new Atomizer(this.db)
+  atom () {
+    return new Atom(this.db)
   }
 
   async clear () {
     const b = this.db.write()
     b.tryDeleteRange(b4a.from([TL.STORAGE_INFO]), INF)
-    await b.flush()
+    await flushAndDestroy(b)
   }
 
   async has (discoveryKey) {
@@ -495,6 +558,7 @@ class HypercoreStorage {
     this.root = root
     this.db = root.db
     this.dbSnapshot = snapshot
+    this.dbRead = snapshot || this.db
     this.mutex = root.mutex
 
     this.discoveryKey = discoveryKey
@@ -512,8 +576,8 @@ class HypercoreStorage {
     return this.dbSnapshot !== null
   }
 
-  atomizer () {
-    return this.root.atomizer()
+  atom () {
+    return this.root.atom()
   }
 
   dependencyLength () {
@@ -535,14 +599,14 @@ class HypercoreStorage {
     return storage
   }
 
-  async registerBatch (name, head) {
+  async registerBatch (name, head, atom) {
     await this.mutex.write.lock()
 
     const storage = new HypercoreStorage(this.root, this.discoveryKey, this.corePointer, this.dataPointer, null)
 
     try {
       const info = await getStorageInfo(this.db)
-      const write = this.db.write()
+      const write = atom ? atom.createBatch() : this.db.write()
 
       storage.dataPointer = info.free++
 
@@ -556,7 +620,8 @@ class HypercoreStorage {
       batch.setBatchPointer(name, storage.dataPointer)
       if (head.rootHash) batch.setCoreHead(head) // if no root hash its the empty core - no head yet
 
-      await write.flush()
+      if (atom) await atom.flush()
+      else await flushAndDestroy(write)
 
       storage.dependencies = await addDependencies(this.db, storage.dataPointer, head.length)
       return storage
@@ -583,7 +648,7 @@ class HypercoreStorage {
 
   snapshot () {
     assert(this.destroyed === false)
-    const s = new HypercoreStorage(this.root, this.discoveryKey, this.corePointer, this.dataPointer, this.db.snapshot())
+    const s = new HypercoreStorage(this.root, this.discoveryKey, this.corePointer, this.dataPointer, this.dbRead.snapshot())
 
     for (const { data, length } of this.dependencies) s.dependencies.push({ data, length })
 
@@ -616,18 +681,35 @@ class HypercoreStorage {
   createReadBatch (opts) {
     assert(this.destroyed === false)
 
-    const snapshot = this.dbSnapshot
-    return new ReadBatch(this, this.db.read({ snapshot }))
+    return new ReadBatch(this, this.dbRead.read())
   }
 
-  createWriteBatch (atomizer) {
+  createWriteBatch (atom) {
     assert(this.destroyed === false)
 
-    if (atomizer) return new WriteBatch(this, atomizer.createBatch(), atomizer)
+    if (atom) return new WriteBatch(this, atom.createBatch(), atom)
 
     return new WriteBatch(this, this.db.write(), null)
   }
 
+  // helper for atomic flows
+  async createWriteBatchAndLock (atomizer, lock) {
+    assert(this.destroyed === false)
+    if (atomizer) atomizer.enter()
+
+    try {
+      await lock.lock()
+    } catch (err) {
+      if (atomizer) atomizer.exit()
+      throw err
+    }
+
+    const batch = this.createWriteBatch(atomizer)
+    if (atomizer) atomizer.exit()
+
+    return batch
+  }
+
   createBlockStream (opts = {}) {
     assert(this.destroyed === false)
     return createStream(this, createBlockStream, opts)
@@ -636,8 +718,8 @@ class HypercoreStorage {
   createUserDataStream (opts = {}) {
     assert(this.destroyed === false)
 
-    const r = encodeUserDataRange(this.dataPointer, DATA.USER_DATA, this.dbSnapshot, opts)
-    const s = this.db.iterator(r)
+    const r = encodeUserDataRange(this.dataPointer, DATA.USER_DATA, opts)
+    const s = this.dbRead.iterator(r)
     s._readableState.map = mapStreamUserData
     return s
   }
@@ -645,8 +727,8 @@ class HypercoreStorage {
   createTreeNodeStream (opts = {}) {
     assert(this.destroyed === false)
 
-    const r = encodeIndexRange(this.dataPointer, DATA.TREE, this.dbSnapshot, opts)
-    const s = this.db.iterator(r)
+    const r = encodeIndexRange(this.dataPointer, DATA.TREE, opts)
+    const s = this.dbRead.iterator(r)
     s._readableState.map = mapStreamTreeNode
     return s
   }
@@ -654,8 +736,8 @@ class HypercoreStorage {
   createBitfieldPageStream (opts = {}) {
     assert(this.destroyed === false)
 
-    const r = encodeIndexRange(this.dataPointer, DATA.BITFIELD, this.dbSnapshot, opts)
-    const s = this.db.iterator(r)
+    const r = encodeIndexRange(this.dataPointer, DATA.BITFIELD, opts)
+    const s = this.dbRead.iterator(r)
     s._readableState.map = mapStreamBitfieldPage
     return s
   }
@@ -663,7 +745,7 @@ class HypercoreStorage {
   async peekLastTreeNode () {
     assert(this.destroyed === false)
 
-    const last = await this.db.peek(encodeIndexRange(this.dataPointer, DATA.TREE, this.dbSnapshot, { reverse: true }))
+    const last = await this.dbRead.peek(encodeIndexRange(this.dataPointer, DATA.TREE, { reverse: true }))
     if (last === null) return null
     return c.decode(m.TreeNode, last.value)
   }
@@ -671,7 +753,7 @@ class HypercoreStorage {
   async peekLastBitfieldPage () {
     assert(this.destroyed === false)
 
-    const last = await this.db.peek(encodeIndexRange(this.dataPointer, DATA.BITFIELD, this.dbSnapshot, { reverse: true }))
+    const last = await this.dbRead.peek(encodeIndexRange(this.dataPointer, DATA.BITFIELD, { reverse: true }))
     if (last === null) return null
     return mapStreamBitfieldPage(last)
   }
@@ -680,19 +762,19 @@ class HypercoreStorage {
     if (this.destroyed) return
     this.destroyed = true
 
-    if (this.dbSnapshot) this.dbSnapshot.destroy()
+    if (this.dbSnapshot) this.dbSnapshot.close().catch(noop)
     this.dbSnapshot = null
   }
 }
 
 function createStream (storage, createStreamType, opts) {
   return storage.dependencies.length === 0
-    ? createStreamType(storage.db, storage.dbSnapshot, storage.dataPointer, opts)
+    ? createStreamType(storage.db, storage.dataPointer, opts)
     : new DependencyStream(storage, createStreamType, opts)
 }
 
-function createBlockStream (db, snap, data, opts) {
-  const r = encodeIndexRange(data, DATA.BLOCK, snap, opts)
+function createBlockStream (db, data, opts) {
+  const r = encodeIndexRange(data, DATA.BLOCK, opts)
   const s = db.iterator(r)
   s._readableState.map = mapStreamBlock
   return s
@@ -766,8 +848,8 @@ function ensureSlab (size) {
   return SLAB
 }
 
-function encodeIndexRange (pointer, type, snapshot, opts) {
-  const bounded = { snapshot, gt: null, gte: null, lte: null, lt: null, reverse: !!opts.reverse, limit: toLimit(opts.limit) }
+function encodeIndexRange (pointer, type, opts) {
+  const bounded = { gt: null, gte: null, lte: null, lt: null, reverse: !!opts.reverse, limit: toLimit(opts.limit) }
 
   if (opts.gt || opts.gt === 0) bounded.gt = encodeDataIndex(pointer, type, opts.gt)
   else if (opts.gte) bounded.gte = encodeDataIndex(pointer, type, opts.gte)
@@ -780,8 +862,8 @@ function encodeIndexRange (pointer, type, snapshot, opts) {
   return bounded
 }
 
-function encodeUserDataRange (pointer, type, snapshot, opts) {
-  const bounded = { snapshot, gt: null, gte: null, lte: null, lt: null, reverse: !!opts.reverse, limit: toLimit(opts.limit) }
+function encodeUserDataRange (pointer, type, opts) {
+  const bounded = { gt: null, gte: null, lte: null, lt: null, reverse: !!opts.reverse, limit: toLimit(opts.limit) }
 
   if (opts.gt || opts.gt === 0) bounded.gt = encodeUserDataIndex(pointer, type, opts.gt)
   else if (opts.gte) bounded.gte = encodeUserDataIndex(pointer, type, opts.gte)
@@ -904,3 +986,24 @@ function initialiseCoreData (db, { userData } = {}) {
     }
   }
 }
+
+function noop () {}
+
+async function tryFlushAndDestroy (batch) {
+  try {
+    await batch.flush()
+  } catch {}
+
+  batch.destroy()
+}
+
+async function flushAndDestroy (batch) {
+  try {
+    await batch.flush()
+  } catch (err) {
+    batch.destroy()
+    throw err
+  }
+
+  batch.destroy()
+}

package/lib/dependency-stream.js

@@ -93,7 +93,7 @@ module.exports = class DependencyStream extends Readable {
   _nextStream () {
     const { data, gte, lt } = this._streams[this._next++]
 
-    const stream = this.createStream(this.storage.db, this.storage.dbSnapshot, data, {
+    const stream = this.createStream(this.storage.dbRead, data, {
       reverse: this._reverse,
       limit: this._limit,
       gte,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore-storage",
-  "version": "0.0.36",
+  "version": "0.0.38",
   "main": "index.js",
   "files": [
     "index.js",
@@ -22,7 +22,7 @@
     "queue-tick": "^1.0.1",
     "read-write-mutexify": "^2.1.0",
     "resolve-reject-promise": "^1.0.0",
-    "rocksdb-native": "^2.2.0",
+    "rocksdb-native": "^3.0.0",
     "streamx": "^2.20.1"
   },
   "devDependencies": {