hypercore 10.9.2 → 10.10.0

package/README.md

@@ -197,6 +197,36 @@ for await (const data of fullStream) {
 }
 ```
 
+#### `const bs = core.createByteStream([options])`
+
+Make a byte stream to read a range of bytes.
+
+``` js
+// Read the full core
+const fullStream = core.createByteStream()
+
+// Read from byte 3, and from there read 50 bytes
+const partialStream = core.createByteStream({ byteOffset: 3, byteLength: 50 })
+
+// Consume it as an async iterator
+for await (const data of fullStream) {
+  console.log('data:', data)
+}
+
+// Or pipe it somewhere like any stream:
+partialStream.pipe(process.stdout)
+```
+
+`options` include:
+
+``` js
+{
+  byteOffset: 0,
+  byteLength: core.byteLength - options.byteOffset,
+  prefetch: 32
+}
+```
+
 #### `await core.clear(start, [end])`
 
 Clear stored blocks between `start` and `end`, reclaiming storage when possible.
@@ -215,6 +245,10 @@ Truncate the core to a smaller length.
 Per default this will update the fork id of the core to `+ 1`, but you can set the fork id you prefer with the option.
 Note that the fork id should be monotonely incrementing.
 
+#### `await core.purge()`
+
+Purge the hypercore from your storage, completely removing all data.
+
 #### `const hash = await core.treeHash([length])`
 
 Get the Merkle Tree hash of the core at a given length, defaulting to the current length of the core.

package/index.js

@@ -14,7 +14,7 @@ const Core = require('./lib/core')
 const BlockEncryption = require('./lib/block-encryption')
 const Info = require('./lib/info')
 const Download = require('./lib/download')
-const { ReadStream, WriteStream } = require('./lib/streams')
+const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
 const { BAD_ARGUMENT, SESSION_CLOSED, SESSION_NOT_WRITABLE, SNAPSHOT_NOT_AVAILABLE } = require('./lib/errors')
 
 const promises = Symbol.for('hypercore.promises')
@@ -173,13 +173,14 @@ module.exports = class Hypercore extends EventEmitter {
     const directory = storage
     const toLock = opts.unlocked ? null : (opts.lock || 'oplog')
     const pool = opts.pool || (opts.poolSize ? RAF.createPool(opts.poolSize) : null)
+    const rmdir = !!opts.rmdir
 
     return createFile
 
     function createFile (name) {
       const lock = toLock === null ? false : isFile(name, toLock)
       const sparse = isFile(name, 'data') || isFile(name, 'bitfield') || isFile(name, 'tree')
-      return new RAF(name, { directory, lock, sparse, pool: lock ? null : pool })
+      return new RAF(name, { directory, lock, sparse, pool: lock ? null : pool, rmdir })
     }
 
     function isFile (name, n) {
@@ -528,9 +529,16 @@ module.exports = class Hypercore extends EventEmitter {
     for (const s of this.sessions) s.emit('conflict', proof.upgrade.length, proof.fork, proof)
 
     const err = new Error('Two conflicting signatures exist for length ' + proof.upgrade.length)
+    await this._closeAllSessions(err)
+  }
+
+  async _closeAllSessions (err) {
+    // this.sessions modifies itself when a session closes
+    // This way we ensure we indeed iterate over all sessions
+    const sessions = [...this.sessions]
 
     const all = []
-    for (const s of this.sessions) all.push(s.close(err))
+    for (const s of sessions) all.push(s.close(err))
     await Promise.allSettled(all)
   }
 
@@ -742,6 +750,11 @@ module.exports = class Hypercore extends EventEmitter {
     await this.core.clear(start, end)
   }
 
+  async purge () {
+    await this._closeAllSessions(null)
+    await this.core.purge()
+  }
+
   async _get (index, opts) {
     let block
 
@@ -794,6 +807,10 @@ module.exports = class Hypercore extends EventEmitter {
     return new WriteStream(this, opts)
   }
 
+  createByteStream (opts) {
+    return new ByteStream(this, opts)
+  }
+
   download (range) {
     const req = this._download(range)
 

package/lib/core.js

@@ -280,6 +280,25 @@ module.exports = class Core {
     }
   }
 
+  async purge () {
+    return new Promise((resolve, reject) => {
+      let missing = 4
+      let error = null
+
+      this.oplog.storage.unlink(done)
+      this.tree.storage.unlink(done)
+      this.bitfield.storage.unlink(done)
+      this.blocks.storage.unlink(done)
+
+      function done (err) {
+        if (err) error = err
+        if (--missing) return
+        if (error) reject(error)
+        else resolve()
+      }
+    })
+  }
+
   async append (values, auth = this.defaultAuth, hooks = {}) {
     await this._mutex.lock()
 

package/lib/streams.js

@@ -54,3 +54,77 @@ class WriteStream extends Writable {
 }
 
 exports.WriteStream = WriteStream
+
+class ByteStream extends Readable {
+  constructor (core, opts = {}) {
+    super()
+
+    this._core = core
+    this._index = 0
+    this._range = null
+
+    this._byteOffset = opts.byteOffset || 0
+    this._byteLength = typeof opts.byteLength === 'number' ? opts.byteLength : -1
+    this._prefetch = typeof opts.prefetch === 'number' ? opts.prefetch : 32
+
+    this._applyOffset = this._byteOffset > 0
+  }
+
+  _open (cb) {
+    this._openp().then(cb, cb)
+  }
+
+  _read (cb) {
+    this._readp().then(cb, cb)
+  }
+
+  async _openp () {
+    if (this._byteLength === -1) {
+      await this._core.update()
+      this._byteLength = Math.max(this._core.byteLength - this._byteOffset, 0)
+    }
+  }
+
+  async _readp () {
+    let data = null
+
+    if (this._byteLength === 0) {
+      this.push(null)
+      return
+    }
+
+    let relativeOffset = 0
+
+    if (this._applyOffset) {
+      this._applyOffset = false
+
+      const [block, byteOffset] = await this._core.seek(this._byteOffset)
+
+      this._index = block
+      relativeOffset = byteOffset
+    }
+
+    this._predownload(this._index + 1)
+    data = await this._core.get(this._index++)
+
+    if (relativeOffset > 0) data = data.subarray(relativeOffset)
+
+    if (data.byteLength > this._byteLength) data = data.subarray(0, this._byteLength)
+    this._byteLength -= data.byteLength
+
+    this.push(data)
+    if (this._byteLength === 0) this.push(null)
+  }
+
+  _predownload (index) {
+    if (this._range) this._range.destroy()
+    this._range = this._core.download({ start: index, end: index + this._prefetch, linear: true })
+  }
+
+  _destroy (cb) {
+    if (this._range) this._range.destroy()
+    cb(null)
+  }
+}
+
+exports.ByteStream = ByteStream

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.9.2",
+  "version": "10.10.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
@@ -57,6 +57,7 @@
     "hyperswarm": "^4.3.6",
     "random-access-memory": "^6.1.0",
     "random-access-memory-overlay": "^3.0.0",
+    "range-parser": "^1.2.1",
     "standard": "^17.0.0",
     "tmp-promise": "^3.0.2"
   }