hypercore 10.9.2 → 10.11.0

package/README.md

@@ -197,7 +197,37 @@ for await (const data of fullStream) {
 }
 ```
 
-#### `await core.clear(start, [end])`
+#### `const bs = core.createByteStream([options])`
+
+Make a byte stream to read a range of bytes.
+
+``` js
+// Read the full core
+const fullStream = core.createByteStream()
+
+// Read from byte 3, and from there read 50 bytes
+const partialStream = core.createByteStream({ byteOffset: 3, byteLength: 50 })
+
+// Consume it as an async iterator
+for await (const data of fullStream) {
+  console.log('data:', data)
+}
+
+// Or pipe it somewhere like any stream:
+partialStream.pipe(process.stdout)
+```
+
+`options` include:
+
+``` js
+{
+  byteOffset: 0,
+  byteLength: core.byteLength - options.byteOffset,
+  prefetch: 32
+}
+```
+
+#### `const cleared = await core.clear(start, [end], [options])`
 
 Clear stored blocks between `start` and `end`, reclaiming storage when possible.
 
@@ -208,6 +238,13 @@ await core.clear(0, 10) // clear block 0-10 from your local cache
 
 The core will also gossip to peers it is connected to, that is no longer has these blocks.
 
+`options` include:
+```js
+{
+  diff: false // Returned `cleared` bytes object is null unless you enable this
+}
+```
+
 #### `await core.truncate(newLength, [forkId])`
 
 Truncate the core to a smaller length.
@@ -215,6 +252,10 @@ Truncate the core to a smaller length.
 Per default this will update the fork id of the core to `+ 1`, but you can set the fork id you prefer with the option.
 Note that the fork id should be monotonely incrementing.
 
+#### `await core.purge()`
+
+Purge the hypercore from your storage, completely removing all data.
+
 #### `const hash = await core.treeHash([length])`
 
 Get the Merkle Tree hash of the core at a given length, defaulting to the current length of the core.

package/index.js

@@ -14,7 +14,7 @@ const Core = require('./lib/core')
 const BlockEncryption = require('./lib/block-encryption')
 const Info = require('./lib/info')
 const Download = require('./lib/download')
-const { ReadStream, WriteStream } = require('./lib/streams')
+const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
 const { BAD_ARGUMENT, SESSION_CLOSED, SESSION_NOT_WRITABLE, SNAPSHOT_NOT_AVAILABLE } = require('./lib/errors')
 
 const promises = Symbol.for('hypercore.promises')
@@ -173,13 +173,14 @@ module.exports = class Hypercore extends EventEmitter {
     const directory = storage
     const toLock = opts.unlocked ? null : (opts.lock || 'oplog')
     const pool = opts.pool || (opts.poolSize ? RAF.createPool(opts.poolSize) : null)
+    const rmdir = !!opts.rmdir
 
     return createFile
 
     function createFile (name) {
       const lock = toLock === null ? false : isFile(name, toLock)
       const sparse = isFile(name, 'data') || isFile(name, 'bitfield') || isFile(name, 'tree')
-      return new RAF(name, { directory, lock, sparse, pool: lock ? null : pool })
+      return new RAF(name, { directory, lock, sparse, pool: lock ? null : pool, rmdir })
     }
 
     function isFile (name, n) {
@@ -528,9 +529,16 @@ module.exports = class Hypercore extends EventEmitter {
     for (const s of this.sessions) s.emit('conflict', proof.upgrade.length, proof.fork, proof)
 
     const err = new Error('Two conflicting signatures exist for length ' + proof.upgrade.length)
+    await this._closeAllSessions(err)
+  }
+
+  async _closeAllSessions (err) {
+    // this.sessions modifies itself when a session closes
+    // This way we ensure we indeed iterate over all sessions
+    const sessions = [...this.sessions]
 
     const all = []
-    for (const s of this.sessions) all.push(s.close(err))
+    for (const s of sessions) all.push(s.close(err))
     await Promise.allSettled(all)
   }
 
@@ -737,9 +745,18 @@ module.exports = class Hypercore extends EventEmitter {
       end = start + 1
     }
 
-    if (start >= end) return
+    const cleared = (opts && opts.diff) ? { blocks: 0 } : null
+
+    if (start >= end) return cleared
+
+    await this.core.clear(start, end, cleared)
 
-    await this.core.clear(start, end)
+    return cleared
+  }
+
+  async purge () {
+    await this._closeAllSessions(null)
+    await this.core.purge()
   }
 
   async _get (index, opts) {
@@ -794,6 +811,10 @@ module.exports = class Hypercore extends EventEmitter {
     return new WriteStream(this, opts)
   }
 
+  createByteStream (opts) {
+    return new ByteStream(this, opts)
+  }
+
   download (range) {
     const req = this._download(range)
 

package/lib/core.js

@@ -5,6 +5,7 @@ const Mutex = require('./mutex')
 const MerkleTree = require('./merkle-tree')
 const BlockStore = require('./block-store')
 const Bitfield = require('./bitfield')
+const Info = require('./info')
 const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE } = require('./errors')
 const m = require('./messages')
 
@@ -238,7 +239,7 @@ module.exports = class Core {
     }
   }
 
-  async clear (start, end) {
+  async clear (start, end, cleared) {
     await this._mutex.lock()
 
     try {
@@ -265,13 +266,20 @@ module.exports = class Core {
       end = this.bitfield.firstSet(end)
 
       if (end === -1) end = this.tree.length
+      if (start >= end || start >= this.tree.length) return
 
       const offset = await this.tree.byteOffset(start * 2)
       const [byteEnd, byteEndLength] = await this.tree.byteRange((end - 1) * 2)
       const length = (byteEnd + byteEndLength) - offset
 
+      const before = cleared ? await Info.bytesUsed(this.blocks.storage) : null
+
       await this.blocks.clear(offset, length)
 
+      const after = cleared ? await Info.bytesUsed(this.blocks.storage) : null
+
+      if (cleared) cleared.blocks = Math.max(before - after, 0)
+
       this.onupdate(0, entry.bitfield, null, null)
 
       if (this._shouldFlush()) await this._flushOplog()
@@ -280,6 +288,25 @@ module.exports = class Core {
     }
   }
 
+  async purge () {
+    return new Promise((resolve, reject) => {
+      let missing = 4
+      let error = null
+
+      this.oplog.storage.unlink(done)
+      this.tree.storage.unlink(done)
+      this.bitfield.storage.unlink(done)
+      this.blocks.storage.unlink(done)
+
+      function done (err) {
+        if (err) error = err
+        if (--missing) return
+        if (error) reject(error)
+        else resolve()
+      }
+    })
+  }
+
   async append (values, auth = this.defaultAuth, hooks = {}) {
     await this._mutex.lock()
 

package/lib/info.js

@@ -27,27 +27,27 @@ module.exports = class Info {
     const { oplog, tree, blocks, bitfield } = session.core
     try {
       return {
-        oplog: await bytesUsed(oplog.storage),
-        tree: await bytesUsed(tree.storage),
-        blocks: await bytesUsed(blocks.storage),
-        bitfield: await bytesUsed(bitfield.storage)
+        oplog: await Info.bytesUsed(oplog.storage),
+        tree: await Info.bytesUsed(tree.storage),
+        blocks: await Info.bytesUsed(blocks.storage),
+        bitfield: await Info.bytesUsed(bitfield.storage)
       }
     } catch {
       return null
     }
+  }
 
-    function bytesUsed (file) {
-      return new Promise((resolve, reject) => {
-        file.stat((err, st) => {
-          if (err) {
-            resolve(0) // prob just file not found (TODO, improve)
-          } else if (typeof st.blocks !== 'number') {
-            reject(new Error('cannot determine bytes used'))
-          } else {
-            resolve(st.blocks * 512)
-          }
-        })
+  static bytesUsed (file) {
+    return new Promise((resolve, reject) => {
+      file.stat((err, st) => {
+        if (err) {
+          resolve(0) // prob just file not found (TODO, improve)
+        } else if (typeof st.blocks !== 'number') {
+          reject(new Error('cannot determine bytes used'))
+        } else {
+          resolve(st.blocks * 512)
+        }
       })
-    }
+    })
   }
 }

package/lib/streams.js

@@ -54,3 +54,77 @@ class WriteStream extends Writable {
 }
 
 exports.WriteStream = WriteStream
+
+class ByteStream extends Readable {
+  constructor (core, opts = {}) {
+    super()
+
+    this._core = core
+    this._index = 0
+    this._range = null
+
+    this._byteOffset = opts.byteOffset || 0
+    this._byteLength = typeof opts.byteLength === 'number' ? opts.byteLength : -1
+    this._prefetch = typeof opts.prefetch === 'number' ? opts.prefetch : 32
+
+    this._applyOffset = this._byteOffset > 0
+  }
+
+  _open (cb) {
+    this._openp().then(cb, cb)
+  }
+
+  _read (cb) {
+    this._readp().then(cb, cb)
+  }
+
+  async _openp () {
+    if (this._byteLength === -1) {
+      await this._core.update()
+      this._byteLength = Math.max(this._core.byteLength - this._byteOffset, 0)
+    }
+  }
+
+  async _readp () {
+    let data = null
+
+    if (this._byteLength === 0) {
+      this.push(null)
+      return
+    }
+
+    let relativeOffset = 0
+
+    if (this._applyOffset) {
+      this._applyOffset = false
+
+      const [block, byteOffset] = await this._core.seek(this._byteOffset)
+
+      this._index = block
+      relativeOffset = byteOffset
+    }
+
+    this._predownload(this._index + 1)
+    data = await this._core.get(this._index++)
+
+    if (relativeOffset > 0) data = data.subarray(relativeOffset)
+
+    if (data.byteLength > this._byteLength) data = data.subarray(0, this._byteLength)
+    this._byteLength -= data.byteLength
+
+    this.push(data)
+    if (this._byteLength === 0) this.push(null)
+  }
+
+  _predownload (index) {
+    if (this._range) this._range.destroy()
+    this._range = this._core.download({ start: index, end: index + this._prefetch, linear: true })
+  }
+
+  _destroy (cb) {
+    if (this._range) this._range.destroy()
+    cb(null)
+  }
+}
+
+exports.ByteStream = ByteStream

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.9.2",
+  "version": "10.11.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
@@ -57,6 +57,7 @@
     "hyperswarm": "^4.3.6",
     "random-access-memory": "^6.1.0",
     "random-access-memory-overlay": "^3.0.0",
+    "range-parser": "^1.2.1",
     "standard": "^17.0.0",
     "tmp-promise": "^3.0.2"
   }