hypercore 10.9.1 → 10.10.0

package/README.md

@@ -197,6 +197,36 @@ for await (const data of fullStream) {
 }
 ```
 
+#### `const bs = core.createByteStream([options])`
+
+Make a byte stream to read a range of bytes.
+
+``` js
+// Read the full core
+const fullStream = core.createByteStream()
+
+// Read from byte 3, and from there read 50 bytes
+const partialStream = core.createByteStream({ byteOffset: 3, byteLength: 50 })
+
+// Consume it as an async iterator
+for await (const data of fullStream) {
+  console.log('data:', data)
+}
+
+// Or pipe it somewhere like any stream:
+partialStream.pipe(process.stdout)
+```
+
+`options` include:
+
+``` js
+{
+  byteOffset: 0,
+  byteLength: core.byteLength - options.byteOffset,
+  prefetch: 32
+}
+```
+
 #### `await core.clear(start, [end])`
 
 Clear stored blocks between `start` and `end`, reclaiming storage when possible.
@@ -215,6 +245,10 @@ Truncate the core to a smaller length.
 Per default this will update the fork id of the core to `+ 1`, but you can set the fork id you prefer with the option.
 Note that the fork id should be monotonely incrementing.
 
+#### `await core.purge()`
+
+Purge the hypercore from your storage, completely removing all data.
+
 #### `const hash = await core.treeHash([length])`
 
 Get the Merkle Tree hash of the core at a given length, defaulting to the current length of the core.

package/index.js

@@ -14,7 +14,7 @@ const Core = require('./lib/core')
 const BlockEncryption = require('./lib/block-encryption')
 const Info = require('./lib/info')
 const Download = require('./lib/download')
-const { ReadStream, WriteStream } = require('./lib/streams')
+const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
 const { BAD_ARGUMENT, SESSION_CLOSED, SESSION_NOT_WRITABLE, SNAPSHOT_NOT_AVAILABLE } = require('./lib/errors')
 
 const promises = Symbol.for('hypercore.promises')
@@ -173,13 +173,14 @@ module.exports = class Hypercore extends EventEmitter {
     const directory = storage
     const toLock = opts.unlocked ? null : (opts.lock || 'oplog')
     const pool = opts.pool || (opts.poolSize ? RAF.createPool(opts.poolSize) : null)
+    const rmdir = !!opts.rmdir
 
     return createFile
 
     function createFile (name) {
       const lock = toLock === null ? false : isFile(name, toLock)
       const sparse = isFile(name, 'data') || isFile(name, 'bitfield') || isFile(name, 'tree')
-      return new RAF(name, { directory, lock, sparse, pool: lock ? null : pool })
+      return new RAF(name, { directory, lock, sparse, pool: lock ? null : pool, rmdir })
     }
 
     function isFile (name, n) {
@@ -528,9 +529,16 @@ module.exports = class Hypercore extends EventEmitter {
     for (const s of this.sessions) s.emit('conflict', proof.upgrade.length, proof.fork, proof)
 
     const err = new Error('Two conflicting signatures exist for length ' + proof.upgrade.length)
+    await this._closeAllSessions(err)
+  }
+
+  async _closeAllSessions (err) {
+    // this.sessions modifies itself when a session closes
+    // This way we ensure we indeed iterate over all sessions
+    const sessions = [...this.sessions]
 
     const all = []
-    for (const s of this.sessions) all.push(s.close(err))
+    for (const s of sessions) all.push(s.close(err))
     await Promise.allSettled(all)
   }
 
@@ -742,6 +750,11 @@ module.exports = class Hypercore extends EventEmitter {
     await this.core.clear(start, end)
   }
 
+  async purge () {
+    await this._closeAllSessions(null)
+    await this.core.purge()
+  }
+
   async _get (index, opts) {
     let block
 
@@ -794,6 +807,10 @@ module.exports = class Hypercore extends EventEmitter {
     return new WriteStream(this, opts)
   }
 
+  createByteStream (opts) {
+    return new ByteStream(this, opts)
+  }
+
   download (range) {
     const req = this._download(range)
 

package/lib/bitfield.js

@@ -123,19 +123,24 @@ class BitfieldSegment {
   findFirst (val, position) {
     position = this.tree.skipFirst(!val, position)
 
-    const j = position & (BITS_PER_PAGE - 1)
-    const i = (position - j) / BITS_PER_PAGE
+    let j = position & (BITS_PER_PAGE - 1)
+    let i = (position - j) / BITS_PER_PAGE
 
-    if (i >= PAGES_PER_SEGMENT) return val ? -1 : position
+    if (i >= PAGES_PER_SEGMENT) return -1
+
+    while (i < this.pages.length) {
+      const p = this.pages[i]
 
-    const p = this.pages[i]
+      let index = -1
 
-    let index = -1
+      if (p) index = p.findFirst(val, j)
+      else if (!val) index = j
 
-    if (p) index = p.findFirst(val, j)
-    else if (!val) index = j
+      if (index !== -1) return i * BITS_PER_PAGE + index
 
-    if (index !== -1) return i * BITS_PER_PAGE + index
+      j = 0
+      i++
+    }
 
     return -1
   }
@@ -143,19 +148,24 @@ class BitfieldSegment {
   findLast (val, position) {
     position = this.tree.skipLast(!val, position)
 
-    const j = position & (BITS_PER_PAGE - 1)
-    const i = (position - j) / BITS_PER_PAGE
+    let j = position & (BITS_PER_PAGE - 1)
+    let i = (position - j) / BITS_PER_PAGE
 
     if (i >= PAGES_PER_SEGMENT) return -1
 
-    const p = this.pages[i]
+    while (i >= 0) {
+      const p = this.pages[i]
+
+      let index = -1
 
-    let index = -1
+      if (p) index = p.findLast(val, j)
+      else if (!val) index = j
 
-    if (p) index = p.findLast(val, j)
-    else if (!val) index = j
+      if (index !== -1) return i * BITS_PER_PAGE + index
 
-    if (index !== -1) return i * BITS_PER_PAGE + index
+      j = BITS_PER_PAGE - 1
+      i--
+    }
 
     return -1
   }

package/lib/core.js

@@ -280,6 +280,25 @@ module.exports = class Core {
     }
   }
 
+  async purge () {
+    return new Promise((resolve, reject) => {
+      let missing = 4
+      let error = null
+
+      this.oplog.storage.unlink(done)
+      this.tree.storage.unlink(done)
+      this.bitfield.storage.unlink(done)
+      this.blocks.storage.unlink(done)
+
+      function done (err) {
+        if (err) error = err
+        if (--missing) return
+        if (error) reject(error)
+        else resolve()
+      }
+    })
+  }
+
   async append (values, auth = this.defaultAuth, hooks = {}) {
     await this._mutex.lock()
 

package/lib/remote-bitfield.js

@@ -84,19 +84,24 @@ class RemoteBitfieldSegment {
   findFirst (val, position) {
     position = this.tree.skipFirst(!val, position)
 
-    const j = position & (BITS_PER_PAGE - 1)
-    const i = (position - j) / BITS_PER_PAGE
+    let j = position & (BITS_PER_PAGE - 1)
+    let i = (position - j) / BITS_PER_PAGE
 
-    if (i >= PAGES_PER_SEGMENT) return val ? -1 : position
+    if (i >= PAGES_PER_SEGMENT) return -1
 
-    const p = this.pages[i]
+    while (i < this.pages.length) {
+      const p = this.pages[i]
 
-    let index = -1
+      let index = -1
+
+      if (p) index = p.findFirst(val, j)
+      else if (!val) index = j
 
-    if (p) index = p.findFirst(val, j)
-    else if (!val) index = j
+      if (index !== -1) return i * BITS_PER_PAGE + index
 
-    if (index !== -1) return i * BITS_PER_PAGE + index
+      j = 0
+      i++
+    }
 
     return -1
   }
@@ -104,19 +109,24 @@ class RemoteBitfieldSegment {
   findLast (val, position) {
     position = this.tree.skipLast(!val, position)
 
-    const j = position & (BITS_PER_PAGE - 1)
-    const i = (position - j) / BITS_PER_PAGE
+    let j = position & (BITS_PER_PAGE - 1)
+    let i = (position - j) / BITS_PER_PAGE
 
-    if (i >= PAGES_PER_SEGMENT) return val ? -1 : position
+    if (i >= PAGES_PER_SEGMENT) return -1
 
-    const p = this.pages[i]
+    while (i >= 0) {
+      const p = this.pages[i]
+
+      let index = -1
 
-    let index = -1
+      if (p) index = p.findLast(val, j)
+      else if (!val) index = j
 
-    if (p) index = p.findLast(val, j)
-    else if (!val) index = j
+      if (index !== -1) return i * BITS_PER_PAGE + index
 
-    if (index !== -1) return i * BITS_PER_PAGE + index
+      j = BITS_PER_PAGE - 1
+      i--
+    }
 
     return -1
   }

package/lib/replicator.js

@@ -1674,12 +1674,14 @@ function clampRange (core, r) {
     const start = core.bitfield.firstUnset(r.start)
 
     if (r.end === -1) r.start = start === -1 ? core.tree.length : start
-    else if (start === -1) r.start = r.end
+    else if (start === -1 || start >= r.end) r.start = r.end
     else {
-      const end = core.bitfield.lastUnset(r.end)
-
       r.start = start
-      r.end = end
+
+      const end = core.bitfield.lastUnset(r.end - 1)
+
+      if (end === -1 || start >= end + 1) r.end = r.start
+      else r.end = end + 1
     }
   } else {
     while (r.start < r.end && core.bitfield.get(r.blocks[r.start])) r.start++

package/lib/streams.js

@@ -54,3 +54,77 @@ class WriteStream extends Writable {
 }
 
 exports.WriteStream = WriteStream
+
+class ByteStream extends Readable {
+  constructor (core, opts = {}) {
+    super()
+
+    this._core = core
+    this._index = 0
+    this._range = null
+
+    this._byteOffset = opts.byteOffset || 0
+    this._byteLength = typeof opts.byteLength === 'number' ? opts.byteLength : -1
+    this._prefetch = typeof opts.prefetch === 'number' ? opts.prefetch : 32
+
+    this._applyOffset = this._byteOffset > 0
+  }
+
+  _open (cb) {
+    this._openp().then(cb, cb)
+  }
+
+  _read (cb) {
+    this._readp().then(cb, cb)
+  }
+
+  async _openp () {
+    if (this._byteLength === -1) {
+      await this._core.update()
+      this._byteLength = Math.max(this._core.byteLength - this._byteOffset, 0)
+    }
+  }
+
+  async _readp () {
+    let data = null
+
+    if (this._byteLength === 0) {
+      this.push(null)
+      return
+    }
+
+    let relativeOffset = 0
+
+    if (this._applyOffset) {
+      this._applyOffset = false
+
+      const [block, byteOffset] = await this._core.seek(this._byteOffset)
+
+      this._index = block
+      relativeOffset = byteOffset
+    }
+
+    this._predownload(this._index + 1)
+    data = await this._core.get(this._index++)
+
+    if (relativeOffset > 0) data = data.subarray(relativeOffset)
+
+    if (data.byteLength > this._byteLength) data = data.subarray(0, this._byteLength)
+    this._byteLength -= data.byteLength
+
+    this.push(data)
+    if (this._byteLength === 0) this.push(null)
+  }
+
+  _predownload (index) {
+    if (this._range) this._range.destroy()
+    this._range = this._core.download({ start: index, end: index + this._prefetch, linear: true })
+  }
+
+  _destroy (cb) {
+    if (this._range) this._range.destroy()
+    cb(null)
+  }
+}
+
+exports.ByteStream = ByteStream

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.9.1",
+  "version": "10.10.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
@@ -43,7 +43,7 @@
     "hypercore-crypto": "^3.2.1",
     "is-options": "^1.0.1",
     "protomux": "^3.4.0",
-    "quickbit-universal": "^2.1.0",
+    "quickbit-universal": "^2.1.1",
     "random-access-file": "^4.0.0",
     "random-array-iterator": "^1.0.0",
     "safety-catch": "^1.0.1",
@@ -57,6 +57,7 @@
     "hyperswarm": "^4.3.6",
     "random-access-memory": "^6.1.0",
     "random-access-memory-overlay": "^3.0.0",
+    "range-parser": "^1.2.1",
     "standard": "^17.0.0",
     "tmp-promise": "^3.0.2"
   }