@peardrive/hyperblobs 2.9.0-pruned.1

package/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/README.md

@@ -0,0 +1,112 @@
+# hyperblobs (Pruned Edition)
+
+> **This is a fork of [holepunchto/hyperblobs](https://github.com/holepunchto/hyperblobs) for the PearDrive pruned stack.**
+
+Part of the pruned hypercore stack:
+- `@peardrive/hypercore` - Core with `onBlockMissing` hook
+- `@peardrive/hyperblobs` - This package
+- `@peardrive/hyperdrive` - Drive with `pruned` mode
+
+The `onBlockMissing` hook from `@peardrive/hypercore` flows through automatically.
+
+---
+
+A simple blob store for Hypercore.
+
+Each blob is identified by its unique bounds within the Hypercore, e.g. `{ byteOffset: 0, blockOffset: 0, blockLength: 5, byteLength: 327680 }`, which makes them easy to save and retrieve:
+
+```js
+const blobs = new Hyperblobs(core)
+// ID is { byteOffset: 0, blockOffset: 0, blockLength: 1, byteLength: 11 }
+const id = await blobs.put(Buffer.from('hello world', 'utf-8'))
+await blobs.get(id) // Buffer.from('hello world', 'utf-8')
+```
+
+You can also get from start/end bounds within a single blob:
+
+```js
+const blobs = new Hyperblobs(core)
+// ID is { byteOffset: 0, blockOffset: 0, blockLength: 1, byteLength: 11 }
+const id = await blobs.put(Buffer.from('hello world', 'utf-8'))
+await blobs.get(id, { start: 1, length: 2 }) // Buffer.from('el', 'utf-8')
+```
+
+If the blob is large, there's a Streams interface (`createReadStream` and `createWriteStream`) too.
+
+## Installation
+
+```
+npm i hyperblobs
+```
+
+## API
+
+```js
+const Hyperblobs = require('hyperblobs')
+```
+
+#### `const blobs = new Hyperblobs(core, opts)`
+
+Create a new blob store wrapping a single Hypercore.
+
+Options can include:
+
+```js
+{
+  blockSize: 64KB // The block size that will be used when storing large blobs.
+}
+```
+
+#### `const id = await blobs.put(blob, opts)`
+
+Store a new blob. If the blob is large, it will be chunked according to `opts.blockSize` (default 64KB).
+
+Options can include:
+
+```js
+{
+  blockSize: 64KB, // The block size that will be used when storing large blobs.
+  start: 0, // Relative offset to start within the blob
+  end: blob.length - 1, // End offset within the blob (inclusive)
+  length: blob.length, // Number of bytes to read.
+  core // A custom core to write (overrides the default core)
+}
+```
+
+#### `const content = await blobs.get(id, opts)`
+
+Return a complete blob as a `Buffer`.
+
+`id` is the value returned by `put`
+
+Options can include:
+
+```js
+{
+  core, // A custom core to read from (overrides the default core)
+  wait: true, // Wait for block to be downloaded
+  timeout: 0 // Wait at max some milliseconds (0 means no timeout)
+}
+```
+
+#### `await blobs.clear(id, opts)`
+
+Remove a blob from the core.
+
+`opts` are the same as `Hypercore.clear` method.
+
+#### `const stream = blobs.createReadStream(id, opts)`
+
+Create a Readable stream that will yield the `id` blob.
+
+Options match the `get` options.
+
+#### `const stream = blobs.createWriteStream(opts)`
+
+Create a Writable stream that will save a blob.
+
+The corresponding ID will be set on the stream at `stream.id`.
+
+## License
+
+Apache-2.0

package/index.js

@@ -0,0 +1,241 @@
+const mutexify = require('mutexify')
+const b4a = require('b4a')
+
+const { BlockMapReadStream, BlobReadStream, BlobWriteStream } = require('./lib/streams')
+const Monitor = require('./lib/monitor')
+const blockMap = require('./lib/block-map')
+
+const DEFAULT_BLOCK_SIZE = 2 ** 16
+
+class HyperBlobsBatch {
+  constructor(blobs) {
+    this.blobs = blobs
+    this.blocks = []
+    this.bytes = 0
+  }
+
+  ready() {
+    return this.blobs.ready()
+  }
+
+  async put(buffer) {
+    if (!this.blobs.core.opened) await this.blobs.core.ready()
+
+    const blockSize = this.blobs.blockSize
+    const result = {
+      blockOffset: this.blobs.core.length + this.blocks.length,
+      blockLength: 0,
+      byteOffset: this.blobs.core.byteLength + this.bytes,
+      byteLength: 0
+    }
+
+    let offset = 0
+    while (offset < buffer.byteLength) {
+      const blk = buffer.subarray(offset, offset + blockSize)
+      offset += blockSize
+
+      result.blockLength++
+      result.byteLength += blk.byteLength
+      this.bytes += blk.byteLength
+      this.blocks.push(blk)
+    }
+
+    return result
+  }
+
+  async get(id) {
+    if (id.blockOffset < this.blobs.core.length) {
+      return this.blobs.get(id)
+    }
+
+    const bufs = []
+
+    for (
+      let i = id.blockOffset - this.blobs.core.length;
+      i < id.blockOffset + id.blockLength;
+      i++
+    ) {
+      if (i >= this.blocks.length) return null
+      bufs.push(this.blocks[i])
+    }
+
+    return bufs.length === 1 ? bufs[0] : b4a.concat(bufs)
+  }
+
+  async flush() {
+    await this.blobs.core.append(this.blocks)
+    this.blocks = []
+    this.bytes = 0
+  }
+
+  close() {
+    // noop, atm nothing to unlink
+  }
+}
+
+class Hyperblobs {
+  constructor(core, opts = {}) {
+    this.core = core
+    this.blockSize = opts.blockSize || DEFAULT_BLOCK_SIZE
+
+    this._lock = mutexify()
+    this._monitors = new Set()
+
+    this._boundUpdatePeers = this._updatePeers.bind(this)
+    this._boundOnUpload = this._onUpload.bind(this)
+    this._boundOnDownload = this._onDownload.bind(this)
+  }
+
+  get key() {
+    return this.core.key
+  }
+
+  get discoveryKey() {
+    return this.core.discoveryKey
+  }
+
+  get feed() {
+    return this.core
+  }
+
+  get locked() {
+    return this._lock.locked
+  }
+
+  replicate(isInitiator, opts) {
+    return this.core.replicate(isInitiator, opts)
+  }
+
+  ready() {
+    return this.core.ready()
+  }
+
+  close() {
+    return this.core.close()
+  }
+
+  batch() {
+    return new HyperBlobsBatch(this)
+  }
+
+  snapshot() {
+    return new Hyperblobs(this.core.snapshot())
+  }
+
+  async put(blob, opts) {
+    if (!b4a.isBuffer(blob)) blob = b4a.from(blob)
+    const blockSize = (opts && opts.blockSize) || this.blockSize
+
+    const stream = this.createWriteStream(opts)
+    for (let i = 0; i < blob.length; i += blockSize) {
+      stream.write(blob.subarray(i, i + blockSize))
+    }
+    stream.end()
+
+    return new Promise((resolve, reject) => {
+      stream.once('error', reject)
+      stream.once('close', () => resolve(stream.id))
+    })
+  }
+
+  async _getAll(id, opts) {
+    if (id.blockLength === 1) return this.core.get(id.blockOffset, opts)
+
+    const promises = new Array(id.blockLength)
+    for (let i = 0; i < id.blockLength; i++) {
+      promises[i] = this.core.get(id.blockOffset + i, opts)
+    }
+
+    const blocks = await Promise.all(promises)
+    for (let i = 0; i < id.blockLength; i++) {
+      if (blocks[i] === null) return null
+    }
+    return b4a.concat(blocks)
+  }
+
+  async get(id, opts) {
+    if (isAll(id, opts)) return this._getAll(id, opts)
+
+    const res = []
+    try {
+      for await (const block of this.createReadStream(id, opts)) {
+        res.push(block)
+      }
+    } catch (error) {
+      if (error.code === 'BLOCK_NOT_AVAILABLE') return null
+      throw error
+    }
+
+    if (res.length === 1) return res[0]
+    return b4a.concat(res)
+  }
+
+  async clear(id, opts) {
+    if (id.blockMap) {
+      const map = await blockMap.get(this.core, id, { wait: false })
+      if (map) {
+        for (const b of map.blocks) await this.core.clear(b.index, b.index + 1, opts)
+      }
+    }
+    return this.core.clear(id.blockOffset, id.blockOffset + id.blockLength, opts)
+  }
+
+  createReadStream(id, opts) {
+    const core = opts && opts.core ? opts.core : this.core
+    return id.blockMap ? new BlockMapReadStream(core, id, opts) : new BlobReadStream(core, id, opts)
+  }
+
+  createWriteStream(opts) {
+    const core = opts && opts.core ? opts.core : this.core
+    return new BlobWriteStream(core, this._lock, opts)
+  }
+
+  monitor(id) {
+    const monitor = new Monitor(this, id)
+    if (this._monitors.size === 0) this._startListening()
+    this._monitors.add(monitor)
+    return monitor
+  }
+
+  _removeMonitor(mon) {
+    this._monitors.delete(mon)
+    if (this._monitors.size === 0) this._stopListening()
+  }
+
+  _updatePeers() {
+    for (const m of this._monitors) m._updatePeers()
+  }
+
+  _onUpload(index, bytes, from) {
+    for (const m of this._monitors) m._onUpload(index, bytes, from)
+  }
+
+  _onDownload(index, bytes, from) {
+    for (const m of this._monitors) m._onDownload(index, bytes, from)
+  }
+
+  _startListening() {
+    this.core.on('peer-add', this._boundUpdatePeers)
+    this.core.on('peer-remove', this._boundUpdatePeers)
+    this.core.on('upload', this._boundOnUpload)
+    this.core.on('download', this._boundOnDownload)
+  }
+
+  _stopListening() {
+    this.core.off('peer-add', this._boundUpdatePeers)
+    this.core.off('peer-remove', this._boundUpdatePeers)
+    this.core.off('upload', this._boundOnUpload)
+    this.core.off('download', this._boundOnDownload)
+  }
+}
+
+module.exports = Hyperblobs
+
+function isAll(id, opts) {
+  if (id.blockMap) return false
+  if (!opts) return true
+  if (opts.start) return false
+  if (opts.length !== undefined || opts.end !== undefined) return false
+  if (opts.core) return false
+  return true
+}

package/lib/block-map.js

@@ -0,0 +1,157 @@
+const c = require('compact-encoding')
+const b4a = require('b4a')
+const crypto = require('hypercore-crypto')
+
+const block = {
+  preencode(state, m) {
+    c.uint.preencode(state, m.index)
+    c.uint.preencode(state, m.byteLength)
+  },
+  encode(state, m) {
+    c.uint.encode(state, m.index)
+    c.uint.encode(state, m.byteLength)
+  },
+  decode(state) {
+    return {
+      index: c.uint.decode(state),
+      byteLength: c.uint.decode(state)
+    }
+  }
+}
+
+const list = c.array(block)
+
+const map = {
+  preencode(state, m) {
+    c.uint.preencode(state, 0)
+    list.preencode(state, m.blocks)
+  },
+  encode(state, m) {
+    c.uint.encode(state, 0)
+    list.encode(state, m.blocks)
+  },
+  decode(state) {
+    const version = c.uint.decode(state)
+    if (version > 0) throw new Error('Unsupported block map version')
+
+    return {
+      version,
+      blocks: list.decode(state)
+    }
+  }
+}
+
+exports.hash = hashId
+
+function hashId(block) {
+  return b4a.toString(crypto.hash(block), 'hex')
+}
+
+exports.get = getBlockMap
+
+async function inferBlockMap(core, id, opts = {}) {
+  const hashes = !!opts.hashes
+
+  const map = {
+    hashes: hashes ? new Map() : null,
+    blocks: []
+  }
+
+  for (let i = id.blockOffset; i < id.blockOffset + id.blockLength; i++) {
+    const block = await core.get(i)
+    const entry = { index: i, byteLength: block.byteLength }
+    map.blocks.push(entry)
+    if (hashes) map.hashes.set(hashId(block), entry)
+  }
+
+  return map
+}
+
+async function getBlockMap(core, id, opts = {}) {
+  if (!id.blockMap) return inferBlockMap(core, id, opts)
+
+  if (id.blockLength > 64) {
+    throw new Error('Block map is too large')
+  }
+
+  const hashes = !!opts.hashes
+
+  const map = {
+    hashes: hashes ? new Map() : null,
+    blocks: null
+  }
+
+  const promises = []
+
+  for (let i = id.blockOffset; i < id.blockOffset + id.blockLength; i++) {
+    promises.push(core.get(i))
+  }
+
+  const buffers = await Promise.all(promises)
+  const m = decodeBlockMap(buffers)
+  if (!m) return null
+
+  map.blocks = m.blocks
+
+  if (hashes && !core.writable) {
+    const blocks = []
+    for (let i = 0; i < map.blocks.length; i++) blocks.push(map.blocks[i].index)
+    core.download({ blocks })
+  }
+
+  if (hashes) {
+    for (let i = 0; i < map.blocks.length; i++) {
+      const b = map.blocks[i]
+      const block = await core.get(b.index)
+      if (block === null) return null
+      map.hashes.set(hashId(block), b)
+    }
+  }
+
+  return map
+}
+
+exports.encode = encodeBlockMap
+
+function encodeBlockMap(header) {
+  const result = []
+
+  for (let i = 0; i < header.blocks.length; i += 8192) {
+    const blocks =
+      i === 0 && header.blocks.length < 8192 ? header.blocks : header.blocks.slice(i, i + 8192)
+
+    const state = { start: 0, end: 0, buffer: null }
+    const m = { version: 0, blocks }
+
+    map.preencode(state, m)
+    state.buffer = b4a.allocUnsafe(state.end)
+    map.encode(state, m)
+
+    result.push(state.buffer)
+  }
+
+  return result
+}
+
+exports.decode = decodeBlockMap
+
+function decodeBlockMap(buffers) {
+  const result = {
+    version: 0,
+    blocks: null
+  }
+
+  for (let i = 0; i < buffers.length; i++) {
+    if (!buffers[i]) return null
+
+    const state = { start: 0, end: buffers[i].byteLength, buffer: buffers[i] }
+    const r = map.decode(state)
+
+    result.version = r.version
+
+    if (result.blocks) result.blocks.push(...r.blocks)
+    else result.blocks = r.blocks
+  }
+
+  return result
+}

package/lib/monitor.js

@@ -0,0 +1,92 @@
+const EventEmitter = require('events')
+const speedometer = require('speedometer')
+
+module.exports = class Monitor extends EventEmitter {
+  constructor(blobs, id) {
+    super()
+
+    if (!id) throw new Error('id is required')
+
+    this.blobs = blobs
+    this.id = id
+    this.peers = 0
+    this.uploadSpeedometer = null
+    this.downloadSpeedometer = null
+
+    const stats = {
+      startTime: 0,
+      percentage: 0,
+      peers: 0,
+      speed: 0,
+      blocks: 0,
+      totalBytes: 0, // local + bytes loaded during monitoring
+      monitoringBytes: 0, // bytes loaded during monitoring
+      targetBytes: 0,
+      targetBlocks: 0
+    }
+
+    this.uploadStats = { ...stats }
+    this.downloadStats = { ...stats }
+    this.uploadStats.targetBytes = this.downloadStats.targetBytes = this.id.byteLength
+    this.uploadStats.targetBlocks = this.downloadStats.targetBlocks = this.id.blockLength
+    this.uploadStats.peers = this.downloadStats.peers = this.peers = this.blobs.core.peers.length
+
+    this.uploadSpeedometer = speedometer()
+    this.downloadSpeedometer = speedometer()
+
+    // Handlers
+  }
+
+  // just an alias
+  destroy() {
+    return this.close()
+  }
+
+  close() {
+    this.blobs._removeMonitor(this)
+  }
+
+  _onUpload(index, bytes, from) {
+    this._updateStats(this.uploadSpeedometer, this.uploadStats, index, bytes, from)
+  }
+
+  _onDownload(index, bytes, from) {
+    this._updateStats(this.downloadSpeedometer, this.downloadStats, index, bytes, from)
+  }
+
+  _updatePeers() {
+    this.uploadStats.peers = this.downloadStats.peers = this.peers = this.blobs.core.peers.length
+    this.emit('update')
+  }
+
+  _updateStats(speed, stats, index, bytes) {
+    if (this.closing) return
+    if (!isWithinRange(index, this.id)) return
+
+    if (!stats.startTime) stats.startTime = Date.now()
+
+    stats.speed = speed(bytes)
+    stats.blocks++
+    stats.totalBytes += bytes
+    stats.monitoringBytes += bytes
+    stats.percentage = toFixed((stats.blocks / stats.targetBlocks) * 100)
+
+    this.emit('update')
+  }
+
+  downloadSpeed() {
+    return this.downloadSpeedometer ? this.downloadSpeedometer() : 0
+  }
+
+  uploadSpeed() {
+    return this.uploadSpeedometer ? this.uploadSpeedometer() : 0
+  }
+}
+
+function isWithinRange(index, { blockOffset, blockLength }) {
+  return index >= blockOffset && index < blockOffset + blockLength
+}
+
+function toFixed(n) {
+  return Math.round(n * 100) / 100
+}

package/lib/prefetcher.js

@@ -0,0 +1,67 @@
+// should move to hypercore itself
+
+const MAX_READAHEAD_TARGET = 0.05 // aim to buffer 5% always
+
+module.exports = class Prefetcher {
+  constructor(core, { max = 64, start = 0, end = core.length, linear = true } = {}) {
+    this.core = core
+    this.max = max
+    this.range = null
+    this.startBound = start
+    this.endBound = end
+    this.maxReadAhead = Math.max(max * 2, Math.floor((end - start) * MAX_READAHEAD_TARGET))
+
+    this.start = start
+    this.end = start
+    this.linear = linear
+    this.missing = 0
+
+    this._ondownloadBound = this._ondownload.bind(this)
+    this.core.on('download', this._ondownloadBound)
+  }
+
+  _ondownload(index) {
+    if (this.range && index < this.end && this.start <= index) {
+      this.missing--
+      this._update()
+    }
+  }
+
+  destroy() {
+    this.core.off('download', this._ondownloadBound)
+    if (this.range) this.range.destroy()
+    this.range = null
+    this.max = 0
+  }
+
+  update(position) {
+    this.start = position
+    if (!this.range) this._update()
+  }
+
+  _update() {
+    if (this.missing >= this.max) return
+    if (this.range) this.range.destroy()
+
+    let end = this.end
+
+    while (end < this.endBound && this.missing < this.max) {
+      end = this.core.core.bitfield.firstUnset(end) + 1
+      if (end >= this.endBound) break
+      this.missing++
+    }
+
+    if (end > this.start + this.maxReadAhead) end = this.start + this.maxReadAhead
+    if (end >= this.endBound) end = this.endBound
+
+    this.end = end
+
+    if (this.start >= this.end) return
+
+    this.range = this.core.download({
+      start: this.start,
+      end: this.end,
+      linear: this.linear
+    })
+  }
+}

package/lib/streams.js

@@ -0,0 +1,343 @@
+const { Readable, Writable } = require('streamx')
+const { BLOCK_NOT_AVAILABLE } = require('hypercore-errors')
+const Prefetcher = require('./prefetcher')
+const blockMap = require('./block-map')
+
+class BlobWriteStream extends Writable {
+  constructor(core, lock, opts = {}) {
+    super(opts)
+    this.id = { blockOffset: 0, byteOffset: 0, blockLength: 0, byteLength: 0 }
+    this.core = core
+
+    this._dedup = !!opts.dedup
+    this._blob = opts.blob || null
+    this._hashes = null
+    this._addBlockMap = !!opts.blockMap || this._dedup
+    this._blockMap = this._addBlockMap ? { version: 0, blocks: [] } : null
+    this._lock = lock
+    this._release = null
+    this._batch = []
+
+    if (this._addBlockMap) this.id.blockMap = true
+  }
+
+  async _openp() {
+    await this.core.ready()
+    const release = await new Promise((resolve) => this._lock(resolve))
+
+    this._release = release
+    this.id.byteOffset = this.core.byteLength
+    this.id.blockOffset = this.core.length
+
+    if (!this._dedup) return
+
+    if (this._blob) {
+      const map = await blockMap.get(this.core, this._blob, { hashes: true })
+      this._hashes = map.hashes
+    } else {
+      this._hashes = new Map()
+    }
+  }
+
+  _open(cb) {
+    this._openp().then(cb, cb)
+  }
+
+  async _finalp() {
+    await this._append()
+
+    if (this._blockMap) {
+      const buffers = await blockMap.encode(this._blockMap)
+      this.id.blockOffset = this.core.length
+      this.id.byteOffset = this.core.byteLength
+      await this.core.append(buffers)
+    }
+
+    this.id.blockLength = this.core.length - this.id.blockOffset
+    this.id.byteLength = this.core.byteLength - this.id.byteOffset
+  }
+
+  _final(cb) {
+    this._finalp().then(cb, cb)
+  }
+
+  _destroy(cb) {
+    if (this._release) this._release()
+    cb(null)
+  }
+
+  async _append() {
+    if (!this._batch.length) return
+
+    const batch = this._batch
+    this._batch = []
+
+    await this.core.append(batch)
+  }
+
+  _write(data, cb) {
+    let dup = false
+
+    if (this._blockMap) {
+      let entry = {
+        index: this.core.length + this._batch.length,
+        byteLength: data.byteLength
+      }
+
+      if (this._hashes) {
+        const id = blockMap.hash(data)
+        const existing = this._hashes.get(id)
+
+        if (existing) {
+          entry = existing
+          dup = true
+        } else {
+          this._hashes.set(id, entry)
+        }
+      }
+
+      this._blockMap.blocks.push(entry)
+    }
+
+    if (dup) return cb()
+
+    this._batch.push(data)
+
+    if (this._batch.length >= 16) {
+      this._append().then(cb, cb)
+      return
+    }
+
+    return cb()
+  }
+}
+
+class BlockMapReadStream extends Readable {
+  constructor(core, id, opts = {}) {
+    super(opts)
+    this.id = id
+    this.core = core.session({ wait: opts.wait, timeout: opts.timeout })
+
+    const noPrefetch = opts.wait === false || opts.prefetch === false || !core.core
+    const start = opts.start || 0
+    const end =
+      opts.end === undefined ? (opts.length === undefined ? -1 : start + opts.length) : opts.end + 1
+
+    this._blockMap = null
+
+    this._rangeStart = start
+    this._rangeEnd = end
+
+    this._startIndex = 0
+    this._startOffset = 0
+    this._endIndex = -1
+    this._endOffset = -1
+
+    this._range = null
+    this._noPrefetch = noPrefetch
+  }
+
+  async _openp() {
+    this._blockMap = await blockMap.get(this.core, this.id)
+
+    const [startIndex, startOffset, endIndex, endLength] = seekBlockMap(
+      this._blockMap,
+      this._rangeStart,
+      this._rangeEnd
+    )
+
+    this._startIndex = startIndex
+    this._startOffset = startOffset
+    this._endIndex = endIndex
+    this._endOffset = endLength
+
+    if (this._endIndex === -1) {
+      this._endIndex = this._blockMap.blocks.length
+      this._endOffset = 0
+    }
+  }
+
+  _open(cb) {
+    this._openp().then(cb, cb)
+  }
+
+  _predestroy() {
+    if (this._range) this._range.destroy()
+    this.core.close().then(noop, noop)
+  }
+
+  _destroy(cb) {
+    if (this._range) this._range.destroy()
+    this.core.close().then(cb, cb)
+  }
+
+  _prefetch(index) {
+    const blocks = []
+    for (; index < this._endIndex; index++) blocks.push(this._blockMap.blocks[index])
+    this._range = this.core.download({ blocks })
+  }
+
+  async _readp() {
+    if (this._startIndex >= this._endIndex) {
+      this.push(null)
+      return
+    }
+
+    let block = null
+
+    const index = this._startIndex++
+    const b = this._blockMap.blocks[index]
+
+    if (!this._range && !this._noPrefetch) {
+      block = await this.core.get(b.index, { wait: false })
+      if (!block) this._prefetch(index)
+    }
+
+    if (!block) {
+      block = await this.core.get(b.index)
+    }
+
+    if (!block) throw BLOCK_NOT_AVAILABLE()
+
+    if (this._startOffset) {
+      block = block.subarray(this._startOffset)
+      this._startOffset = 0
+    }
+
+    if (this._startIndex === this._endIndex && this._endOffset) {
+      block = block.subarray(0, block.byteLength - this._endOffset)
+    }
+
+    this.push(block)
+  }
+
+  _read(cb) {
+    this._readp().then(cb, cb)
+  }
+}
+
+class BlobReadStream extends Readable {
+  constructor(core, id, opts = {}) {
+    super(opts)
+    this.id = id
+    this.core = core.session({ wait: opts.wait, timeout: opts.timeout })
+
+    const start = id.blockOffset
+    const end = id.blockOffset + id.blockLength
+    const noPrefetch = opts.wait === false || opts.prefetch === false || !core.core
+
+    this._prefetch = noPrefetch
+      ? null
+      : new Prefetcher(this.core, { max: opts.prefetch, start, end })
+    this._lastPrefetch = null
+
+    this._pos = opts.start !== undefined ? id.byteOffset + opts.start : id.byteOffset
+
+    if (opts.length !== undefined) this._end = this._pos + opts.length
+    else if (opts.end !== undefined) this._end = id.byteOffset + opts.end + 1
+    else this._end = id.byteOffset + id.byteLength
+
+    this._index = 0
+    this._relativeOffset = 0
+    this._bytesRead = 0
+  }
+
+  async _openp() {
+    if (this._pos === this.id.byteOffset) {
+      this._index = this.id.blockOffset
+      this._relativeOffset = 0
+      return
+    }
+
+    const result = await this.core.seek(this._pos, {
+      start: this.id.blockOffset,
+      end: this.id.blockOffset + this.id.blockLength
+    })
+
+    if (!result) throw BLOCK_NOT_AVAILABLE()
+
+    this._index = result[0]
+    this._relativeOffset = result[1]
+  }
+
+  _open(cb) {
+    this._openp().then(cb, cb)
+  }
+
+  _predestroy() {
+    if (this._prefetch) this._prefetch.destroy()
+    this.core.close().then(noop, noop)
+  }
+
+  _destroy(cb) {
+    if (this._prefetch) this._prefetch.destroy()
+    this.core.close().then(cb, cb)
+  }
+
+  async _readp() {
+    if (this._pos >= this._end) {
+      this.push(null)
+      return
+    }
+
+    if (this._prefetch) this._prefetch.update(this._index)
+
+    let block = await this.core.get(this._index)
+    if (!block) throw BLOCK_NOT_AVAILABLE()
+
+    const remainder = this._end - this._pos
+    if (this._relativeOffset || remainder < block.length) {
+      block = block.subarray(this._relativeOffset, this._relativeOffset + remainder)
+    }
+
+    this._index++
+    this._relativeOffset = 0
+    this._pos += block.length
+    this._bytesRead += block.length
+
+    this.push(block)
+  }
+
+  _read(cb) {
+    this._readp().then(cb, cb)
+  }
+}
+
+module.exports = {
+  BlockMapReadStream,
+  BlobReadStream,
+  BlobWriteStream
+}
+
+function noop() {}
+
+function seekBlockMap(map, start, end) {
+  let s = -1
+  let so = -1
+  let e = -1
+  let eo = -1
+
+  for (let i = 0; i < map.blocks.length; i++) {
+    const b = map.blocks[i]
+
+    if (s === -1) {
+      if (start < b.byteLength) {
+        s = i
+        so = start
+      } else {
+        start -= b.byteLength
+      }
+    }
+
+    if (e === -1 && end > -1) {
+      if (end <= b.byteLength) {
+        e = i + 1
+        eo = b.byteLength - end
+      } else {
+        end -= b.byteLength
+      }
+    }
+  }
+
+  return [s, so, e, eo]
+}

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "@peardrive/hyperblobs",
+  "version": "2.9.0-pruned.1",
+  "description": "A blob store for Hypercore",
+  "main": "index.js",
+  "scripts": {
+    "format": "prettier --write .",
+    "lint": "prettier --check . && lunte",
+    "test": "npm run test:node && npm run test:bare",
+    "test:node": "brittle-node test/*.js",
+    "test:bare": "brittle-bare test/*.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/holepunchto/hyperblobs.git"
+  },
+  "keywords": [
+    "hypercore",
+    "blob",
+    "store"
+  ],
+  "author": "Andrew Osheroff <andrewosh@gmail.com>",
+  "license": "Apache-2.0",
+  "bugs": {
+    "url": "https://github.com/holepunchto/hyperblobs/issues"
+  },
+  "homepage": "https://github.com/holepunchto/hyperblobs#readme",
+  "files": [
+    "index.js",
+    "lib/**.js"
+  ],
+  "imports": {
+    "events": {
+      "bare": "bare-events",
+      "default": "events"
+    }
+  },
+  "dependencies": {
+    "b4a": "^1.6.1",
+    "bare-events": "^2.5.0",
+    "compact-encoding": "^2.18.0",
+    "hypercore-crypto": "^3.6.1",
+    "hypercore-errors": "^1.1.1",
+    "mutexify": "^1.4.0",
+    "speedometer": "^1.1.0",
+    "streamx": "^2.13.2"
+  },
+  "devDependencies": {
+    "brittle": "^3.1.0",
+    "hypercore": "^11.0.0",
+    "lunte": "^1.0.0",
+    "prettier": "^3.6.2",
+    "prettier-config-holepunch": "^2.0.0"
+  }
+}