hypercore 10.0.0-alpha.8 → 10.1.0

package/lib/streams.js

@@ -0,0 +1,56 @@
+const { Writable, Readable } = require('streamx')
+
+class ReadStream extends Readable {
+  constructor (core, opts = {}) {
+    super()
+
+    this.core = core
+    this.start = opts.start || 0
+    this.end = typeof opts.end === 'number' ? opts.end : -1
+    this.snapshot = !opts.live && opts.snapshot !== false
+    this.live = !!opts.live
+  }
+
+  _open (cb) {
+    this._openP().then(cb, cb)
+  }
+
+  _read (cb) {
+    this._readP().then(cb, cb)
+  }
+
+  async _openP () {
+    if (this.end === -1) await this.core.update()
+    else await this.core.ready()
+    if (this.snapshot && this.end === -1) this.end = this.core.length
+  }
+
+  async _readP () {
+    const end = this.live ? -1 : (this.end === -1 ? this.core.length : this.end)
+    if (end >= 0 && this.start >= end) {
+      this.push(null)
+      return
+    }
+
+    this.push(await this.core.get(this.start++))
+  }
+}
+
+exports.ReadStream = ReadStream
+
+class WriteStream extends Writable {
+  constructor (core) {
+    super()
+    this.core = core
+  }
+
+  _writev (batch, cb) {
+    this._writevP(batch).then(cb, cb)
+  }
+
+  async _writevP (batch) {
+    await this.core.append(batch)
+  }
+}
+
+exports.WriteStream = WriteStream

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.8",
-  "description": "Hypercore 10",
+  "version": "10.1.0",
+  "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
-    "test": "standard && brittle test/*.js"
+    "test": "standard && node test/all.js",
+    "test:generate": "brittle -r test/all.js test/*.js"
   },
   "repository": {
     "type": "git",
@@ -27,30 +28,35 @@
     "url": "https://github.com/hypercore-protocol/hypercore/issues"
   },
   "homepage": "https://github.com/hypercore-protocol/hypercore#readme",
+  "files": [
+    "index.js",
+    "lib/**.js"
+  ],
   "dependencies": {
-    "@hyperswarm/secret-stream": "^5.0.0",
+    "@hyperswarm/secret-stream": "^6.0.0",
+    "b4a": "^1.1.0",
     "big-sparse-array": "^1.0.2",
-    "codecs": "^2.2.0",
-    "compact-encoding": "^2.5.0",
+    "bits-to-bytes": "^1.3.0",
+    "compact-encoding": "^2.11.0",
     "crc32-universal": "^1.0.1",
+    "events": "^3.3.0",
     "flat-tree": "^1.9.0",
-    "hypercore-crypto": "^2.1.1",
+    "hypercore-crypto": "^3.2.1",
     "is-options": "^1.0.1",
-    "random-access-file": "^2.1.4",
+    "protomux": "^3.2.0",
+    "random-access-file": "^3.2.2",
     "random-array-iterator": "^1.0.0",
     "safety-catch": "^1.0.1",
     "sodium-universal": "^3.0.4",
-    "uint64le": "^1.0.0",
-    "xache": "^1.0.0"
+    "streamx": "^2.12.4",
+    "xache": "^1.1.0"
   },
   "devDependencies": {
-    "brittle": "^1.4.1",
-    "hyperswarm": "next",
-    "random-access-memory": "^3.1.2",
+    "brittle": "^3.0.0",
+    "hyperswarm": "^4.1.1",
+    "random-access-memory": "^5.0.1",
+    "random-access-memory-overlay": "^2.0.0",
     "standard": "^16.0.3",
     "tmp-promise": "^3.0.2"
-  },
-  "optionalDependencies": {
-    "fsctl": "^1.0.0"
   }
 }

package/.github/workflows/test-node.yml

@@ -1,23 +0,0 @@
-name: Build Status
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-jobs:
-  build:
-    strategy:
-      matrix:
-        node-version: [lts/*]
-        os: [ubuntu-latest, macos-latest, windows-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v2
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2
-      with:
-        node-version: ${{ matrix.node-version }}
-    - run: npm install
-    - run: npm test

package/CHANGELOG.md

@@ -1,37 +0,0 @@
-# CHANGELOG
-
-See [UPGRADE.md](UPGRADE.md) for notes on breaking changes for downstream developers.
-
-## Current
-
-## v9.5.0
-
-- Feed close makes the replication detach the channel used for this particular stream.
-
-## v9.4.0
-
-- feed.get accepts onwait hook, that is called if get is waiting for a network peer.
-
-## v9.3.0
-
-- feed.get returns an id, that can be used to cancel a pending get with feed.cancel(getId)
-
-## v9.2.0
-
-- Add `maxBlockSize` on write streams to auto chunk big blocks being written.
-
-## v9.1.0
-
-- Make peer.remoteOpened public ([#268](https://github.com/hypercore-protocol/hypercore/pull/268))
-
-## v9.0.1
-
-- Upgraded standard to v14 with subsequent formatting tweaks
-- createReadStream is up to 8x faster now! ([#261](https://github.com/hypercore-protocol/hypercore/pull/261) by [@tinchoz49](https://github.com/tinchoz49))
-- Fixed benchmarks ([#266](https://github.com/hypercore-protocol/hypercore/pull/266) by [@fwip](https://github.com/fwip))
-
-## v9.0.0
-
-- Ease of use update to signatures, https://github.com/mafintosh/hypercore/issues/260
-- Updates [noise-protocol](https://github.com/emilbayes/noise-protocol) to latest, which uses chacha instead of xchacha and moves avoid from the sodium kx api for better compatability with the rest of the Noise ecosystem.
-- Updates [sodium-native](https://github.com/sodium-friends/sodium-native) from 2 to 3 across the board. 3 uses n-api, so no more builds needed when Node is released.

package/UPGRADE.md

@@ -1,9 +0,0 @@
-# Upgrade Notes
-
-Notes for downstream developers who are upgrading their modules to new, breaking versions of hypercore.
-
-## 9.0.0
-
-- The format of signatures [has been changed](https://github.com/mafintosh/hypercore/issues/260). This is backwards-compatible (v9 can read v8 signatures), but forward-incompatible (v8 cannot read v9 signatures). If a v8 peer replicates with a v9 peer, it will emit a "REMOTE SIGNTURE INVALID" error on the replication stream.
-- The encryption ([NOISE](https://github.com/emilbayes/noise-protocol)) handshake has been changed in an backwards- and forwards-incompatible way. v8 peers can not handshake with v9 peers, and vice-versa. A NOISE-related error is emitted on the replication stream.
-- There is no way (yet) to detect whether a peer is running an incompatible version of hypercore at the replication level. One workaround for downstream developers is to include their own application-level handshake before piping to the replication stream, to communicate a "app protocol version" (maybe "v8" and "v9") and abort the connection if the peer is running an incompatible version.

package/examples/announce.js

@@ -1,19 +0,0 @@
-const Hypercore = require('../')
-const Hyperswarm = require('hyperswarm')
-
-const core = new Hypercore('./source')
-
-start()
-
-async function start () {
-  await core.ready()
-  while (core.length < 1000) {
-    await core.append('block #' + core.length)
-  }
-
-  const swarm = new Hyperswarm()
-  swarm.on('connection', socket => core.replicate(socket))
-  swarm.join(core.discoveryKey, { server: true, client: false })
-
-  console.log('Core:', core.key.toString('hex'))
-}

package/examples/basic.js

@@ -1,10 +0,0 @@
-const Hypercore = require('../')
-
-start()
-
-async function start () {
-  const core = new Hypercore('/tmp/basic')
-  await core.append(['Hello', 'World'])
-  console.log(core)
-  await core.close()
-}

package/examples/http.js

@@ -1,123 +0,0 @@
-const Hypercore = require('../')
-const streamx = require('streamx')
-const replicator = require('@hyperswarm/replicator')
-
-const core = new Hypercore('/tmp/movie')
-
-if (process.argv[2] === 'bench') bench()
-else if (process.argv[2]) importData()
-else start()
-
-class ByteStream extends streamx.Readable {
-  constructor (core, byteOffset, byteLength) {
-    super()
-
-    this.core = core
-    this.byteOffset = byteOffset
-    this.byteLength = byteLength
-    this.index = 0
-    this.range = null
-  }
-
-  async _read (cb) {
-    let data = null
-
-    if (!this.byteLength) {
-      this.push(null)
-      return cb(null)
-    }
-
-    if (this.byteOffset > 0) {
-      const [block, byteOffset] = await core.seek(this.byteOffset)
-      this.byteOffset = 0
-      this.index = block + 1
-      this._select(this.index)
-      data = (await core.get(block)).slice(byteOffset)
-    } else {
-      this._select(this.index + 1)
-      data = await core.get(this.index++)
-    }
-
-    if (data.length >= this.byteLength) {
-      data = data.slice(0, this.byteLength)
-      this.push(data)
-      this.push(null)
-    } else {
-      this.push(data)
-    }
-
-    this.byteLength -= data.length
-
-    cb(null)
-  }
-
-  _select (index) {
-    if (this.range !== null) this.range.destroy(null)
-    this.range = this.core.download({ start: index, end: index + 32, linear: true })
-  }
-
-  _destroy (cb) {
-    if (this.range) this.range.destroy(null)
-    cb(null)
-  }
-}
-
-async function bench () {
-  await core.ready()
-
-  console.time()
-  for (let i = 0; i < core.length; i++) {
-    await core.get(i)
-  }
-  console.timeEnd()
-}
-
-async function start () {
-  const http = require('http')
-  const parse = require('range-parser')
-
-  await core.ready()
-
-  core.on('download', (index) => console.log('Downloaded block #' + index))
-  core.download({ start: 0, end: 1 })
-
-  // hack until we update the replicator
-  core.ready = (cb) => cb(null)
-
-  replicator(core, {
-    discoveryKey: require('crypto').createHash('sha256').update('http').digest(),
-    announce: true,
-    lookup: true
-  })
-
-  http.createServer(function (req, res) {
-    res.setHeader('Content-Type', 'video/x-matroska')
-    res.setHeader('Accept-Ranges', 'bytes')
-
-    let s
-
-    if (req.headers.range) {
-      const range = parse(core.byteLength, req.headers.range)[0]
-      const byteLength = range.end - range.start + 1
-      res.statusCode = 206
-      res.setHeader('Content-Range', 'bytes ' + range.start + '-' + range.end + '/' + core.byteLength)
-      s = new ByteStream(core, range.start, byteLength)
-    } else {
-      s = new ByteStream(core, 0, core.byteLength)
-    }
-
-    res.setHeader('Content-Length', s.byteLength)
-    s.pipe(res, () => {})
-  }).listen(10101)
-}
-
-async function importData () {
-  const fs = require('fs')
-  const rs = fs.createReadStream(process.argv[2])
-
-  for await (const data of rs) {
-    await core.append(data)
-  }
-
-  console.log('done!', core)
-}

package/examples/lookup.js

@@ -1,20 +0,0 @@
-const Hypercore = require('../')
-const Hyperswarm = require('hyperswarm')
-
-const core = new Hypercore('./clone', process.argv[2])
-
-start()
-
-async function start () {
-  await core.ready()
-
-  const swarm = new Hyperswarm()
-  swarm.on('connection', socket => core.replicate(socket))
-  swarm.join(core.discoveryKey, { server: false, client: true })
-
-  console.log((await core.get(42)).toString())
-  console.log((await core.get(142)).toString())
-  console.log((await core.get(511)).toString())
-  console.log((await core.get(512)).toString())
-  console.log((await core.get(513)).toString())
-}

package/lib/extensions.js

@@ -1,76 +0,0 @@
-class Extension {
-  constructor (extensions, name, handlers) {
-    this.extensions = extensions
-    this.name = name
-    this.encoding = handlers.encoding
-    this.destroyed = false
-    // TODO: should avoid the bind here by calling directly on handlers instead?
-    this.onmessage = (handlers.onmessage || noop).bind(handlers)
-    this.onremotesupports = (handlers.onremotesupports || noop).bind(handlers)
-  }
-
-  send (message, peer) {
-    if (this.destroyed) return
-    const ext = peer.extensions.get(this.name)
-    if (ext) ext.send(message)
-  }
-
-  broadcast (message) {
-    if (this.extensions.replicator === null || this.destroyed) return
-    for (const peer of this.extensions.replicator.peers) this.send(message, peer)
-  }
-
-  destroy () {
-    if (this.destroyed) return
-    this.destroyed = true
-    this.extensions.all.delete(this.name)
-    if (this.extensions.replicator === null) return
-    for (const peer of this.extensions.replicator.peers) {
-      const ext = peer.extensions.get(this.name)
-      if (ext) ext.destroy()
-    }
-  }
-}
-
-module.exports = class Extensions {
-  constructor () {
-    this.replicator = null
-    this.all = new Map()
-  }
-
-  [Symbol.iterator] () {
-    return this.all[Symbol.iterator]()
-  }
-
-  attach (replicator) {
-    if (replicator === this.replicator) return
-    this.replicator = replicator
-
-    for (const [name, ext] of this.all) {
-      for (const peer of this.replicator.peers) {
-        peer.registerExtension(name, ext)
-      }
-    }
-  }
-
-  register (name, handlers, ext = new Extension(this, name, handlers)) {
-    if (this.all.has(name)) this.all.get(name).destroy()
-    this.all.set(name, ext)
-
-    if (this.replicator !== null) {
-      for (const peer of this.replicator.peers) {
-        peer.registerExtension(name, ext)
-      }
-    }
-
-    return ext
-  }
-
-  update (peer) {
-    for (const ext of this.all.values()) {
-      peer.registerExtension(ext.name, ext)
-    }
-  }
-}
-
-function noop () {}