hypercore 10.0.0 → 10.2.1

package/README.md

@@ -83,12 +83,31 @@ valueEncodings will be applied to individually blocks, even if you append batche
 Append a block of data (or an array of blocks) to the core.
 Returns the new length and byte length of the core.
 
+``` js
+// simple call append with a new block of data
+await core.append(Buffer.from('I am a block of data'))
+
+// pass an array to append multiple blocks as a batch
+await core.append([Buffer.from('batch block 1'), Buffer.from('batch block 2')])
+```
+
 #### `const block = await core.get(index, [options])`
 
 Get a block of data.
 If the data is not available locally this method will prioritize and wait for the data to be downloaded.
 
-Options include
+``` js
+// get block #42
+const block = await core.get(42)
+
+// get block #43, but only wait 5s
+const blockIfFast = await core.get(43, { timeout: 5000 })
+
+// get block #44, but only if we have it locally
+const blockLocal = await core.get(44, { wait: false })
+```
+
+Additional options include
 
 ``` js
 {
@@ -99,20 +118,52 @@ Options include
 }
 ```
 
-#### `await core.truncate(newLength, [forkId])`
+#### `const updated = await core.update()`
 
-Truncate the core to a smaller length.
+Wait for the core to try and find a signed update to it's length.
+Does not download any data from peers except for a proof of the new core length.
 
-Per default this will update the fork id of the core to `+ 1`, but you can set the fork id you prefer with the option.
-Note that the fork id should be monotonely incrementing.
+``` js
+const updated = await core.update()
 
-#### `const hash = await core.treeHash([length])`
+console.log('core was updated?', updated, 'length is', core.length)
+```
 
-Get the Merkle Tree hash of the core at a given length, defaulting to the current length of the core.
+#### `const [index, relativeOffset] = await core.seek(byteOffset)`
+
+Seek to a byte offset.
+
+Returns `[index, relativeOffset]`, where `index` is the data block the byteOffset is contained in and `relativeOffset` is
+the relative byte offset in the data block.
+
+``` js
+await core.append([Buffer.from('abc'), Buffer.from('d'), Buffer.from('efg')])
+
+const first = await core.seek(1) // returns [0, 1]
+const second = await core.seek(3) // returns [1, 0]
+const third = await core.seek(5) // returns [2, 1]
+```
 
 #### `const stream = core.createReadStream([options])`
 
-Make a read stream. Options include:
+Make a read stream to read a range of data out at once.
+
+``` js
+// read the full core
+const fullStream = core.createReadStream()
+
+// read from block 10-15
+const partialStream = core.createReadStream({ start: 10, end: 15 })
+
+// pipe the stream somewhere using the .pipe method on Node.js or consume it as
+// an async iterator
+
+for await (const data of fullStream) {
+  console.log('data:', data)
+}
+```
+
+Additional options include:
 
 ``` js
 {
@@ -123,6 +174,28 @@ Make a read stream. Options include:
 }
 ```
 
+#### `await core.clear(start, [end])`
+
+Clear stored blocks between `start` and `end`, reclaiming storage when possible.
+
+``` js
+await core.clear(4) // clear block 4 from your local cache
+await core.clear(0, 10) // clear block 0-10 from your local cache
+```
+
+The core will also gossip to peers it is connected to, that is no longer has these blocks.
+
+#### `await core.truncate(newLength, [forkId])`
+
+Truncate the core to a smaller length.
+
+Per default this will update the fork id of the core to `+ 1`, but you can set the fork id you prefer with the option.
+Note that the fork id should be monotonely incrementing.
+
+#### `const hash = await core.treeHash([length])`
+
+Get the Merkle Tree hash of the core at a given length, defaulting to the current length of the core.
+
 #### `const range = core.download([range])`
 
 Download a range of data.
@@ -130,7 +203,7 @@ Download a range of data.
 You can await when the range has been fully downloaded by doing:
 
 ```js
-await range.downloaded()
+await range.done()
 ```
 
 A range can have the following properties:
@@ -165,23 +238,6 @@ To cancel downloading a range simply destroy the range instance.
 range.destroy()
 ```
 
-#### `const [index, relativeOffset] = await core.seek(byteOffset)`
-
-Seek to a byte offset.
-
-Returns `(index, relativeOffset)`, where `index` is the data block the byteOffset is contained in and `relativeOffset` is
-the relative byte offset in the data block.
-
-#### `const updated = await core.update()`
-
-Wait for the core to try and find a signed update to it's length.
-Does not download any data from peers except for a proof of the new core length.
-
-``` js
-const updated = await core.update()
-console.log('core was updated?', updated, 'length is', core.length)
-```
-
 #### `const info = await core.info()`
 
 Get information about this core, such as its total size in bytes.
@@ -191,6 +247,7 @@ The object will look like this:
 ```js
 Info {
   key: Buffer(...),
+  discoveryKey: Buffer(...),
   length: 18,
   contiguousLength: 16,
   byteLength: 742,
@@ -232,6 +289,12 @@ Can we read from this core? After closing the core this will be false.
 
 Populated after `ready` has been emitted. Will be `false` before the event.
 
+#### `core.id`
+
+String containing the id (z-base-32 of the public key) identifying this core.
+
+Populated after `ready` has been emitted. Will be `null` before the event.
+
 #### `core.key`
 
 Buffer containing the public key identifying this core.
@@ -267,12 +330,6 @@ How many blocks are contiguously available starting from the first block of this
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
-#### `core.contiguousByteLength`
-
-How much data is contiguously available starting from the first block of this core?
-
-Populated after `ready` has been emitted. Will be `0` before the event.
-
 #### `core.fork`
 
 What is the current fork id of this core?

package/index.js

@@ -7,11 +7,13 @@ const b4a = require('b4a')
 const Xache = require('xache')
 const NoiseSecretStream = require('@hyperswarm/secret-stream')
 const Protomux = require('protomux')
+const z32 = require('z32')
 
 const Replicator = require('./lib/replicator')
 const Core = require('./lib/core')
 const BlockEncryption = require('./lib/block-encryption')
 const Info = require('./lib/info')
+const Download = require('./lib/download')
 const { ReadStream, WriteStream } = require('./lib/streams')
 const { BAD_ARGUMENT, SESSION_CLOSED, SESSION_NOT_WRITABLE, SNAPSHOT_NOT_AVAILABLE } = require('./lib/errors')
 
@@ -57,6 +59,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.encodeBatch = null
     this.activeRequests = []
 
+    this.id = null
     this.key = key || null
     this.keyPair = null
     this.readable = true
@@ -145,7 +148,7 @@ module.exports = class Hypercore extends EventEmitter {
     if (!noiseStream) throw BAD_ARGUMENT('Invalid stream')
 
     if (!noiseStream.userData) {
-      const protocol = new Protomux(noiseStream)
+      const protocol = Protomux.from(noiseStream)
 
       if (opts.ondiscoverykey) {
         protocol.pair({ protocol: 'hypercore/alpha' }, opts.ondiscoverykey)
@@ -224,6 +227,7 @@ module.exports = class Hypercore extends EventEmitter {
     if (!this.auth) this.auth = o.auth
 
     this.crypto = o.crypto
+    this.id = o.id
     this.key = o.key
     this.core = o.core
     this.replicator = o.replicator
@@ -325,6 +329,7 @@ module.exports = class Hypercore extends EventEmitter {
 
     this.key = this.core.header.signer.publicKey
     this.keyPair = this.core.header.signer
+    this.id = z32.encode(this.key)
 
     this.replicator = new Replicator(this.core, this.key, {
       eagerUpdate: true,
@@ -612,8 +617,7 @@ module.exports = class Hypercore extends EventEmitter {
     if (this.opened === false) await this.opening
     if (this.closing !== null) return false
 
-    // TODO: add an option where a writer can bootstrap it's state from the network also
-    if (this.writable) {
+    if (this.writable && (!opts || opts.force !== true)) {
       if (!this.snapshotted) return false
       return this._updateSnapshot()
     }
@@ -629,7 +633,7 @@ module.exports = class Hypercore extends EventEmitter {
       const end = this.core.tree.length
       const contig = this.contiguousLength
 
-      await this.download({ start, end, ifAvailable: true }).downloaded()
+      await this.download({ start, end, ifAvailable: true }).done()
 
       if (!upgraded) upgraded = this.contiguousLength !== contig
     }
@@ -740,21 +744,12 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   download (range) {
-    const reqP = this._download(range)
+    const req = this._download(range)
 
     // do not crash in the background...
-    reqP.catch(noop)
+    req.catch(noop)
 
-    // TODO: turn this into an actual object...
-    return {
-      async downloaded () {
-        const req = await reqP
-        return req.promise
-      },
-      destroy () {
-        reqP.then(req => req.context && req.context.detach(req), noop)
-      }
-    }
+    return new Download(req)
   }
 
   async _download (range) {

package/lib/bitfield.js

@@ -152,7 +152,7 @@ module.exports = class Bitfield {
 
   clear () {
     return new Promise((resolve, reject) => {
-      this.storage.del(0, Infinity, (err) => {
+      this.storage.truncate(0, (err) => {
         if (err) return reject(err)
         this.pages = new BigSparseArray()
         this.unflushed = []

package/lib/block-store.js

@@ -20,12 +20,15 @@ module.exports = class BlockStore {
     return this.put(i, batch.length === 1 ? batch[0] : b4a.concat(batch), offset)
   }
 
-  clear (offset = 0, length = Infinity) {
+  clear (offset = 0, length = -1) {
     return new Promise((resolve, reject) => {
-      this.storage.del(offset, length, (err) => {
+      if (length === -1) this.storage.truncate(offset, done)
+      else this.storage.del(offset, length, done)
+
+      function done (err) {
         if (err) reject(err)
         else resolve()
-      })
+      }
     })
   }
 

package/lib/core.js

@@ -37,18 +37,8 @@ module.exports = class Core {
     try {
       return await this.resume(oplogFile, treeFile, bitfieldFile, dataFile, opts)
     } catch (err) {
-      return new Promise((resolve, reject) => {
-        let missing = 4
-
-        oplogFile.close(done)
-        treeFile.close(done)
-        bitfieldFile.close(done)
-        dataFile.close(done)
-
-        function done () {
-          if (--missing === 0) reject(err)
-        }
-      })
+      await closeAll(oplogFile, treeFile, bitfieldFile, dataFile)
+      throw err
     }
   }
 
@@ -596,4 +586,25 @@ function updateUserData (list, key, value) {
   if (value) list.push({ key, value })
 }
 
+function closeAll (...storages) {
+  let missing = 1
+  let error = null
+
+  return new Promise((resolve, reject) => {
+    for (const s of storages) {
+      missing++
+      s.close(done)
+    }
+
+    done(null)
+
+    function done (err) {
+      if (err) error = err
+      if (--missing) return
+      if (error) reject(error)
+      else resolve()
+    }
+  })
+}
+
 function noop () {}

package/lib/download.js

@@ -0,0 +1,22 @@
+module.exports = class Download {
+  constructor (req) {
+    this.req = req
+  }
+
+  async done () {
+    return (await this.req).promise
+  }
+
+  /**
+   * Deprecated. Use `range.done()`.
+   */
+  downloaded () {
+    return this.done()
+  }
+
+  destroy () {
+    this.req.then(req => req.context && req.context.detach(req), noop)
+  }
+}
+
+function noop () {}

package/lib/info.js

@@ -9,6 +9,7 @@ module.exports = class Info {
   static async from (core, padding, snapshot) {
     return new Info({
       key: core.key,
+      discoveryKey: core.discoveryKey,
       length: snapshot
         ? snapshot.length
         : core.tree.length,

package/lib/merkle-tree.js

@@ -454,7 +454,7 @@ module.exports = class MerkleTree {
       const t = this.truncateTo
       const offset = t === 0 ? 0 : (t - 1) * 80 + 40
 
-      this.storage.del(offset, Infinity, (err) => {
+      this.storage.truncate(offset, (err) => {
         if (err) return reject(err)
 
         if (this.truncateTo === t) {

package/lib/oplog.js

@@ -121,7 +121,7 @@ module.exports = class Oplog {
     if (size === buffer.byteLength) return result
 
     await new Promise((resolve, reject) => {
-      this.storage.del(size, Infinity, err => {
+      this.storage.truncate(size, err => {
         if (err) return reject(err)
         resolve()
       })
@@ -164,7 +164,7 @@ module.exports = class Oplog {
       this.storage.write(i === 0 ? 0 : this._pageSize, buf, err => {
         if (err) return reject(err)
 
-        this.storage.del(this._entryOffset, Infinity, err => {
+        this.storage.truncate(this._entryOffset, err => {
           if (err) return reject(err)
 
           this._headers[i] = bit

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "hypercore",
-  "version": "10.0.0",
+  "version": "10.2.1",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
-    "test": "standard && brittle test/*.js"
+    "test": "standard && node test/all.js",
+    "test:generate": "brittle -r test/all.js test/*.js"
   },
   "repository": {
     "type": "git",
@@ -42,18 +43,19 @@
     "flat-tree": "^1.9.0",
     "hypercore-crypto": "^3.2.1",
     "is-options": "^1.0.1",
-    "protomux": "^3.2.0",
-    "random-access-file": "^3.0.1",
+    "protomux": "^3.4.0",
+    "random-access-file": "^3.2.2",
     "random-array-iterator": "^1.0.0",
     "safety-catch": "^1.0.1",
     "sodium-universal": "^3.0.4",
     "streamx": "^2.12.4",
-    "xache": "^1.1.0"
+    "xache": "^1.1.0",
+    "z32": "^1.0.0"
   },
   "devDependencies": {
-    "brittle": "^2.0.0",
+    "brittle": "^3.0.0",
     "hyperswarm": "^4.1.1",
-    "random-access-memory": "^5.0.0",
+    "random-access-memory": "^5.0.1",
     "random-access-memory-overlay": "^2.0.0",
     "standard": "^16.0.3",
     "tmp-promise": "^3.0.2"