hypercore 10.0.0-alpha.7 → 10.0.0

package/lib/messages.js

@@ -1,6 +1,9 @@
 const c = require('compact-encoding')
+const b4a = require('b4a')
 
-const node = exports.node = {
+const EMPTY = b4a.alloc(0)
+
+const node = {
   preencode (state, n) {
     c.uint.preencode(state, n.index)
     c.uint.preencode(state, n.size)
@@ -22,6 +25,124 @@ const node = exports.node = {
 
 const nodeArray = c.array(node)
 
+const wire = exports.wire = {}
+
+wire.handshake = {
+  preencode (state, m) {
+    c.uint.preencode(state, 0) // flags for the future
+    c.fixed32.preencode(state, m.capability)
+  },
+  encode (state, m) {
+    c.uint.encode(state, 0) // flags for the future
+    c.fixed32.encode(state, m.capability)
+  },
+  decode (state) {
+    c.uint.decode(state) // flags for the future
+    return {
+      capability: c.fixed32.decode(state)
+    }
+  }
+}
+
+const requestBlock = {
+  preencode (state, b) {
+    c.uint.preencode(state, b.index)
+    c.uint.preencode(state, b.nodes)
+  },
+  encode (state, b) {
+    c.uint.encode(state, b.index)
+    c.uint.encode(state, b.nodes)
+  },
+  decode (state) {
+    return {
+      index: c.uint.decode(state),
+      nodes: c.uint.decode(state)
+    }
+  }
+}
+
+const requestSeek = {
+  preencode (state, s) {
+    c.uint.preencode(state, s.bytes)
+  },
+  encode (state, s) {
+    c.uint.encode(state, s.bytes)
+  },
+  decode (state) {
+    return {
+      bytes: c.uint.decode(state)
+    }
+  }
+}
+
+const requestUpgrade = {
+  preencode (state, u) {
+    c.uint.preencode(state, u.start)
+    c.uint.preencode(state, u.length)
+  },
+  encode (state, u) {
+    c.uint.encode(state, u.start)
+    c.uint.encode(state, u.length)
+  },
+  decode (state) {
+    return {
+      start: c.uint.decode(state),
+      length: c.uint.decode(state)
+    }
+  }
+}
+
+wire.request = {
+  preencode (state, m) {
+    state.end++ // flags
+    c.uint.preencode(state, m.id)
+    c.uint.preencode(state, m.fork)
+
+    if (m.block) requestBlock.preencode(state, m.block)
+    if (m.hash) requestBlock.preencode(state, m.hash)
+    if (m.seek) requestSeek.preencode(state, m.seek)
+    if (m.upgrade) requestUpgrade.preencode(state, m.upgrade)
+  },
+  encode (state, m) {
+    const flags = (m.block ? 1 : 0) | (m.hash ? 2 : 0) | (m.seek ? 4 : 0) | (m.upgrade ? 8 : 0)
+
+    c.uint.encode(state, flags)
+    c.uint.encode(state, m.id)
+    c.uint.encode(state, m.fork)
+
+    if (m.block) requestBlock.encode(state, m.block)
+    if (m.hash) requestBlock.encode(state, m.hash)
+    if (m.seek) requestSeek.encode(state, m.seek)
+    if (m.upgrade) requestUpgrade.encode(state, m.upgrade)
+  },
+  decode (state) {
+    const flags = c.uint.decode(state)
+
+    return {
+      id: c.uint.decode(state),
+      fork: c.uint.decode(state),
+      block: flags & 1 ? requestBlock.decode(state) : null,
+      hash: flags & 2 ? requestBlock.decode(state) : null,
+      seek: flags & 4 ? requestSeek.decode(state) : null,
+      upgrade: flags & 8 ? requestUpgrade.decode(state) : null
+    }
+  }
+}
+
+wire.cancel = {
+  preencode (state, m) {
+    c.uint.preencode(state, m.request)
+  },
+  encode (state, m) {
+    c.uint.encode(state, m.request)
+  },
+  decode (state, m) {
+    return {
+      request: c.uint.decode(state)
+    }
+  }
+}
+
 const dataUpgrade = {
   preencode (state, u) {
     c.uint.preencode(state, u.start)
@@ -79,259 +200,214 @@ const dataBlock = {
   decode (state) {
     return {
       index: c.uint.decode(state),
-      value: c.buffer.decode(state),
+      value: c.buffer.decode(state) || EMPTY,
       nodes: nodeArray.decode(state)
     }
   }
 }
 
-exports.data = {
-  preencode (state, d) {
-    c.uint.preencode(state, d.fork)
-    state.end++ // flags
-    if (d.block) dataBlock.preencode(state, d.block)
-    if (d.seek) dataSeek.preencode(state, d.seek)
-    if (d.upgrade) dataUpgrade.preencode(state, d.upgrade)
-  },
-  encode (state, d) {
-    c.uint.encode(state, d.fork)
-
-    const s = state.start++
-    let flags = 0
-
-    if (d.block) {
-      flags |= 1
-      dataBlock.encode(state, d.block)
-    }
-    if (d.seek) {
-      flags |= 2
-      dataSeek.encode(state, d.seek)
-    }
-    if (d.upgrade) {
-      flags |= 4
-      dataUpgrade.encode(state, d.upgrade)
-    }
-
-    state.buffer[s] = flags
-  },
-  decode (state) {
-    const fork = c.uint.decode(state)
-    const flags = c.uint.decode(state)
-    return {
-      fork,
-      block: (flags & 1) === 0 ? null : dataBlock.decode(state),
-      seek: (flags & 2) === 0 ? null : dataSeek.decode(state),
-      upgrade: (flags & 4) === 0 ? null : dataUpgrade.decode(state)
-    }
-  }
-}
-
-const requestBlock = {
+const dataHash = {
   preencode (state, b) {
     c.uint.preencode(state, b.index)
-    c.bool.preencode(state, b.value)
-    c.uint.preencode(state, b.nodes)
+    nodeArray.preencode(state, b.nodes)
   },
   encode (state, b) {
     c.uint.encode(state, b.index)
-    c.bool.encode(state, b.value)
-    c.uint.encode(state, b.nodes)
+    nodeArray.encode(state, b.nodes)
   },
   decode (state) {
     return {
       index: c.uint.decode(state),
-      value: c.bool.decode(state),
-      nodes: c.uint.decode(state)
+      nodes: nodeArray.decode(state)
     }
   }
 }
 
-const requestSeek = {
-  preencode (state, s) {
-    c.uint.preencode(state, s.bytes)
-  },
-  encode (state, s) {
-    c.uint.encode(state, s.bytes)
-  },
-  decode (state) {
-    return {
-      bytes: c.uint.decode(state)
-    }
-  }
-}
+wire.data = {
+  preencode (state, m) {
+    state.end++ // flags
+    c.uint.preencode(state, m.request)
+    c.uint.preencode(state, m.fork)
 
-const requestUpgrade = {
-  preencode (state, u) {
-    c.uint.preencode(state, u.start)
-    c.uint.preencode(state, u.length)
+    if (m.block) dataBlock.preencode(state, m.block)
+    if (m.hash) dataHash.preencode(state, m.hash)
+    if (m.seek) dataSeek.preencode(state, m.seek)
+    if (m.upgrade) dataUpgrade.preencode(state, m.upgrade)
   },
-  encode (state, u) {
-    c.uint.encode(state, u.start)
-    c.uint.encode(state, u.length)
+  encode (state, m) {
+    const flags = (m.block ? 1 : 0) | (m.hash ? 2 : 0) | (m.seek ? 4 : 0) | (m.upgrade ? 8 : 0)
+
+    c.uint.encode(state, flags)
+    c.uint.encode(state, m.request)
+    c.uint.encode(state, m.fork)
+
+    if (m.block) dataBlock.encode(state, m.block)
+    if (m.hash) dataHash.encode(state, m.hash)
+    if (m.seek) dataSeek.encode(state, m.seek)
+    if (m.upgrade) dataUpgrade.encode(state, m.upgrade)
   },
   decode (state) {
+    const flags = c.uint.decode(state)
+
     return {
-      start: c.uint.decode(state),
-      length: c.uint.decode(state)
+      request: c.uint.decode(state),
+      fork: c.uint.decode(state),
+      block: flags & 1 ? dataBlock.decode(state) : null,
+      hash: flags & 2 ? dataHash.decode(state) : null,
+      seek: flags & 4 ? dataSeek.decode(state) : null,
+      upgrade: flags & 8 ? dataUpgrade.decode(state) : null
     }
   }
 }
 
-exports.request = {
-  preencode (state, r) {
-    c.uint.preencode(state, r.fork)
-    state.end++ // flags
-    if (r.block) requestBlock.preencode(state, r.block)
-    if (r.seek) requestSeek.preencode(state, r.seek)
-    if (r.upgrade) requestUpgrade.preencode(state, r.upgrade)
+wire.noData = {
+  preencode (state, m) {
+    c.uint.preencode(state, m.request)
   },
-  encode (state, r) {
-    c.uint.encode(state, r.fork)
-
-    const s = state.start++
-    let flags = 0
-
-    if (r.block) {
-      flags |= 1
-      requestBlock.encode(state, r.block)
-    }
-    if (r.seek) {
-      flags |= 2
-      requestSeek.encode(state, r.seek)
-    }
-    if (r.upgrade) {
-      flags |= 4
-      requestUpgrade.encode(state, r.upgrade)
-    }
-
-    state.buffer[s] = flags
+  encode (state, m) {
+    c.uint.encode(state, m.request)
   },
-  decode (state) {
-    const fork = c.uint.decode(state)
-    const flags = c.uint.decode(state)
+  decode (state, m) {
     return {
-      fork,
-      block: (flags & 1) === 0 ? null : requestBlock.decode(state),
-      seek: (flags & 2) === 0 ? null : requestSeek.decode(state),
-      upgrade: (flags & 4) === 0 ? null : requestUpgrade.decode(state)
+      request: c.uint.decode(state)
     }
   }
 }
 
-exports.have = {
-  preencode (state, h) {
-    c.uint.preencode(state, h.start)
-    if (h.length > 1) c.uint.preencode(state, h.length)
+wire.want = {
+  preencode (state, m) {
+    c.uint.preencode(state, m.start)
+    c.uint.preencode(state, m.length)
   },
-  encode (state, h) {
-    c.uint.encode(state, h.start)
-    if (h.length > 1) c.uint.encode(state, h.length)
+  encode (state, m) {
+    c.uint.encode(state, m.start)
+    c.uint.encode(state, m.length)
   },
   decode (state) {
     return {
       start: c.uint.decode(state),
-      length: state.start < state.end ? c.uint.decode(state) : 1
+      length: c.uint.decode(state)
     }
   }
 }
 
-exports.bitfield = {
-  preencode (state, b) {
-    c.uint.preencode(state, b.start)
-    c.uint32array.preencode(state, b.bitfield)
+wire.unwant = {
+  preencode (state, m) {
+    c.uint.preencode(state, m.start)
+    c.uint.preencode(state, m.length)
   },
-  encode (state, b) {
-    c.uint.encode(state, b.start)
-    c.uint32array.encode(state, b.bitfield)
+  encode (state, m) {
+    c.uint.encode(state, m.start)
+    c.uint.encode(state, m.length)
   },
-  decode (state) {
+  decode (state, m) {
     return {
       start: c.uint.decode(state),
-      bitfield: c.uint32array.decode(state)
+      length: c.uint.decode(state)
     }
   }
 }
 
-exports.info = {
-  preencode (state, i) {
-    c.uint.preencode(state, i.length)
-    c.uint.preencode(state, i.fork)
+wire.range = {
+  preencode (state, m) {
+    state.end++ // flags
+    c.uint.preencode(state, m.start)
+    if (m.length !== 1) c.uint.preencode(state, m.length)
   },
-  encode (state, i) {
-    c.uint.encode(state, i.length)
-    c.uint.encode(state, i.fork)
+  encode (state, m) {
+    c.uint.encode(state, (m.drop ? 1 : 0) | (m.length === 1 ? 2 : 0))
+    c.uint.encode(state, m.start)
+    if (m.length !== 1) c.uint.encode(state, m.length)
   },
   decode (state) {
+    const flags = c.uint.decode(state)
+
     return {
-      length: c.uint.decode(state),
-      fork: c.uint.decode(state)
+      drop: (flags & 1) !== 0,
+      start: c.uint.decode(state),
+      length: (flags & 2) !== 0 ? 1 : c.uint.decode(state)
     }
   }
 }
 
-exports.handshake = {
-  preencode (state, h) {
-    c.uint.preencode(state, h.protocolVersion)
-    c.string.preencode(state, h.userAgent)
+wire.bitfield = {
+  preencode (state, m) {
+    c.uint.preencode(state, m.start)
+    c.uint32array.preencode(state, m.bitfield)
   },
-  encode (state, h) {
-    c.uint.encode(state, h.protocolVersion)
-    c.string.encode(state, h.userAgent)
+  encode (state, m) {
+    c.uint.encode(state, m.start)
+    c.uint32array.encode(state, m.bitfield)
   },
-  decode (state) {
+  decode (state, m) {
     return {
-      protocolVersion: c.uint.decode(state),
-      userAgent: c.string.decode(state)
+      start: c.uint.decode(state),
+      bitfield: c.uint32array.decode(state)
     }
   }
 }
 
-exports.extension = {
-  preencode (state, a) {
-    c.uint.preencode(state, a.alias)
-    c.string.preencode(state, a.name)
+wire.sync = {
+  preencode (state, m) {
+    state.end++ // flags
+    c.uint.preencode(state, m.fork)
+    c.uint.preencode(state, m.length)
+    c.uint.preencode(state, m.remoteLength)
   },
-  encode (state, a) {
-    c.uint.encode(state, a.alias)
-    c.string.encode(state, a.name)
+  encode (state, m) {
+    c.uint.encode(state, (m.canUpgrade ? 1 : 0) | (m.uploading ? 2 : 0) | (m.downloading ? 4 : 0))
+    c.uint.encode(state, m.fork)
+    c.uint.encode(state, m.length)
+    c.uint.encode(state, m.remoteLength)
   },
   decode (state) {
+    const flags = c.uint.decode(state)
+
     return {
-      alias: c.uint.decode(state),
-      name: c.string.decode(state)
+      fork: c.uint.decode(state),
+      length: c.uint.decode(state),
+      remoteLength: c.uint.decode(state),
+      canUpgrade: (flags & 1) !== 0,
+      uploading: (flags & 2) !== 0,
+      downloading: (flags & 4) !== 0
     }
   }
 }
 
-exports.core = {
+wire.reorgHint = {
   preencode (state, m) {
-    c.uint.preencode(state, m.alias)
-    c.fixed32.preencode(state, m.discoveryKey)
-    c.fixed32.preencode(state, m.capability)
+    c.uint.preencode(state, m.from)
+    c.uint.preencode(state, m.to)
+    c.uint.preencode(state, m.ancestors)
   },
   encode (state, m) {
-    c.uint.encode(state, m.alias)
-    c.fixed32.encode(state, m.discoveryKey)
-    c.fixed32.encode(state, m.capability)
+    c.uint.encode(state, m.from)
+    c.uint.encode(state, m.to)
+    c.uint.encode(state, m.ancestors)
   },
   decode (state) {
     return {
-      alias: c.uint.decode(state),
-      discoveryKey: c.fixed32.decode(state),
-      capability: c.fixed32.decode(state)
+      from: c.uint.encode(state),
+      to: c.uint.encode(state),
+      ancestors: c.uint.encode(state)
     }
   }
 }
 
-exports.unknownCore = {
+wire.extension = {
   preencode (state, m) {
-    c.fixed32.preencode(state, m.discoveryKey)
+    c.string.preencode(state, m.name)
+    c.raw.preencode(state, m.message)
   },
   encode (state, m) {
-    c.fixed32.encode(state, m.discoveryKey)
+    c.string.encode(state, m.name)
+    c.raw.encode(state, m.message)
   },
   decode (state) {
-    return { discoveryKey: c.fixed32.decode(state) }
+    return {
+      name: c.string.decode(state),
+      message: c.raw.decode(state)
+    }
   }
 }
 
@@ -396,7 +472,9 @@ const bitfieldUpdate = { // TODO: can maybe be folded into a HAVE later on with
   }
 }
 
-exports.oplogEntry = {
+const oplog = exports.oplog = {}
+
+oplog.entry = {
   preencode (state, m) {
     state.end++ // flags
     if (m.userData) keyValue.preencode(state, m.userData)
@@ -536,7 +614,7 @@ const types = {
 
 const keyValueArray = c.array(keyValue)
 
-exports.oplogHeader = {
+oplog.header = {
   preencode (state, h) {
     state.end += 1 // version
     types.preencode(state, h.types)
@@ -544,6 +622,7 @@ exports.oplogHeader = {
     treeHeader.preencode(state, h.tree)
     keyPair.preencode(state, h.signer)
     hints.preencode(state, h.hints)
+    c.uint.preencode(state, h.contiguousLength)
   },
   encode (state, h) {
     state.buffer[state.start++] = 0 // version
@@ -552,6 +631,7 @@ exports.oplogHeader = {
     treeHeader.encode(state, h.tree)
     keyPair.encode(state, h.signer)
     hints.encode(state, h.hints)
+    c.uint.encode(state, h.contiguousLength)
   },
   decode (state) {
     const version = c.uint.decode(state)
@@ -565,7 +645,8 @@ exports.oplogHeader = {
       userData: keyValueArray.decode(state),
       tree: treeHeader.decode(state),
       signer: keyPair.decode(state),
-      hints: hints.decode(state)
+      hints: hints.decode(state),
+      contiguousLength: state.end > state.start ? c.uint.decode(state) : 0
     }
   }
 }

package/lib/oplog.js

@@ -1,4 +1,5 @@
 const cenc = require('compact-encoding')
+const b4a = require('b4a')
 const crc32 = require('crc32-universal')
 
 module.exports = class Oplog {
@@ -133,7 +134,7 @@ module.exports = class Oplog {
     return new Promise((resolve, reject) => {
       this.storage.open(err => {
         if (err && err.code !== 'ENOENT') return reject(err)
-        if (err) return resolve(Buffer.alloc(0))
+        if (err) return resolve(b4a.alloc(0))
         this.storage.stat((err, stat) => {
           if (err && err.code !== 'ENOENT') return reject(err)
           this.storage.read(0, stat.size, (err, buf) => {
@@ -151,7 +152,7 @@ module.exports = class Oplog {
     const bit = (this._headers[i] + 1) & 1
 
     this.headerEncoding.preencode(state, header)
-    state.buffer = Buffer.allocUnsafe(state.end)
+    state.buffer = b4a.allocUnsafe(state.end)
     this.headerEncoding.encode(state, header)
     this._addHeader(state, state.end - 8, bit, 0)
 
@@ -187,7 +188,7 @@ module.exports = class Oplog {
       this.entryEncoding.preencode(state, batch[i])
     }
 
-    state.buffer = Buffer.allocUnsafe(state.end)
+    state.buffer = b4a.allocUnsafe(state.end)
 
     for (let i = 0; i < batch.length; i++) {
       const start = state.start += 8 // space for header

package/lib/remote-bitfield.js

@@ -1,24 +1,45 @@
 const BigSparseArray = require('big-sparse-array')
+const bits = require('bits-to-bytes')
 
 module.exports = class RemoteBitfield {
   constructor () {
+    this.pageSize = 32768
     this.pages = new BigSparseArray()
   }
 
   get (index) {
-    const r = index & 32767
-    const i = (index - r) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
+
     const p = this.pages.get(i)
 
-    return p ? (p[r >>> 5] & (1 << (r & 31))) !== 0 : false
+    return p ? bits.get(p, j) : false
   }
 
   set (index, val) {
-    const r = index & 32767
-    const i = (index - r) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
+
     const p = this.pages.get(i) || this.pages.set(i, new Uint32Array(1024))
 
-    if (val) p[r >>> 5] |= (1 << (r & 31))
-    else p[r >>> 5] &= ~(1 << (r & 31))
+    bits.set(p, j, val)
+  }
+
+  setRange (start, length, val) {
+    let j = start & (this.pageSize - 1)
+    let i = (start - j) / this.pageSize
+
+    while (length > 0) {
+      const p = this.pages.get(i) || this.pages.set(i, new Uint32Array(1024))
+
+      const end = Math.min(j + length, this.pageSize)
+      const range = end - j
+
+      bits.fill(p, val, j, end)
+
+      j = 0
+      i++
+      length -= range
+    }
   }
 }