corestore 6.0.0-beta1 → 6.0.1-alpha.11

package/.github/workflows/test-node.yml

@@ -3,16 +3,16 @@ name: Test on Node.js
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
-      - master
+      - main
 jobs:
   build:
     strategy:
       matrix:
-        node-version: [12.x, 14.x]
-        os: [ubuntu-16.04, macos-latest, windows-latest]
+        node-version: [12.x, 14.x, 16.x]
+        os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2

package/README.md

@@ -1,2 +1,59 @@
-# neocorestore
-A new take on Corestore
+# Corestore v6
+
+Corestore is a Hypercore factory that makes it easier to manage large collections of named Hypercores.
+
+Corestore provides:
+1. __Key Derivation__ - All writable Hypercore keys are derived from a single master key and a user-provided name.
+2. __Session Handling__ - If a single Hypercore is loaded multiple times through the `get` method, the underlying resources will only be opened once (using Hypercore 10's new session feature). Once all sessions are closed, the resources will be released.
+3. __Storage Management__ - Hypercores can be stored in any random-access-storage instance, where they will be keyed by their discovery keys.
+4. __Namespacing__ - You can share a single Corestore instance between multiple applications or components without worrying about naming collisions by creating "namespaces" (e.g. `corestore.namespace('my-app').get({ name: 'main' }) 
+
+### Installation
+`npm install corestore@next`
+
+### Usage
+A corestore instance can be constructed with a random-access-storage module, a function that returns a random-access-storage module given a path, or a string. If a string is specified, it will be assumed to be a path to a local storage directory:
+```js
+const Corestore = require('corestore')
+
+const store = new Corestore('./my-storage')
+const core1 = store.get({ name: 'core-1' })
+const core2 = store.get({ name: 'core-2' })
+```
+
+### API
+#### `const store = new Corestore(storage)`
+Create a new Corestore instance.
+
+`storage` can be either a random-access-storage module, a string, or a function that takes a path and returns an random-access-storage instance.
+
+#### `const core = store.get(key | { name: 'a-name', ...hypercoreOpts})`
+Loads a Hypercore, either by name (if the `name` option is provided), or from the provided key (if the first argument is a Buffer, or if the `key` options is set).
+
+If that Hypercore has previously been loaded, subsequent calls to `get` will return a new Hypercore session on the existing core.
+
+All other options besides `name` and `key` will be forwarded to the Hypercore constructor.
+
+#### `const stream = store.replicate(opts)`
+Creates a replication stream that's capable of replicating all Hypercores that are managed by the Corestore, assuming the remote peer has the correct capabilities.
+
+`opts` will be forwarded to Hypercore's `replicate` function.
+
+Corestore replicates in an "all-to-all" fashion, meaning that when replication begins, it will attempt to replicate every Hypercore that's currently loaded and in memory. These attempts will fail if the remote side doesn't have a Hypercore's capability -- Corestore replication does not exchange Hypercore keys.
+
+If the remote side dynamically adds a new Hypercore to the replication stream, Corestore will load and replicatethat core if possible.
+
+#### `const store = store.namespace(name)`
+Create a new namespaced Corestore. Namespacing is useful if you're going to be sharing a single Corestore instance between many applications or components, as it prevents name collisions.
+
+Namespaces can be chained:
+```js
+const ns1 = store.namespace('a')
+const ns2 = ns1.namespace('b')
+const core1 = ns1.get({ name: 'main' }) // These will load different Hypercores
+const core2 = ns2.get({ name: 'main' })
+```
+
+### License
+MIT
+

package/index.js

@@ -1,122 +1,256 @@
-const { NanoresourcePromise: Nanoresource } = require('nanoresource-promise/emitter')
-const raf = require('random-access-file')
+const { EventEmitter } = require('events')
+const safetyCatch = require('safety-catch')
+const crypto = require('hypercore-crypto')
+const sodium = require('sodium-universal')
+const Hypercore = require('hypercore')
+const b4a = require('b4a')
 
-const Replicator = require('./lib/replicator')
-const Index = require('./lib/db')
-const Loader = require('./lib/loader')
-const errors = require('./lib/errors')
+const KeyManager = require('./lib/keys')
 
-module.exports = class Corestore extends Nanoresource {
+const CORES_DIR = 'cores'
+const PROFILES_DIR = 'profiles'
+const USERDATA_NAME_KEY = '@corestore/name'
+const USERDATA_NAMESPACE_KEY = '@corestore/namespace'
+const DEFAULT_NAMESPACE = generateNamespace('@corestore/default')
+
+module.exports = class Corestore extends EventEmitter {
   constructor (storage, opts = {}) {
     super()
 
-    if (typeof storage === 'string') storage = defaultStorage(storage)
-    if (typeof storage !== 'function') throw new errors.InvalidStorageError()
-    this.storage = storage
+    this.storage = Hypercore.defaultStorage(storage, { lock: PROFILES_DIR + '/default' })
 
-    this._namespace = opts.namespace || []
-    this._db = opts._db || new Index(this.storage, opts)
-    this._loader = opts._loader || new Loader(this.storage, this._db, opts)
-    this._replicator = opts._replicator || new Replicator(this._loader, opts)
+    this.cores = opts._cores || new Map()
+    this.keys = opts.keys
 
-    this._db.on('error', err => this.emit('db-error', err))
-    this._loader.on('error', err => this.emit('error', err))
-    this._loader.on('core', (core, opts) => this.emit('core', core, opts))
-    this._loader.on('core', (core, opts) => this.emit('feed', core, opts))
+    this._namespace = opts._namespace || DEFAULT_NAMESPACE
+    this._replicationStreams = opts._streams || []
+    this._streamSessions = opts._streamSessions || new Map()
 
-    this.ready = this.open.bind(this)
+    this._opening = opts._opening ? opts._opening.then(() => this._open()) : this._open()
+    this._opening.catch(safetyCatch)
+    this.ready = () => this._opening
   }
 
-  get cache () {
-    return this._loader.cache
+  async _open () {
+    if (this.keys) {
+      this.keys = await this.keys // opts.keys can be a Promise that resolves to a KeyManager
+    } else {
+      this.keys = await KeyManager.fromStorage(p => this.storage(PROFILES_DIR + '/' + p))
+    }
   }
 
-  // Nanoresource Methods
+  async _generateKeys (opts) {
+    if (opts._discoveryKey) {
+      return {
+        keyPair: null,
+        sign: null,
+        discoveryKey: opts._discoveryKey
+      }
+    }
+    if (!opts.name) {
+      return {
+        keyPair: {
+          publicKey: opts.publicKey,
+          secretKey: opts.secretKey
+        },
+        sign: opts.sign,
+        discoveryKey: crypto.discoveryKey(opts.publicKey)
+      }
+    }
+    const { publicKey, sign } = await this.keys.createHypercoreKeyPair(opts.name, this._namespace)
+    return {
+      keyPair: {
+        publicKey,
+        secretKey: null
+      },
+      sign,
+      discoveryKey: crypto.discoveryKey(publicKey)
+    }
+  }
 
-  async _open () {
-    await this._db.open()
-    return this._loader.open()
+  _getPrereadyUserData (core, key) {
+    for (const { key: savedKey, value } of core.core.header.userData) {
+      if (key === savedKey) return value
+    }
+    return null
   }
 
-  async _close () {
-    await this._replicator.close()
-    await this._loader.close()
-    return this._db.close()
+  async _preready (core) {
+    const name = this._getPrereadyUserData(core, USERDATA_NAME_KEY)
+    if (!name) return
+
+    const namespace = this._getPrereadyUserData(core, USERDATA_NAMESPACE_KEY)
+    const { publicKey, sign } = await this.keys.createHypercoreKeyPair(b4a.toString(name), namespace)
+    if (!b4a.equals(publicKey, core.key)) throw new Error('Stored core key does not match the provided name')
+
+    // TODO: Should Hypercore expose a helper for this, or should preready return keypair/sign?
+    core.sign = sign
+    core.key = publicKey
+    core.writable = true
   }
 
-  // Private Methods
+  async _preload (opts) {
+    await this.ready()
+
+    const { discoveryKey, keyPair, sign } = await this._generateKeys(opts)
+    const id = b4a.toString(discoveryKey, 'hex')
 
-  _validateGetOptions (opts) {
-    if (typeof opts === 'object') {
-      if (!Buffer.isBuffer(opts)) {
-        if (!opts.key && !opts.name && !opts.keyPair) throw new errors.InvalidOptionsError()
-      } else {
-        opts = { key: opts }
+    while (this.cores.has(id)) {
+      const existing = this.cores.get(id)
+      if (existing.opened && !existing.closing) return { from: existing, keyPair, sign }
+      if (!existing.opened) {
+        await existing.ready().catch(safetyCatch)
+      } else if (existing.closing) {
+        await existing.close()
       }
-    } else {
-      opts = { key: Buffer.from(opts, 'hex') }
     }
-    if (opts.key && typeof opts.key === 'string') opts.key = Buffer.from(opts.key, 'hex')
-    if (opts.key && opts.key.length !== 32) throw new errors.InvalidKeyError()
-    if (opts.name && !Array.isArray(opts.name)) opts.name = [opts.name]
-    return opts
-  }
 
-  // Public Methods
+    const userData = {}
+    if (opts.name) {
+      userData[USERDATA_NAME_KEY] = b4a.from(opts.name)
+      userData[USERDATA_NAMESPACE_KEY] = this._namespace
+    }
 
-  get (opts = {}) {
-    opts = this._validateGetOptions(opts)
-    if (!this.opened) this.open().catch(err => this.emit('error', err))
-    return this._loader.get(this._namespace, opts)
+    // No more async ticks allowed after this point -- necessary for caching
+
+    const storageRoot = [CORES_DIR, id.slice(0, 2), id.slice(2, 4), id].join('/')
+    const core = new Hypercore(p => this.storage(storageRoot + '/' + p), {
+      _preready: this._preready.bind(this),
+      autoClose: true,
+      encryptionKey: opts.encryptionKey || null,
+      userData,
+      sign: null,
+      createIfMissing: !opts._discoveryKey,
+      keyPair: keyPair && keyPair.publicKey
+        ? {
+            publicKey: keyPair.publicKey,
+            secretKey: null
+          }
+        : null
+    })
+
+    this.cores.set(id, core)
+    core.ready().then(() => {
+      for (const { stream } of this._replicationStreams) {
+        const sessions = this._streamSessions.get(stream)
+        const session = core.session()
+        sessions.push(session)
+        core.replicate(stream)
+      }
+    }, () => {
+      this.cores.delete(id)
+    })
+    core.once('close', () => {
+      this.cores.delete(id)
+    })
+
+    return { from: core, keyPair, sign }
   }
 
-  namespace (name) {
-    if (!name) throw new Error('A name must be provided as the first argument.')
-    if (Buffer.isBuffer(name)) name = name.toString('hex')
-    return new Corestore(this.storage, {
-      namespace: [...this._namespace, name],
-      _db: this._db,
-      _loader: this._loader,
-      _replicator: this._replicator
+  get (opts = {}) {
+    opts = validateGetOptions(opts)
+    const core = new Hypercore(null, {
+      ...opts,
+      name: null,
+      preload: () => this._preload(opts)
     })
+    return core
   }
 
   replicate (isInitiator, opts) {
-    return this._replicator.replicate(isInitiator, opts)
+    const isExternal = isStream(isInitiator) || !!(opts && opts.stream)
+    const stream = Hypercore.createProtocolStream(isInitiator, {
+      ...opts,
+      ondiscoverykey: discoveryKey => {
+        const core = this.get({ _discoveryKey: discoveryKey })
+        return core.ready().catch(safetyCatch)
+      }
+    })
+
+    const sessions = []
+    for (const core of this.cores.values()) {
+      if (!core.opened) continue // If the core is not opened, it will be replicated in preload.
+      const session = core.session()
+      sessions.push(session)
+      core.replicate(stream)
+    }
+
+    const streamRecord = { stream, isExternal }
+    this._replicationStreams.push(streamRecord)
+    this._streamSessions.set(stream, sessions)
+
+    stream.once('close', () => {
+      this._replicationStreams.splice(this._replicationStreams.indexOf(streamRecord), 1)
+      this._streamSessions.delete(stream)
+      Promise.all(sessions.map(s => s.close())).catch(safetyCatch)
+    })
+    return stream
   }
 
-  // Backup/Restore
+  namespace (name) {
+    if (!b4a.isBuffer(name)) name = b4a.from(name)
+    return new Corestore(this.storage, {
+      _namespace: generateNamespace(this._namespace, name),
+      _opening: this._opening,
+      _cores: this.cores,
+      _streams: this._replicationStreams,
+      _streamSessions: this._streamSessions,
+      keys: this._opening.then(() => this.keys)
+    })
+  }
 
-  async backup () {
-    if (!this.opened) await this.open()
-    const allCores = await this._db.getAllCores()
-    return {
-      masterKey: this._loader.masterKey.toString('hex'),
-      cores: allCores
+  async _close () {
+    await this._opening
+    if (!this._namespace.equals(DEFAULT_NAMESPACE)) return // namespaces should not release resources on close
+    const closePromises = []
+    for (const core of this.cores.values()) {
+      closePromises.push(core.close())
+    }
+    await Promise.allSettled(closePromises)
+    for (const { stream, isExternal } of this._replicationStreams) {
+      // Only close streams that were created by the Corestore
+      if (!isExternal) stream.destroy()
     }
+    await this.keys.close()
   }
 
-  restore (manifest) {
-    return this._db.restore(manifest)
+  close () {
+    if (this._closing) return this._closing
+    this._closing = this._close()
+    this._closing.catch(safetyCatch)
+    return this._closing
   }
 
-  static async restore (manifest, target) {
-    if (!manifest || !manifest.masterKey) throw new Error('Malformed manifest.')
-    const store = new this(target, {
-      overwriteMasterKey: true,
-      masterKey: Buffer.from(manifest.masterKey, 'hex')
-    })
-    await store.restore(manifest)
-    return store
+  static createToken () {
+    return KeyManager.createToken()
   }
 }
 
-function defaultStorage (dir) {
-  return function (name) {
-    let lock = null
-    try {
-      lock = name.endsWith('/bitfield') ? require('fd-lock') : null
-    } catch (err) {}
-    return raf(name, { directory: dir, lock: lock })
+function validateGetOptions (opts) {
+  if (b4a.isBuffer(opts)) return { key: opts, publicKey: opts }
+  if (opts.key) {
+    opts.publicKey = opts.key
+  }
+  if (opts.keyPair) {
+    opts.publicKey = opts.keyPair.publicKey
+    opts.secretKey = opts.keyPair.secretKey
   }
+  if (opts.name && typeof opts.name !== 'string') throw new Error('name option must be a String')
+  if (opts.name && opts.secretKey) throw new Error('Cannot provide both a name and a secret key')
+  if (opts.publicKey && !b4a.isBuffer(opts.publicKey)) throw new Error('publicKey option must be a Buffer or Uint8Array')
+  if (opts.secretKey && !b4a.isBuffer(opts.secretKey)) throw new Error('secretKey option must be a Buffer or Uint8Array')
+  if (!opts._discoveryKey && (!opts.name && !opts.publicKey)) throw new Error('Must provide either a name or a publicKey')
+  return opts
+}
+
+function generateNamespace (first, second) {
+  if (!b4a.isBuffer(first)) first = b4a.from(first)
+  if (second && !b4a.isBuffer(second)) second = b4a.from(second)
+  const out = b4a.allocUnsafe(32)
+  sodium.crypto_generichash(out, second ? b4a.concat([first, second]) : first)
+  return out
+}
+
+function isStream (s) {
+  return typeof s === 'object' && s && typeof s.pipe === 'function'
 }

package/lib/keys.js

@@ -0,0 +1,105 @@
+// TODO: Extract this into a standalone module
+
+const sodium = require('sodium-universal')
+const blake2b = require('blake2b-universal')
+const b4a = require('b4a')
+
+const DEFAULT_TOKEN = b4a.alloc(0)
+const NAMESPACE = b4a.from('@hyperspace/key-manager')
+
+module.exports = class KeyManager {
+  constructor (storage, profile, opts = {}) {
+    this.storage = storage
+    this.profile = profile
+  }
+
+  _sign (keyPair, message) {
+    if (!keyPair._secretKey) throw new Error('Invalid key pair')
+    const signature = b4a.allocUnsafe(sodium.crypto_sign_BYTES)
+    sodium.crypto_sign_detached(signature, message, keyPair._secretKey)
+    return signature
+  }
+
+  createSecret (name, token) {
+    return deriveSeed(this.profile, token, name)
+  }
+
+  createHypercoreKeyPair (name, token) {
+    const keyPair = {
+      publicKey: b4a.allocUnsafe(sodium.crypto_sign_PUBLICKEYBYTES),
+      _secretKey: b4a.alloc(sodium.crypto_sign_SECRETKEYBYTES),
+      sign: (msg) => this._sign(keyPair, msg)
+    }
+
+    sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair._secretKey, this.createSecret(name, token))
+
+    return keyPair
+  }
+
+  createNetworkIdentity (name, token) {
+    const keyPair = {
+      publicKey: b4a.alloc(32),
+      secretKey: b4a.alloc(64)
+    }
+
+    sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair.secretKey, this.createSecret(name, token))
+
+    return keyPair
+  }
+
+  close () {
+    return new Promise((resolve, reject) => {
+      this.storage.close(err => {
+        if (err) return reject(err)
+        return resolve()
+      })
+    })
+  }
+
+  static createToken () {
+    return randomBytes(32)
+  }
+
+  static async fromStorage (storage, opts = {}) {
+    const profileStorage = storage(opts.name || 'default')
+
+    const profile = await new Promise((resolve, reject) => {
+      profileStorage.stat((err, st) => {
+        if (err && err.code !== 'ENOENT') return reject(err)
+        if (err || st.size < 32 || opts.overwrite) {
+          const key = randomBytes(32)
+          return profileStorage.write(0, key, err => {
+            if (err) return reject(err)
+            return resolve(key)
+          })
+        }
+        profileStorage.read(0, 32, (err, key) => {
+          if (err) return reject(err)
+          return resolve(key)
+        })
+      })
+    })
+
+    return new this(profileStorage, profile, opts)
+  }
+}
+
+function deriveSeed (profile, token, name, output) {
+  if (token && token.length < 32) throw new Error('Token must be a Buffer with length >= 32')
+  if (!name || typeof name !== 'string') throw new Error('name must be a String')
+  if (!output) output = b4a.alloc(32)
+
+  blake2b.batch(output, [
+    NAMESPACE,
+    token || DEFAULT_TOKEN,
+    b4a.from(b4a.byteLength(name, 'ascii') + '\n' + name, 'ascii')
+  ], profile)
+
+  return output
+}
+
+function randomBytes (n) {
+  const buf = b4a.allocUnsafe(n)
+  sodium.randombytes_buf(buf)
+  return buf
+}

package/package.json

@@ -1,14 +1,14 @@
 {
   "name": "corestore",
-  "version": "6.0.0-beta1",
-  "description": "A Hypercore factory that stores and replicates linked cores.",
+  "version": "6.0.1-alpha.11",
+  "description": "A Hypercore factory that simplifies managing collections of cores.",
   "main": "index.js",
   "scripts": {
-    "test": "standard && tape test/*.js"
+    "test": "standard && brittle test/*.js"
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/andrewosh/corestore.git"
+    "url": "git+https://github.com/hypercore-protocol/corestore.git"
   },
   "keywords": [
     "corestore"
@@ -16,30 +16,23 @@
   "author": "Andrew Osheroff <andrewosh@gmail.com>",
   "license": "MIT",
   "bugs": {
-    "url": "https://github.com/andrewosh/corestore/issues"
+    "url": "https://github.com/hypercore-protocol/corestore/issues"
   },
-  "homepage": "https://github.com/andrewosh/corestore#readme",
+  "homepage": "https://github.com/hypercore-protocol/corestore#readme",
   "devDependencies": {
-    "@andrewosh/corestore-migration": "^5.8.2",
-    "hypercore-promisifier": "^1.0.3",
-    "random-access-memory": "^3.1.1",
-    "standard": "^16.0.3",
-    "tape": "^5.0.1"
+    "brittle": "^1.6.0",
+    "random-access-file": "^2.2.0",
+    "random-access-memory": "^3.1.2",
+    "standardx": "^7.0.0",
+    "tmp-promise": "^3.0.2"
   },
   "dependencies": {
-    "@jsdevtools/readdir-enhanced": "^6.0.4",
-    "custom-error-class": "^1.0.0",
+    "b4a": "^1.3.1",
+    "blake2b-universal": "^1.0.1",
     "derive-key": "^1.0.1",
-    "end-of-stream": "^1.4.4",
-    "fd-lock": "^1.1.1",
-    "hyperbee": "^1.1.1",
-    "hypercore": "^9.6.0",
-    "hypercore-crypto": "^2.2.0",
-    "hypercore-protocol": "^8.0.7",
-    "nanoresource-promise": "^2.0.0",
-    "random-access-file": "^2.1.4",
-    "random-access-storage": "^1.4.1",
-    "refpool": "^1.2.2",
-    "varint": "^6.0.0"
+    "hypercore": "next",
+    "hypercore-crypto": "^3.2.1",
+    "safety-catch": "^1.0.1",
+    "sodium-universal": "^3.0.4"
   }
 }