corestore 5.8.0 → 6.0.0-alpha.1

package/index.js

@@ -1,432 +1,223 @@
-const HypercoreProtocol = require('hypercore-protocol')
-const Nanoresource = require('nanoresource/emitter')
-const hypercore = require('hypercore')
-const hypercoreCrypto = require('hypercore-crypto')
-const datEncoding = require('dat-encoding')
-const maybe = require('call-me-maybe')
+const { EventEmitter } = require('events')
+const crypto = require('hypercore-crypto')
+const sodium = require('sodium-universal')
+const Hypercore = require('hypercore')
 
-const RefPool = require('refpool')
-const deriveSeed = require('derive-key')
-const derivedStorage = require('derived-key-storage')
-const raf = require('random-access-file')
+const KeyManager = require('./lib/keys')
 
-const MASTER_KEY_FILENAME = 'master_key'
-const NAMESPACE = 'corestore'
-const NAMESPACE_SEPERATOR = ':'
+const CORES_DIR = 'cores'
+const KEYS_DIR = 'keys'
+const USERDATA_NAME_KEY = '@corestore/name'
+const USERDATA_NAMESPACE_KEY = '@corestore/namespace'
+const DEFAULT_NAMESPACE = generateNamespace('@corestore/default')
 
-class InnerCorestore extends Nanoresource {
+module.exports = class Corestore extends EventEmitter {
   constructor (storage, opts = {}) {
     super()
 
-    if (typeof storage === 'string') storage = defaultStorage(storage)
-    if (typeof storage !== 'function') throw new Error('Storage should be a function or string')
-    this.storage = storage
+    this.storage = Hypercore.defaultStorage(storage, { lock: KEYS_DIR + '/profile' })
 
-    this.opts = opts
+    this.cores = opts._cores || new Map()
+    this.keys = opts.keys
 
-    this._replicationStreams = []
-    this.cache = new RefPool({
-      maxSize: opts.cacheSize || 1000,
-      close: core => {
-        core.close(err => {
-          if (err) this.emit('error', err)
-        })
-      }
-    })
-
-    // Generated in _open
-    this._masterKey = opts.masterKey || null
-    this._id = hypercoreCrypto.randomBytes(8)
-  }
-
-  // Nanoresource Methods
-
-  _open (cb) {
-    if (this._masterKey) return cb()
-    const keyStorage = this.storage(MASTER_KEY_FILENAME)
-    keyStorage.read(0, 32, (err, key) => {
-      if (err) {
-        this._masterKey = hypercoreCrypto.randomBytes(32)
-        return keyStorage.write(0, this._masterKey, err => {
-          if (err) return cb(err)
-          keyStorage.close(cb)
-        })
-      }
-      this._masterKey = key
-      keyStorage.close(cb)
-    })
-  }
-
-  _close (cb) {
-    let error = null
-    for (const { stream } of this._replicationStreams) {
-      stream.destroy()
-    }
-    if (!this.cache.size) return process.nextTick(cb, null)
-    let remaining = this.cache.size
-    for (const { value: core } of this.cache.entries.values()) {
-      core.close(err => {
-        if (err) error = err
-        if (!--remaining) {
-          if (error) return cb(error)
-          return cb(null)
-        }
-      })
-    }
-  }
-
-  // Private Methods
-
-  _checkIfExists (dkey, cb) {
-    dkey = encodeKey(dkey)
-    if (this.cache.has(dkey)) return process.nextTick(cb, null, true)
+    this._namespace = opts._namespace || DEFAULT_NAMESPACE
+    this._replicationStreams = opts._streams || []
 
-    const coreStorage = this.storage([dkey.slice(0, 2), dkey.slice(2, 4), dkey, 'key'].join('/'))
-
-    coreStorage.read(0, 32, (err, key) => {
-      if (err) return cb(err)
-      coreStorage.close(err => {
-        if (err) return cb(err)
-        if (!key) return cb(null, false)
-        return cb(null, true)
-      })
-    })
+    this._opening = opts._opening ? opts._opening.then(() => this._open()) : this._open()
+    this._opening.catch(noop)
+    this.ready = () => this._opening
   }
 
-  _injectIntoReplicationStreams (core) {
-    for (const { stream, opts } of this._replicationStreams) {
-      this._replicateCore(false, core, stream, { ...opts })
+  async _open () {
+    if (this.keys) {
+      this.keys = await this.keys // opts.keys can be a Promise that resolves to a KeyManager
+    } else {
+      this.keys = await KeyManager.fromStorage(p => this.storage(KEYS_DIR + '/' + p))
     }
   }
 
-  _replicateCore (isInitiator, core, mainStream, opts) {
-    if (!core) return
-    core.ready(function (err) {
-      if (err) return
-      core.replicate(isInitiator, {
-        ...opts,
-        stream: mainStream
-      })
-    })
-  }
-
-  _deriveSecret (namespace, name) {
-    return deriveSeed(namespace, this._masterKey, name)
-  }
-
-  _generateKeyPair (name) {
-    if (typeof name === 'string') name = Buffer.from(name)
-    else if (!name) name = hypercoreCrypto.randomBytes(32)
-
-    const seed = this._deriveSecret(NAMESPACE, name)
-
-    const keyPair = hypercoreCrypto.keyPair(seed)
-    const discoveryKey = hypercoreCrypto.discoveryKey(keyPair.publicKey)
-    return { name, publicKey: keyPair.publicKey, secretKey: keyPair.secretKey, discoveryKey }
-  }
-
-  _generateKeys (coreOpts) {
-    if (!coreOpts) coreOpts = {}
-    if (typeof coreOpts === 'string') coreOpts = Buffer.from(coreOpts, 'hex')
-    if (Buffer.isBuffer(coreOpts)) coreOpts = { key: coreOpts }
-
-    if (coreOpts.keyPair) {
-      const publicKey = coreOpts.keyPair.publicKey
-      const secretKey = coreOpts.keyPair.secretKey
+  async _generateKeys (opts) {
+    if (opts.discoveryKey) {
       return {
-        publicKey,
-        secretKey,
-        discoveryKey: hypercoreCrypto.discoveryKey(publicKey),
-        name: null
+        keyPair: null,
+        sign: null,
+        discoveryKey: opts.discoveryKey
       }
     }
-    if (coreOpts.key) {
-      const publicKey = decodeKey(coreOpts.key)
+    if (!opts.name) {
       return {
-        publicKey,
-        secretKey: null,
-        discoveryKey: hypercoreCrypto.discoveryKey(publicKey),
-        name: null
+        keyPair: {
+          publicKey: opts.publicKey,
+          secretKey: opts.secretKey
+        },
+        sign: opts.sign,
+        discoveryKey: crypto.discoveryKey(opts.publicKey)
       }
     }
-    if (coreOpts.default || coreOpts.name) {
-      if (!coreOpts.name) throw new Error('If the default option is set, a name must be specified.')
-      return this._generateKeyPair(coreOpts.name)
-    }
-    if (coreOpts.discoveryKey) {
-      const discoveryKey = decodeKey(coreOpts.discoveryKey)
-      return {
-        publicKey: null,
-        secretKey: null,
-        discoveryKey,
-        name: null
-      }
+    const { publicKey, sign } = await this.keys.createHypercoreKeyPair(opts.name, this._namespace)
+    return {
+      keyPair: {
+        publicKey,
+        secretKey: null
+      },
+      sign,
+      discoveryKey: crypto.discoveryKey(publicKey)
     }
-    return this._generateKeyPair(null)
   }
 
-  // Public Methods
+  async _postload (core) {
+    const name = await core.getUserData(USERDATA_NAME_KEY)
+    if (!name) return
 
-  isLoaded (coreOpts) {
-    const generatedKeys = this._generateKeys(coreOpts)
-    return this.cache.has(encodeKey(generatedKeys.discoveryKey))
-  }
+    const namespace = await core.getUserData(USERDATA_NAMESPACE_KEY)
+    const { publicKey, sign } = await this.keys.createHypercoreKeyPair(name.toString(), namespace)
+    if (!publicKey.equals(core.key)) throw new Error('Stored core key does not match the provided name')
 
-  isExternal (coreOpts) {
-    const generatedKeys = this._generateKeys(coreOpts)
-    const entry = this._cache.entry(encodeKey(generatedKeys.discoveryKey))
-    if (!entry) return false
-    return entry.refs !== 0
+    // TODO: Should Hypercore expose a helper for this, or should postload return keypair/sign?
+    core.sign = sign
+    core.key = publicKey
+    core.writable = true
   }
 
-  get (coreOpts = {}) {
-    if (!this.opened) throw new Error('Corestore.ready must be called before get.')
-
-    const self = this
+  async _preload (opts) {
+    await this.ready()
 
-    const generatedKeys = this._generateKeys(coreOpts)
-    const { publicKey, discoveryKey, secretKey } = generatedKeys
-    const id = encodeKey(discoveryKey)
+    const { discoveryKey, keyPair, sign } = await this._generateKeys(opts)
+    const id = discoveryKey.toString('hex')
 
-    const cached = this.cache.get(id)
-    if (cached) return cached
-
-    const storageRoot = [id.slice(0, 2), id.slice(2, 4), id].join('/')
-
-    const keyStorage = derivedStorage(createStorage, (name, cb) => {
-      if (name) {
-        const res = this._generateKeyPair(name)
-        if (discoveryKey && (!discoveryKey.equals((res.discoveryKey)))) {
-          return cb(new Error('Stored an incorrect name.'))
-        }
-        return cb(null, res)
+    while (this.cores.has(id)) {
+      const existing = this.cores.get(id)
+      if (existing) {
+        if (!existing.closing) return { from: existing, keyPair, sign }
+        await existing.close()
       }
-      if (secretKey) return cb(null, generatedKeys)
-      if (publicKey) return cb(null, { name: null, publicKey, secretKey: null })
-      const err = new Error('Unknown key pair.')
-      err.unknownKeyPair = true
-      return cb(err)
-    })
-
-    const cacheOpts = { ...this.opts.cache }
-    if (coreOpts.cache) {
-      if (coreOpts.cache.data === false) delete cacheOpts.data
-      if (coreOpts.cache.tree === false) delete cacheOpts.tree
     }
-    if (cacheOpts.data) cacheOpts.data = cacheOpts.data.namespace()
-    if (cacheOpts.tree) cacheOpts.tree = cacheOpts.tree.namespace()
-
-    const core = hypercore(name => {
-      if (name === 'key') return keyStorage.key
-      if (name === 'secret_key') return keyStorage.secretKey
-      return createStorage(name)
-    }, publicKey, {
-      ...this.opts,
-      ...coreOpts,
-      cache: cacheOpts,
-      createIfMissing: !!publicKey
-    })
-
-    this.cache.set(id, core)
-    core.ifAvailable.wait()
 
-    var errored = false
-    core.once('error', onerror)
-    core.once('ready', onready)
-    core.once('close', onclose)
-
-    return core
-
-    function onready () {
-      if (errored) return
-      self.emit('feed', core, coreOpts)
-      core.removeListener('error', onerror)
-      self._injectIntoReplicationStreams(core)
-      // TODO: nexttick here needed? prob not, just legacy
-      process.nextTick(() => core.ifAvailable.continue())
+    const userData = {}
+    if (opts.name) {
+      userData[USERDATA_NAME_KEY] = Buffer.from(opts.name)
+      userData[USERDATA_NAMESPACE_KEY] = this._namespace
     }
 
-    function onerror (err) {
-      errored = true
-      core.ifAvailable.continue()
-      self.cache.delete(id)
-      if (err.unknownKeyPair) {
-        // If an error occurs during creation by discovery key, then that core does not exist on disk.
-        // TODO: This should not throw, but should propagate somehow.
-      }
-    }
+    // No more async ticks allowed after this point -- necessary for caching
 
-    function onclose () {
-      self.cache.delete(id)
-    }
-
-    function createStorage (name) {
-      return self.storage(storageRoot + '/' + name)
-    }
-  }
-
-  replicate (isInitiator, cores, replicationOpts = {}) {
-    const self = this
-
-    const finalOpts = { ...this.opts, ...replicationOpts }
-    const mainStream = replicationOpts.stream || new HypercoreProtocol(isInitiator, { ...finalOpts })
-    var closed = false
-
-    for (const core of cores) {
-      this._replicateCore(isInitiator, core, mainStream, { ...finalOpts })
-    }
-
-    mainStream.on('discovery-key', ondiscoverykey)
-    mainStream.on('finish', onclose)
-    mainStream.on('end', onclose)
-    mainStream.on('close', onclose)
-
-    const streamState = { stream: mainStream, opts: finalOpts }
-    this._replicationStreams.push(streamState)
-
-    return mainStream
-
-    function ondiscoverykey (dkey) {
-      // Get will automatically add the core to all replication streams.
-      self._checkIfExists(dkey, (err, exists) => {
-        if (closed) return
-        if (err || !exists) return mainStream.close(dkey)
-        const passiveCore = self.get({ discoveryKey: dkey })
-        self._replicateCore(false, passiveCore, mainStream, { ...finalOpts })
-      })
-    }
-
-    function onclose () {
-      if (!closed) {
-        self._replicationStreams.splice(self._replicationStreams.indexOf(streamState), 1)
-        closed = true
-      }
-    }
-  }
-}
-
-class Corestore extends Nanoresource {
-  constructor (storage, opts = {}) {
-    super()
-
-    this.storage = storage
-    this.name = opts.name || 'default'
-    this.inner = opts.inner || new InnerCorestore(storage, opts)
-    this.cache = this.inner.cache
-    this.store = this // Backwards-compat for NamespacedCorestore
-
-    this._parent = opts.parent
-    this._isNamespaced = !!opts.name
-    this._openedCores = new Map()
-
-    const onfeed = feed => this.emit('feed', feed)
-    const onerror = err => this.emit('error', err)
-    this.inner.on('feed', onfeed)
-    this.inner.on('error', onerror)
-    this._unlisten = () => {
-      this.inner.removeListener('feed', onfeed)
-      this.inner.removeListener('error', onerror)
-    }
-  }
-
-  ready (cb) {
-    return maybe(cb, new Promise((resolve, reject) => {
-      this.open(err => {
-        if (err) return reject(err)
-        return resolve()
-      })
-    }))
-  }
-
-  // Nanoresource Methods
-
-  _open (cb) {
-    return this.inner.open(cb)
-  }
+    const storageRoot = [CORES_DIR, id.slice(0, 2), id.slice(2, 4), id].join('/')
+    const core = new Hypercore(p => this.storage(storageRoot + '/' + p), {
+      autoClose: true,
+      keyPair: {
+        publicKey: keyPair.publicKey,
+        secretKey: null
+      },
+      userData,
+      sign: null,
+      postload: this._postload.bind(this),
+      createIfMissing: !!opts.keyPair
+    })
 
-  _close (cb) {
-    this._unlisten()
-    if (!this._parent) return this.inner.close(cb)
-    for (const dkey of this._openedCores) {
-      this.cache.decrement(dkey)
+    this.cores.set(id, core)
+    for (const stream of this._replicationStreams) {
+      core.replicate(stream)
     }
-    return process.nextTick(cb, null)
-  }
-
-  // Private Methods
+    core.once('close', () => {
+      this.cores.delete(id)
+    })
 
-  _maybeIncrement (core) {
-    const id = encodeKey(core.discoveryKey)
-    if (this._openedCores.has(id)) return
-    this._openedCores.set(id, core)
-    this.cache.increment(id)
+    return { from: core, keyPair, sign }
   }
 
-  // Public Methods
-
-  get (coreOpts = {}) {
-    if (Buffer.isBuffer(coreOpts)) coreOpts = { key: coreOpts }
-    const core = this.inner.get(coreOpts)
-    this._maybeIncrement(core)
+  get (opts = {}) {
+    opts = validateGetOptions(opts)
+    const core = new Hypercore(null, {
+      ...opts,
+      name: null,
+      preload: () => this._preload(opts)
+    })
     return core
   }
 
-  default (coreOpts = {}) {
-    if (Buffer.isBuffer(coreOpts)) coreOpts = { key: coreOpts }
-    return this.get({ ...coreOpts, name: this.name })
+  replicate (opts = {}) {
+    const stream = isStream(opts) ? opts : (opts.stream || Hypercore.createProtocolStream(opts))
+    for (const core of this.cores.values()) {
+      core.replicate(stream)
+    }
+    stream.on('discovery-key', discoveryKey => {
+      const core = this.get({ discoveryKey })
+      core.ready().then(() => {
+        core.replicate(stream)
+      }, () => {
+        stream.close(discoveryKey)
+      })
+    })
+    this._replicationStreams.push(stream)
+    stream.once('close', () => {
+      this._replicationStreams.splice(this._replicationStreams.indexOf(stream), 1)
+    })
+    return stream
   }
 
   namespace (name) {
-    if (!name) name = hypercoreCrypto.randomBytes(32)
-    if (Buffer.isBuffer(name)) name = name.toString('hex')
-    name = this._isNamespaced ? this.name + NAMESPACE_SEPERATOR + name : name
+    if (!Buffer.isBuffer(name)) name = Buffer.from(name)
     return new Corestore(this.storage, {
-      inner: this.inner,
-      parent: this,
-      name
+      _namespace: generateNamespace(this._namespace, name),
+      _opening: this._opening,
+      _cores: this.cores,
+      _streams: this._replicationStreams,
+      keys: this._opening.then(() => this.keys)
     })
   }
 
-  replicate (isInitiator, opts) {
-    const cores = !this._parent ? allReferenced(this.cache) : this._openedCores.values()
-    return this.inner.replicate(isInitiator, cores, opts)
-  }
-
-  isLoaded (coreOpts) {
-    return this.inner.isLoaded(coreOpts)
-  }
-
-  isExternal (coreOpts) {
-    return this.inner.isExternal(coreOpts)
+  async _close () {
+    if (this._closing) return this._closing
+    const closePromises = []
+    for (const core of this.cores.values()) {
+      closePromises.push(core.close())
+    }
+    await Promise.allSettled(closePromises)
+    for (const stream of this._replicationStreams) {
+      stream.destroy()
+    }
   }
 
-  list () {
-    return new Map([...this._openedCores])
+  close () {
+    if (this._closing) return this._closing
+    this._closing = this._close()
+    this._closing.catch(noop)
+    return this._closing
   }
-}
 
-function * allReferenced (cache) {
-  for (const entry of cache.entries.values()) {
-    if (entry.refs > 0) yield entry.value
-    continue
+  static createToken () {
+    return KeyManager.createToken()
   }
 }
 
-function encodeKey (key) {
-  return Buffer.isBuffer(key) ? datEncoding.encode(key) : key
+function validateGetOptions (opts) {
+  if (Buffer.isBuffer(opts)) return { key: opts, publicKey: opts }
+  if (opts.key) {
+    opts.publicKey = opts.key
+  }
+  if (opts.keyPair) {
+    opts.publicKey = opts.keyPair.publicKey
+    opts.secretKey = opts.keyPair.secretKey
+  }
+  if (opts.name && typeof opts.name !== 'string') throw new Error('name option must be a String')
+  if (opts.name && opts.secretKey) throw new Error('Cannot provide both a name and a secret key')
+  if (opts.publicKey && !Buffer.isBuffer(opts.publicKey)) throw new Error('publicKey option must be a Buffer')
+  if (opts.secretKey && !Buffer.isBuffer(opts.secretKey)) throw new Error('secretKey option must be a Buffer')
+  if (opts.discoveryKey && !Buffer.isBuffer(opts.discoveryKey)) throw new Error('discoveryKey option must be a Buffer')
+  if (!opts.name && !opts.publicKey) throw new Error('Must provide either a name or a publicKey')
+  return opts
 }
 
-function decodeKey (key) {
-  return (typeof key === 'string') ? datEncoding.decode(key) : key
+function generateNamespace (first, second) {
+  if (!Buffer.isBuffer(first)) first = Buffer.from(first)
+  if (second && !Buffer.isBuffer(second)) second = Buffer.from(second)
+  const out = Buffer.allocUnsafe(32)
+  sodium.crypto_generichash(out, second ? Buffer.concat([first, second]) : first)
+  return out
 }
 
-function defaultStorage (dir) {
-  return function (name) {
-    try {
-      var lock = name.endsWith('/bitfield') ? require('fd-lock') : null
-    } catch (err) {}
-    return raf(name, { directory: dir, lock: lock })
-  }
+function isStream (s) {
+  return typeof s === 'object' && s && typeof s.pipe === 'function'
 }
 
-module.exports = Corestore
+function noop () {}