corestore 6.0.0-beta2 → 6.0.0

package/.github/workflows/test-node.yml

@@ -1,5 +1,4 @@
-name: Test on Node.js
-
+name: Build Status
 on:
   push:
     branches:
@@ -11,13 +10,13 @@ jobs:
   build:
     strategy:
       matrix:
-        node-version: [12.x, 14.x]
-        os: [ubuntu-16.04, macos-latest, windows-latest]
+        node-version: [lts/*]
+        os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v2
       with:
         node-version: ${{ matrix.node-version }}
     - run: npm install

package/README.md

@@ -1,2 +1,59 @@
-# neocorestore
-A new take on Corestore
+# Corestore
+
+Corestore is a Hypercore factory that makes it easier to manage large collections of named Hypercores.
+
+Corestore provides:
+1. __Key Derivation__ - All writable Hypercore keys are derived from a single master key and a user-provided name.
+2. __Session Handling__ - If a single Hypercore is loaded multiple times through the `get` method, the underlying resources will only be opened once (using Hypercore 10's new session feature). Once all sessions are closed, the resources will be released.
+3. __Storage Management__ - Hypercores can be stored in any random-access-storage instance, where they will be keyed by their discovery keys.
+4. __Namespacing__ - You can share a single Corestore instance between multiple applications or components without worrying about naming collisions by creating "namespaces" (e.g. `corestore.namespace('my-app').get({ name: 'main' }) 
+
+### Installation
+`npm install corestore`
+
+### Usage
+A corestore instance can be constructed with a random-access-storage module, a function that returns a random-access-storage module given a path, or a string. If a string is specified, it will be assumed to be a path to a local storage directory:
+```js
+const Corestore = require('corestore')
+
+const store = new Corestore('./my-storage')
+const core1 = store.get({ name: 'core-1' })
+const core2 = store.get({ name: 'core-2' })
+```
+
+### API
+#### `const store = new Corestore(storage)`
+Create a new Corestore instance.
+
+`storage` can be either a random-access-storage module, a string, or a function that takes a path and returns an random-access-storage instance.
+
+#### `const core = store.get(key | { name: 'a-name', ...hypercoreOpts})`
+Loads a Hypercore, either by name (if the `name` option is provided), or from the provided key (if the first argument is a Buffer, or if the `key` options is set).
+
+If that Hypercore has previously been loaded, subsequent calls to `get` will return a new Hypercore session on the existing core.
+
+All other options besides `name` and `key` will be forwarded to the Hypercore constructor.
+
+#### `const stream = store.replicate(opts)`
+Creates a replication stream that's capable of replicating all Hypercores that are managed by the Corestore, assuming the remote peer has the correct capabilities.
+
+`opts` will be forwarded to Hypercore's `replicate` function.
+
+Corestore replicates in an "all-to-all" fashion, meaning that when replication begins, it will attempt to replicate every Hypercore that's currently loaded and in memory. These attempts will fail if the remote side doesn't have a Hypercore's capability -- Corestore replication does not exchange Hypercore keys.
+
+If the remote side dynamically adds a new Hypercore to the replication stream, Corestore will load and replicatethat core if possible.
+
+#### `const store = store.namespace(name)`
+Create a new namespaced Corestore. Namespacing is useful if you're going to be sharing a single Corestore instance between many applications or components, as it prevents name collisions.
+
+Namespaces can be chained:
+```js
+const ns1 = store.namespace('a')
+const ns2 = ns1.namespace('b')
+const core1 = ns1.get({ name: 'main' }) // These will load different Hypercores
+const core2 = ns2.get({ name: 'main' })
+```
+
+### License
+MIT
+

package/index.js

@@ -1,122 +1,373 @@
-const { NanoresourcePromise: Nanoresource } = require('nanoresource-promise/emitter')
-const raf = require('random-access-file')
+const { EventEmitter } = require('events')
+const safetyCatch = require('safety-catch')
+const crypto = require('hypercore-crypto')
+const sodium = require('sodium-universal')
+const Hypercore = require('hypercore')
+const Xache = require('xache')
+const b4a = require('b4a')
 
-const Replicator = require('./lib/replicator')
-const Index = require('./lib/db')
-const Loader = require('./lib/loader')
-const errors = require('./lib/errors')
+const [NS] = crypto.namespace('corestore', 1)
+const DEFAULT_NAMESPACE = b4a.alloc(32) // This is meant to be 32 0-bytes
 
-module.exports = class Corestore extends Nanoresource {
+const CORES_DIR = 'cores'
+const PRIMARY_KEY_FILE_NAME = 'primary-key'
+const USERDATA_NAME_KEY = 'corestore/name'
+const USERDATA_NAMESPACE_KEY = 'corestore/namespace'
+
+module.exports = class Corestore extends EventEmitter {
   constructor (storage, opts = {}) {
     super()
 
-    if (typeof storage === 'string') storage = defaultStorage(storage)
-    if (typeof storage !== 'function') throw new errors.InvalidStorageError()
-    this.storage = storage
+    this.storage = Hypercore.defaultStorage(storage, { lock: PRIMARY_KEY_FILE_NAME })
+    this.cores = opts._cores || new Map()
+    this.primaryKey = null
+    this.cache = !!opts.cache
+
+    this._keyStorage = null
+    this._primaryKey = opts.primaryKey
+    this._namespace = opts.namespace || DEFAULT_NAMESPACE
+    this._replicationStreams = opts._streams || []
+    this._overwrite = opts.overwrite === true
+
+    this._streamSessions = opts._streamSessions || new Map()
+    this._sessions = new Set() // sessions for THIS namespace
 
-    this._namespace = opts.namespace || []
-    this._db = opts._db || new Index(this.storage, opts)
-    this._loader = opts._loader || new Loader(this.storage, this._db, opts)
-    this._replicator = opts._replicator || new Replicator(this._loader, opts)
+    this._findingPeersCount = 0
+    this._findingPeers = []
 
-    this._db.on('error', err => this.emit('db-error', err))
-    this._loader.on('error', err => this.emit('error', err))
-    this._loader.on('core', (core, opts) => this.emit('core', core, opts))
-    this._loader.on('core', (core, opts) => this.emit('feed', core, opts))
+    if (this._namespace.byteLength !== 32) throw new Error('Namespace must be a 32-byte Buffer or Uint8Array')
 
-    this.ready = this.open.bind(this)
+    this._opening = opts._opening ? opts._opening.then(() => this._open()) : this._open()
+    this._opening.catch(safetyCatch)
+    this.ready = () => this._opening
   }
 
-  get cache () {
-    return this._loader.cache
+  findingPeers () {
+    let done = false
+    this._incFindingPeers()
+
+    return () => {
+      if (done) return
+      done = true
+      this._decFindingPeers()
+    }
   }
 
-  // Nanoresource Methods
+  _incFindingPeers () {
+    if (++this._findingPeersCount !== 1) return
+
+    for (const core of this._sessions) {
+      this._findingPeers.push(core.findingPeers())
+    }
+  }
+
+  _decFindingPeers () {
+    if (--this._findingPeersCount !== 0) return
+
+    while (this._findingPeers.length > 0) {
+      this._findingPeers.pop()()
+    }
+  }
 
   async _open () {
-    await this._db.open()
-    return this._loader.open()
+    if (this._primaryKey) {
+      this.primaryKey = await this._primaryKey
+      return this.primaryKey
+    }
+    this._keyStorage = this.storage(PRIMARY_KEY_FILE_NAME)
+    this.primaryKey = await new Promise((resolve, reject) => {
+      this._keyStorage.stat((err, st) => {
+        if (err && err.code !== 'ENOENT') return reject(err)
+        if (err || st.size < 32 || this._overwrite) {
+          const key = crypto.randomBytes(32)
+          return this._keyStorage.write(0, key, err => {
+            if (err) return reject(err)
+            return resolve(key)
+          })
+        }
+        this._keyStorage.read(0, 32, (err, key) => {
+          if (err) return reject(err)
+          return resolve(key)
+        })
+      })
+    })
+    return this.primaryKey
   }
 
-  async _close () {
-    await this._replicator.close()
-    await this._loader.close()
-    return this._db.close()
+  async _generateKeys (opts) {
+    if (opts._discoveryKey) {
+      return {
+        keyPair: null,
+        auth: null,
+        discoveryKey: opts._discoveryKey
+      }
+    }
+    if (!opts.name) {
+      return {
+        keyPair: {
+          publicKey: opts.publicKey,
+          secretKey: opts.secretKey
+        },
+        sign: opts.sign,
+        auth: opts.auth,
+        discoveryKey: crypto.discoveryKey(opts.publicKey)
+      }
+    }
+    const { publicKey, auth } = await this.createKeyPair(opts.name)
+    return {
+      keyPair: {
+        publicKey,
+        secretKey: null
+      },
+      auth,
+      discoveryKey: crypto.discoveryKey(publicKey)
+    }
   }
 
-  // Private Methods
+  _getPrereadyUserData (core, key) {
+    // Need to manually read the header values before the Hypercore is ready, hence the ugliness.
+    for (const { key: savedKey, value } of core.core.header.userData) {
+      if (key === savedKey) return value
+    }
+    return null
+  }
+
+  async _preready (core) {
+    const name = this._getPrereadyUserData(core, USERDATA_NAME_KEY)
+    if (!name) return
+
+    const namespace = this._getPrereadyUserData(core, USERDATA_NAMESPACE_KEY)
+    const { publicKey, auth } = await this.createKeyPair(b4a.toString(name), namespace)
+    if (!b4a.equals(publicKey, core.key)) throw new Error('Stored core key does not match the provided name')
+
+    // TODO: Should Hypercore expose a helper for this, or should preready return keypair/auth?
+    core.auth = auth
+    core.key = publicKey
+    core.writable = true
+  }
 
-  _validateGetOptions (opts) {
-    if (typeof opts === 'object') {
-      if (!Buffer.isBuffer(opts)) {
-        if (!opts.key && !opts.name && !opts.keyPair) throw new errors.InvalidOptionsError()
+  async _preload (opts) {
+    await this.ready()
+
+    const { discoveryKey, keyPair, auth } = await this._generateKeys(opts)
+    const id = b4a.toString(discoveryKey, 'hex')
+
+    while (this.cores.has(id)) {
+      const existing = this.cores.get(id)
+      if (existing.opened && !existing.closing) return { from: existing, keyPair, auth }
+      if (existing.closing) {
+        await existing.close()
       } else {
-        opts = { key: opts }
+        await existing.ready().catch(safetyCatch)
       }
-    } else {
-      opts = { key: Buffer.from(opts, 'hex') }
     }
-    if (opts.key && typeof opts.key === 'string') opts.key = Buffer.from(opts.key, 'hex')
-    if (opts.key && opts.key.length !== 32) throw new errors.InvalidKeyError()
-    if (opts.name && !Array.isArray(opts.name)) opts.name = [opts.name]
-    return opts
+
+    const userData = {}
+    if (opts.name) {
+      userData[USERDATA_NAME_KEY] = b4a.from(opts.name)
+      userData[USERDATA_NAMESPACE_KEY] = this._namespace
+    }
+
+    // No more async ticks allowed after this point -- necessary for caching
+
+    const storageRoot = [CORES_DIR, id.slice(0, 2), id.slice(2, 4), id].join('/')
+    const core = new Hypercore(p => this.storage(storageRoot + '/' + p), {
+      _preready: this._preready.bind(this),
+      autoClose: true,
+      encryptionKey: opts.encryptionKey || null,
+      userData,
+      auth,
+      cache: opts.cache,
+      createIfMissing: !opts._discoveryKey,
+      keyPair: keyPair && keyPair.publicKey
+        ? {
+            publicKey: keyPair.publicKey,
+            secretKey: null
+          }
+        : null
+    })
+
+    this.cores.set(id, core)
+    core.ready().then(() => {
+      for (const { stream } of this._replicationStreams) {
+        const sessions = this._streamSessions.get(stream)
+        const session = core.session()
+        sessions.push(session)
+        core.replicate(stream)
+      }
+    }, () => {
+      this.cores.delete(id)
+    })
+    core.once('close', () => {
+      this.cores.delete(id)
+    })
+
+    return { from: core, keyPair, auth }
   }
 
-  // Public Methods
+  async createKeyPair (name, namespace = this._namespace) {
+    if (!this.primaryKey) await this._opening
 
-  get (opts = {}) {
-    opts = this._validateGetOptions(opts)
-    if (!this.opened) this.open().catch(err => this.emit('error', err))
-    return this._loader.get(this._namespace, opts)
+    const keyPair = {
+      publicKey: b4a.allocUnsafe(sodium.crypto_sign_PUBLICKEYBYTES),
+      secretKey: b4a.alloc(sodium.crypto_sign_SECRETKEYBYTES),
+      auth: {
+        sign: (msg) => sign(keyPair, msg),
+        verify: (signable, signature) => {
+          return crypto.verify(signable, signature, keyPair.publicKey)
+        }
+      }
+    }
+
+    const seed = deriveSeed(this.primaryKey, namespace, name)
+    sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair.secretKey, seed)
+
+    return keyPair
   }
 
-  namespace (name) {
-    if (!name) throw new Error('A name must be provided as the first argument.')
-    if (Buffer.isBuffer(name)) name = name.toString('hex')
-    return new Corestore(this.storage, {
-      namespace: [...this._namespace, name],
-      _db: this._db,
-      _loader: this._loader,
-      _replicator: this._replicator
+  get (opts = {}) {
+    opts = validateGetOptions(opts)
+
+    if (opts.cache !== false) {
+      opts.cache = opts.cache === true || (this.cache && !opts.cache) ? defaultCache() : opts.cache
+    }
+
+    const core = new Hypercore(null, {
+      ...opts,
+      name: null,
+      preload: () => this._preload(opts)
     })
-  }
 
-  replicate (isInitiator, opts) {
-    return this._replicator.replicate(isInitiator, opts)
+    this._sessions.add(core)
+    if (this._findingPeersCount > 0) {
+      this._findingPeers.push(core.findingPeers())
+    }
+
+    core.once('close', () => {
+      // technically better to also clear _findingPeers if we added it,
+      // but the lifecycle for those are pretty short so prob not worth the complexity
+      // as _decFindingPeers clear them all.
+      this._sessions.delete(core)
+    })
+
+    return core
   }
 
-  // Backup/Restore
+  replicate (isInitiator, opts) {
+    const isExternal = isStream(isInitiator) || !!(opts && opts.stream)
+    const stream = Hypercore.createProtocolStream(isInitiator, {
+      ...opts,
+      ondiscoverykey: discoveryKey => {
+        const core = this.get({ _discoveryKey: discoveryKey })
+        return core.ready().catch(safetyCatch)
+      }
+    })
 
-  async backup () {
-    if (!this.opened) await this.open()
-    const allCores = await this._db.getAllCores()
-    return {
-      masterKey: this._loader.masterKey.toString('hex'),
-      cores: allCores
+    const sessions = []
+    for (const core of this.cores.values()) {
+      if (!core.opened) continue // If the core is not opened, it will be replicated in preload.
+      const session = core.session()
+      sessions.push(session)
+      core.replicate(stream)
     }
+
+    const streamRecord = { stream, isExternal }
+    this._replicationStreams.push(streamRecord)
+    this._streamSessions.set(stream, sessions)
+
+    stream.once('close', () => {
+      this._replicationStreams.splice(this._replicationStreams.indexOf(streamRecord), 1)
+      this._streamSessions.delete(stream)
+      Promise.all(sessions.map(s => s.close())).catch(safetyCatch)
+    })
+    return stream
   }
 
-  restore (manifest) {
-    return this._db.restore(manifest)
+  namespace (name) {
+    return new Corestore(this.storage, {
+      primaryKey: this._opening.then(() => this.primaryKey),
+      namespace: generateNamespace(this._namespace, name),
+      cache: this.cache,
+      _opening: this._opening,
+      _cores: this.cores,
+      _streams: this._replicationStreams,
+      _streamSessions: this._streamSessions
+    })
   }
 
-  static async restore (manifest, target) {
-    if (!manifest || !manifest.masterKey) throw new Error('Malformed manifest.')
-    const store = new this(target, {
-      overwriteMasterKey: true,
-      masterKey: Buffer.from(manifest.masterKey, 'hex')
+  async _close () {
+    await this._opening
+    if (!b4a.equals(this._namespace, DEFAULT_NAMESPACE)) {
+      // namespaces should not release resources on close
+      // TODO: Refactor the namespace close logic to actually close sessions with ref counting
+      return
+    }
+    const closePromises = []
+    for (const core of this.cores.values()) {
+      closePromises.push(core.close())
+    }
+    await Promise.allSettled(closePromises)
+    for (const { stream, isExternal } of this._replicationStreams) {
+      // Only close streams that were created by the Corestore
+      if (!isExternal) stream.destroy()
+    }
+    if (!this._keyStorage) return
+    await new Promise((resolve, reject) => {
+      this._keyStorage.close(err => {
+        if (err) return reject(err)
+        return resolve(null)
+      })
     })
-    await store.restore(manifest)
-    return store
   }
+
+  close () {
+    if (this._closing) return this._closing
+    this._closing = this._close()
+    this._closing.catch(safetyCatch)
+    return this._closing
+  }
+}
+
+function sign (keyPair, message) {
+  if (!keyPair.secretKey) throw new Error('Invalid key pair')
+  return crypto.sign(message, keyPair.secretKey)
 }
 
-function defaultStorage (dir) {
-  return function (name) {
-    let lock = null
-    try {
-      lock = name.endsWith('/bitfield') ? require('fd-lock') : null
-    } catch (err) {}
-    return raf(name, { directory: dir, lock: lock })
+function validateGetOptions (opts) {
+  if (b4a.isBuffer(opts)) return { key: opts, publicKey: opts }
+  if (opts.key) {
+    opts.publicKey = opts.key
+  }
+  if (opts.keyPair) {
+    opts.publicKey = opts.keyPair.publicKey
+    opts.secretKey = opts.keyPair.secretKey
   }
+  if (opts.name && typeof opts.name !== 'string') throw new Error('name option must be a String')
+  if (opts.name && opts.secretKey) throw new Error('Cannot provide both a name and a secret key')
+  if (opts.publicKey && !b4a.isBuffer(opts.publicKey)) throw new Error('publicKey option must be a Buffer or Uint8Array')
+  if (opts.secretKey && !b4a.isBuffer(opts.secretKey)) throw new Error('secretKey option must be a Buffer or Uint8Array')
+  if (!opts._discoveryKey && (!opts.name && !opts.publicKey)) throw new Error('Must provide either a name or a publicKey')
+  return opts
+}
+
+function generateNamespace (namespace, name) {
+  if (!b4a.isBuffer(name)) name = b4a.from(name)
+  const out = b4a.allocUnsafe(32)
+  sodium.crypto_generichash_batch(out, [namespace, name])
+  return out
+}
+
+function deriveSeed (primaryKey, namespace, name) {
+  if (!b4a.isBuffer(name)) name = b4a.from(name)
+  const out = b4a.alloc(32)
+  sodium.crypto_generichash_batch(out, [NS, namespace, name], primaryKey)
+  return out
+}
+
+function defaultCache () {
+  return new Xache({ maxSize: 65536, maxAge: 0 })
+}
+
+function isStream (s) {
+  return typeof s === 'object' && s && typeof s.pipe === 'function'
 }

package/package.json

@@ -1,14 +1,14 @@
 {
   "name": "corestore",
-  "version": "6.0.0-beta2",
-  "description": "A Hypercore factory that stores and replicates linked cores.",
+  "version": "6.0.0",
+  "description": "A Hypercore factory that simplifies managing collections of cores.",
   "main": "index.js",
   "scripts": {
-    "test": "standard && tape test/*.js"
+    "test": "standard && brittle test/*.js"
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/andrewosh/corestore.git"
+    "url": "git+https://github.com/hypercore-protocol/corestore.git"
   },
   "keywords": [
     "corestore"
@@ -16,30 +16,20 @@
   "author": "Andrew Osheroff <andrewosh@gmail.com>",
   "license": "MIT",
   "bugs": {
-    "url": "https://github.com/andrewosh/corestore/issues"
+    "url": "https://github.com/hypercore-protocol/corestore/issues"
   },
-  "homepage": "https://github.com/andrewosh/corestore#readme",
+  "homepage": "https://github.com/hypercore-protocol/corestore#readme",
   "devDependencies": {
-    "@andrewosh/corestore-migration": "^5.8.2",
-    "hypercore-promisifier": "^1.0.3",
-    "random-access-memory": "^3.1.1",
-    "standard": "^16.0.3",
-    "tape": "^5.0.1"
+    "brittle": "^3.0.0",
+    "random-access-memory": "^4.0.0",
+    "standardx": "^7.0.0"
   },
   "dependencies": {
-    "@jsdevtools/readdir-enhanced": "^6.0.4",
-    "custom-error-class": "^1.0.0",
-    "derive-key": "^1.0.1",
-    "end-of-stream": "^1.4.4",
-    "fd-lock": "^1.1.1",
-    "hyperbee": "^1.1.1",
-    "hypercore": "^9.6.0",
-    "hypercore-crypto": "^2.2.0",
-    "hypercore-protocol": "^8.0.7",
-    "nanoresource-promise": "^2.0.0",
-    "random-access-file": "^2.1.4",
-    "random-access-storage": "^1.4.1",
-    "refpool": "^1.2.2",
-    "varint": "^6.0.0"
+    "b4a": "^1.3.1",
+    "hypercore": "v10.0.0",
+    "hypercore-crypto": "^3.2.1",
+    "safety-catch": "^1.0.1",
+    "sodium-universal": "^3.0.4",
+    "xache": "^1.1.0"
   }
 }