corestore 6.0.0-alpha.4 → 6.0.0

package/.github/workflows/test-node.yml

@@ -1,23 +1,22 @@
-name: Test on Node.js
-
+name: Build Status
 on:
   push:
     branches:
-      - main
+      - master
   pull_request:
     branches:
-      - main
+      - master
 jobs:
   build:
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 16.x]
+        node-version: [lts/*]
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v2
       with:
         node-version: ${{ matrix.node-version }}
     - run: npm install

package/README.md

@@ -1,4 +1,4 @@
-# Corestore v6
+# Corestore
 
 Corestore is a Hypercore factory that makes it easier to manage large collections of named Hypercores.
 
@@ -9,7 +9,7 @@ Corestore provides:
 4. __Namespacing__ - You can share a single Corestore instance between multiple applications or components without worrying about naming collisions by creating "namespaces" (e.g. `corestore.namespace('my-app').get({ name: 'main' }) 
 
 ### Installation
-`npm install corestore@next`
+`npm install corestore`
 
 ### Usage
 A corestore instance can be constructed with a random-access-storage module, a function that returns a random-access-storage module given a path, or a string. If a string is specified, it will be assumed to be a path to a local storage directory:

package/index.js

@@ -1,47 +1,105 @@
 const { EventEmitter } = require('events')
+const safetyCatch = require('safety-catch')
 const crypto = require('hypercore-crypto')
 const sodium = require('sodium-universal')
 const Hypercore = require('hypercore')
+const Xache = require('xache')
+const b4a = require('b4a')
 
-const KeyManager = require('./lib/keys')
+const [NS] = crypto.namespace('corestore', 1)
+const DEFAULT_NAMESPACE = b4a.alloc(32) // This is meant to be 32 0-bytes
 
 const CORES_DIR = 'cores'
-const PROFILES_DIR = 'profiles'
-const USERDATA_NAME_KEY = '@corestore/name'
-const USERDATA_NAMESPACE_KEY = '@corestore/namespace'
-const DEFAULT_NAMESPACE = generateNamespace('@corestore/default')
+const PRIMARY_KEY_FILE_NAME = 'primary-key'
+const USERDATA_NAME_KEY = 'corestore/name'
+const USERDATA_NAMESPACE_KEY = 'corestore/namespace'
 
 module.exports = class Corestore extends EventEmitter {
   constructor (storage, opts = {}) {
     super()
 
-    this.storage = Hypercore.defaultStorage(storage, { lock: PROFILES_DIR + '/default' })
-
+    this.storage = Hypercore.defaultStorage(storage, { lock: PRIMARY_KEY_FILE_NAME })
     this.cores = opts._cores || new Map()
-    this.keys = opts.keys
+    this.primaryKey = null
+    this.cache = !!opts.cache
 
-    this._namespace = opts._namespace || DEFAULT_NAMESPACE
+    this._keyStorage = null
+    this._primaryKey = opts.primaryKey
+    this._namespace = opts.namespace || DEFAULT_NAMESPACE
     this._replicationStreams = opts._streams || []
+    this._overwrite = opts.overwrite === true
+
+    this._streamSessions = opts._streamSessions || new Map()
+    this._sessions = new Set() // sessions for THIS namespace
+
+    this._findingPeersCount = 0
+    this._findingPeers = []
+
+    if (this._namespace.byteLength !== 32) throw new Error('Namespace must be a 32-byte Buffer or Uint8Array')
 
     this._opening = opts._opening ? opts._opening.then(() => this._open()) : this._open()
-    this._opening.catch(noop)
+    this._opening.catch(safetyCatch)
     this.ready = () => this._opening
   }
 
+  findingPeers () {
+    let done = false
+    this._incFindingPeers()
+
+    return () => {
+      if (done) return
+      done = true
+      this._decFindingPeers()
+    }
+  }
+
+  _incFindingPeers () {
+    if (++this._findingPeersCount !== 1) return
+
+    for (const core of this._sessions) {
+      this._findingPeers.push(core.findingPeers())
+    }
+  }
+
+  _decFindingPeers () {
+    if (--this._findingPeersCount !== 0) return
+
+    while (this._findingPeers.length > 0) {
+      this._findingPeers.pop()()
+    }
+  }
+
   async _open () {
-    if (this.keys) {
-      this.keys = await this.keys // opts.keys can be a Promise that resolves to a KeyManager
-    } else {
-      this.keys = await KeyManager.fromStorage(p => this.storage(PROFILES_DIR + '/' + p))
+    if (this._primaryKey) {
+      this.primaryKey = await this._primaryKey
+      return this.primaryKey
     }
+    this._keyStorage = this.storage(PRIMARY_KEY_FILE_NAME)
+    this.primaryKey = await new Promise((resolve, reject) => {
+      this._keyStorage.stat((err, st) => {
+        if (err && err.code !== 'ENOENT') return reject(err)
+        if (err || st.size < 32 || this._overwrite) {
+          const key = crypto.randomBytes(32)
+          return this._keyStorage.write(0, key, err => {
+            if (err) return reject(err)
+            return resolve(key)
+          })
+        }
+        this._keyStorage.read(0, 32, (err, key) => {
+          if (err) return reject(err)
+          return resolve(key)
+        })
+      })
+    })
+    return this.primaryKey
   }
 
   async _generateKeys (opts) {
-    if (opts.discoveryKey) {
+    if (opts._discoveryKey) {
       return {
         keyPair: null,
-        sign: null,
-        discoveryKey: opts.discoveryKey
+        auth: null,
+        discoveryKey: opts._discoveryKey
       }
     }
     if (!opts.name) {
@@ -51,30 +109,39 @@ module.exports = class Corestore extends EventEmitter {
           secretKey: opts.secretKey
         },
         sign: opts.sign,
+        auth: opts.auth,
         discoveryKey: crypto.discoveryKey(opts.publicKey)
       }
     }
-    const { publicKey, sign } = await this.keys.createHypercoreKeyPair(opts.name, this._namespace)
+    const { publicKey, auth } = await this.createKeyPair(opts.name)
     return {
       keyPair: {
         publicKey,
         secretKey: null
       },
-      sign,
+      auth,
       discoveryKey: crypto.discoveryKey(publicKey)
     }
   }
 
-  async _postload (core) {
-    const name = await core.getUserData(USERDATA_NAME_KEY)
+  _getPrereadyUserData (core, key) {
+    // Need to manually read the header values before the Hypercore is ready, hence the ugliness.
+    for (const { key: savedKey, value } of core.core.header.userData) {
+      if (key === savedKey) return value
+    }
+    return null
+  }
+
+  async _preready (core) {
+    const name = this._getPrereadyUserData(core, USERDATA_NAME_KEY)
     if (!name) return
 
-    const namespace = await core.getUserData(USERDATA_NAMESPACE_KEY)
-    const { publicKey, sign } = await this.keys.createHypercoreKeyPair(name.toString(), namespace)
-    if (!publicKey.equals(core.key)) throw new Error('Stored core key does not match the provided name')
+    const namespace = this._getPrereadyUserData(core, USERDATA_NAMESPACE_KEY)
+    const { publicKey, auth } = await this.createKeyPair(b4a.toString(name), namespace)
+    if (!b4a.equals(publicKey, core.key)) throw new Error('Stored core key does not match the provided name')
 
-    // TODO: Should Hypercore expose a helper for this, or should postload return keypair/sign?
-    core.sign = sign
+    // TODO: Should Hypercore expose a helper for this, or should preready return keypair/auth?
+    core.auth = auth
     core.key = publicKey
     core.writable = true
   }
@@ -82,20 +149,22 @@ module.exports = class Corestore extends EventEmitter {
   async _preload (opts) {
     await this.ready()
 
-    const { discoveryKey, keyPair, sign } = await this._generateKeys(opts)
-    const id = discoveryKey.toString('hex')
+    const { discoveryKey, keyPair, auth } = await this._generateKeys(opts)
+    const id = b4a.toString(discoveryKey, 'hex')
 
     while (this.cores.has(id)) {
       const existing = this.cores.get(id)
-      if (existing) {
-        if (!existing.closing) return { from: existing, keyPair, sign }
+      if (existing.opened && !existing.closing) return { from: existing, keyPair, auth }
+      if (existing.closing) {
         await existing.close()
+      } else {
+        await existing.ready().catch(safetyCatch)
       }
     }
 
     const userData = {}
     if (opts.name) {
-      userData[USERDATA_NAME_KEY] = Buffer.from(opts.name)
+      userData[USERDATA_NAME_KEY] = b4a.from(opts.name)
       userData[USERDATA_NAMESPACE_KEY] = this._namespace
     }
 
@@ -103,98 +172,169 @@ module.exports = class Corestore extends EventEmitter {
 
     const storageRoot = [CORES_DIR, id.slice(0, 2), id.slice(2, 4), id].join('/')
     const core = new Hypercore(p => this.storage(storageRoot + '/' + p), {
+      _preready: this._preready.bind(this),
       autoClose: true,
       encryptionKey: opts.encryptionKey || null,
-      keyPair: {
-        publicKey: keyPair.publicKey,
-        secretKey: null
-      },
       userData,
-      sign: null,
-      postload: this._postload.bind(this),
-      createIfMissing: !!opts.keyPair
+      auth,
+      cache: opts.cache,
+      createIfMissing: !opts._discoveryKey,
+      keyPair: keyPair && keyPair.publicKey
+        ? {
+            publicKey: keyPair.publicKey,
+            secretKey: null
+          }
+        : null
     })
 
     this.cores.set(id, core)
-    for (const stream of this._replicationStreams) {
-      core.replicate(stream)
-    }
+    core.ready().then(() => {
+      for (const { stream } of this._replicationStreams) {
+        const sessions = this._streamSessions.get(stream)
+        const session = core.session()
+        sessions.push(session)
+        core.replicate(stream)
+      }
+    }, () => {
+      this.cores.delete(id)
+    })
     core.once('close', () => {
       this.cores.delete(id)
     })
 
-    return { from: core, keyPair, sign }
+    return { from: core, keyPair, auth }
+  }
+
+  async createKeyPair (name, namespace = this._namespace) {
+    if (!this.primaryKey) await this._opening
+
+    const keyPair = {
+      publicKey: b4a.allocUnsafe(sodium.crypto_sign_PUBLICKEYBYTES),
+      secretKey: b4a.alloc(sodium.crypto_sign_SECRETKEYBYTES),
+      auth: {
+        sign: (msg) => sign(keyPair, msg),
+        verify: (signable, signature) => {
+          return crypto.verify(signable, signature, keyPair.publicKey)
+        }
+      }
+    }
+
+    const seed = deriveSeed(this.primaryKey, namespace, name)
+    sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair.secretKey, seed)
+
+    return keyPair
   }
 
   get (opts = {}) {
     opts = validateGetOptions(opts)
+
+    if (opts.cache !== false) {
+      opts.cache = opts.cache === true || (this.cache && !opts.cache) ? defaultCache() : opts.cache
+    }
+
     const core = new Hypercore(null, {
       ...opts,
       name: null,
       preload: () => this._preload(opts)
     })
+
+    this._sessions.add(core)
+    if (this._findingPeersCount > 0) {
+      this._findingPeers.push(core.findingPeers())
+    }
+
+    core.once('close', () => {
+      // technically better to also clear _findingPeers if we added it,
+      // but the lifecycle for those are pretty short so prob not worth the complexity
+      // as _decFindingPeers clear them all.
+      this._sessions.delete(core)
+    })
+
     return core
   }
 
-  replicate (opts = {}) {
-    const stream = isStream(opts) ? opts : (opts.stream || Hypercore.createProtocolStream(opts))
+  replicate (isInitiator, opts) {
+    const isExternal = isStream(isInitiator) || !!(opts && opts.stream)
+    const stream = Hypercore.createProtocolStream(isInitiator, {
+      ...opts,
+      ondiscoverykey: discoveryKey => {
+        const core = this.get({ _discoveryKey: discoveryKey })
+        return core.ready().catch(safetyCatch)
+      }
+    })
+
+    const sessions = []
     for (const core of this.cores.values()) {
+      if (!core.opened) continue // If the core is not opened, it will be replicated in preload.
+      const session = core.session()
+      sessions.push(session)
       core.replicate(stream)
     }
-    stream.on('discovery-key', discoveryKey => {
-      const core = this.get({ discoveryKey })
-      core.ready().then(() => {
-        core.replicate(stream)
-      }, () => {
-        stream.close(discoveryKey)
-      })
-    })
-    this._replicationStreams.push(stream)
+
+    const streamRecord = { stream, isExternal }
+    this._replicationStreams.push(streamRecord)
+    this._streamSessions.set(stream, sessions)
+
     stream.once('close', () => {
-      this._replicationStreams.splice(this._replicationStreams.indexOf(stream), 1)
+      this._replicationStreams.splice(this._replicationStreams.indexOf(streamRecord), 1)
+      this._streamSessions.delete(stream)
+      Promise.all(sessions.map(s => s.close())).catch(safetyCatch)
     })
     return stream
   }
 
   namespace (name) {
-    if (!Buffer.isBuffer(name)) name = Buffer.from(name)
     return new Corestore(this.storage, {
-      _namespace: generateNamespace(this._namespace, name),
+      primaryKey: this._opening.then(() => this.primaryKey),
+      namespace: generateNamespace(this._namespace, name),
+      cache: this.cache,
       _opening: this._opening,
       _cores: this.cores,
       _streams: this._replicationStreams,
-      keys: this._opening.then(() => this.keys)
+      _streamSessions: this._streamSessions
     })
   }
 
   async _close () {
-    if (this._closing) return this._closing
     await this._opening
+    if (!b4a.equals(this._namespace, DEFAULT_NAMESPACE)) {
+      // namespaces should not release resources on close
+      // TODO: Refactor the namespace close logic to actually close sessions with ref counting
+      return
+    }
     const closePromises = []
     for (const core of this.cores.values()) {
       closePromises.push(core.close())
     }
     await Promise.allSettled(closePromises)
-    for (const stream of this._replicationStreams) {
-      stream.destroy()
+    for (const { stream, isExternal } of this._replicationStreams) {
+      // Only close streams that were created by the Corestore
+      if (!isExternal) stream.destroy()
     }
-    await this.keys.close()
+    if (!this._keyStorage) return
+    await new Promise((resolve, reject) => {
+      this._keyStorage.close(err => {
+        if (err) return reject(err)
+        return resolve(null)
+      })
+    })
   }
 
   close () {
     if (this._closing) return this._closing
     this._closing = this._close()
-    this._closing.catch(noop)
+    this._closing.catch(safetyCatch)
     return this._closing
   }
+}
 
-  static createToken () {
-    return KeyManager.createToken()
-  }
+function sign (keyPair, message) {
+  if (!keyPair.secretKey) throw new Error('Invalid key pair')
+  return crypto.sign(message, keyPair.secretKey)
 }
 
 function validateGetOptions (opts) {
-  if (Buffer.isBuffer(opts)) return { key: opts, publicKey: opts }
+  if (b4a.isBuffer(opts)) return { key: opts, publicKey: opts }
   if (opts.key) {
     opts.publicKey = opts.key
   }
@@ -204,23 +344,30 @@ function validateGetOptions (opts) {
   }
   if (opts.name && typeof opts.name !== 'string') throw new Error('name option must be a String')
   if (opts.name && opts.secretKey) throw new Error('Cannot provide both a name and a secret key')
-  if (opts.publicKey && !Buffer.isBuffer(opts.publicKey)) throw new Error('publicKey option must be a Buffer')
-  if (opts.secretKey && !Buffer.isBuffer(opts.secretKey)) throw new Error('secretKey option must be a Buffer')
-  if (opts.discoveryKey && !Buffer.isBuffer(opts.discoveryKey)) throw new Error('discoveryKey option must be a Buffer')
-  if (!opts.name && !opts.publicKey) throw new Error('Must provide either a name or a publicKey')
+  if (opts.publicKey && !b4a.isBuffer(opts.publicKey)) throw new Error('publicKey option must be a Buffer or Uint8Array')
+  if (opts.secretKey && !b4a.isBuffer(opts.secretKey)) throw new Error('secretKey option must be a Buffer or Uint8Array')
+  if (!opts._discoveryKey && (!opts.name && !opts.publicKey)) throw new Error('Must provide either a name or a publicKey')
   return opts
 }
 
-function generateNamespace (first, second) {
-  if (!Buffer.isBuffer(first)) first = Buffer.from(first)
-  if (second && !Buffer.isBuffer(second)) second = Buffer.from(second)
-  const out = Buffer.allocUnsafe(32)
-  sodium.crypto_generichash(out, second ? Buffer.concat([first, second]) : first)
+function generateNamespace (namespace, name) {
+  if (!b4a.isBuffer(name)) name = b4a.from(name)
+  const out = b4a.allocUnsafe(32)
+  sodium.crypto_generichash_batch(out, [namespace, name])
+  return out
+}
+
+function deriveSeed (primaryKey, namespace, name) {
+  if (!b4a.isBuffer(name)) name = b4a.from(name)
+  const out = b4a.alloc(32)
+  sodium.crypto_generichash_batch(out, [NS, namespace, name], primaryKey)
   return out
 }
 
+function defaultCache () {
+  return new Xache({ maxSize: 65536, maxAge: 0 })
+}
+
 function isStream (s) {
   return typeof s === 'object' && s && typeof s.pipe === 'function'
 }
-
-function noop () {}

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "corestore",
-  "version": "6.0.0-alpha.4",
+  "version": "6.0.0",
   "description": "A Hypercore factory that simplifies managing collections of cores.",
   "main": "index.js",
   "scripts": {
-    "test": "standard && tape test/*.js"
+    "test": "standard && brittle test/*.js"
   },
   "repository": {
     "type": "git",
@@ -20,17 +20,16 @@
   },
   "homepage": "https://github.com/hypercore-protocol/corestore#readme",
   "devDependencies": {
-    "random-access-file": "^2.2.0",
-    "random-access-memory": "^3.1.2",
-    "standardx": "^7.0.0",
-    "tape": "^5.3.1",
-    "tmp-promise": "^3.0.2"
+    "brittle": "^3.0.0",
+    "random-access-memory": "^4.0.0",
+    "standardx": "^7.0.0"
   },
   "dependencies": {
-    "blake2b-universal": "^1.0.1",
-    "derive-key": "^1.0.1",
-    "hypercore": "next",
-    "hypercore-crypto": "^2.3.0",
-    "sodium-universal": "^3.0.4"
+    "b4a": "^1.3.1",
+    "hypercore": "v10.0.0",
+    "hypercore-crypto": "^3.2.1",
+    "safety-catch": "^1.0.1",
+    "sodium-universal": "^3.0.4",
+    "xache": "^1.1.0"
   }
 }

package/test/all.js

@@ -1,7 +1,10 @@
-const test = require('tape')
+const test = require('brittle')
 const crypto = require('hypercore-crypto')
 const ram = require('random-access-memory')
-const tmp = require('tmp-promise')
+const os = require('os')
+const path = require('path')
+const b4a = require('b4a')
+const sodium = require('sodium-universal')
 
 const Corestore = require('..')
 
@@ -13,15 +16,13 @@ test('basic get with caching', async function (t) {
 
   await Promise.all([core1a.ready(), core1b.ready(), core2.ready()])
 
-  t.same(core1a.key, core1b.key)
-  t.notSame(core1a.key, core2.key)
+  t.alike(core1a.key, core1b.key)
+  t.unlike(core1a.key, core2.key)
 
-  t.true(core1a.writable)
-  t.true(core1b.writable)
+  t.ok(core1a.writable)
+  t.ok(core1b.writable)
 
-  t.same(store.cores.size, 2)
-
-  t.end()
+  t.is(store.cores.size, 2)
 })
 
 test('basic get with custom keypair', async function (t) {
@@ -33,12 +34,10 @@ test('basic get with custom keypair', async function (t) {
   const core2 = store.get(kp2)
   await Promise.all([core1.ready(), core2.ready()])
 
-  t.same(core1.key, kp1.publicKey)
-  t.same(core2.key, kp2.publicKey)
-  t.true(core1.writable)
-  t.true(core2.writable)
-
-  t.end()
+  t.alike(core1.key, kp1.publicKey)
+  t.alike(core2.key, kp2.publicKey)
+  t.ok(core1.writable)
+  t.ok(core2.writable)
 })
 
 test('basic namespaces', async function (t) {
@@ -52,12 +51,12 @@ test('basic namespaces', async function (t) {
   const core3 = ns3.get({ name: 'main' })
   await Promise.all([core1.ready(), core2.ready(), core3.ready()])
 
-  t.false(core1.key.equals(core2.key))
-  t.true(core1.key.equals(core3.key))
-  t.true(core1.writable)
-  t.true(core2.writable)
-  t.true(core3.writable)
-  t.same(store.cores.size, 2)
+  t.absent(core1.key.equals(core2.key))
+  t.ok(core1.key.equals(core3.key))
+  t.ok(core1.writable)
+  t.ok(core2.writable)
+  t.ok(core3.writable)
+  t.is(store.cores.size, 2)
 
   t.end()
 })
@@ -77,10 +76,46 @@ test('basic replication', async function (t) {
   const s = store1.replicate(true)
   s.pipe(store2.replicate(false)).pipe(s)
 
-  t.same(await core3.get(0), Buffer.from('hello'))
-  t.same(await core4.get(0), Buffer.from('world'))
+  t.alike(await core3.get(0), Buffer.from('hello'))
+  t.alike(await core4.get(0), Buffer.from('world'))
+})
 
-  t.end()
+test('replicating cores created after replication begins', async function (t) {
+  const store1 = new Corestore(ram)
+  const store2 = new Corestore(ram)
+
+  const s = store1.replicate(true, { live: true })
+  s.pipe(store2.replicate(false, { live: true })).pipe(s)
+
+  const core1 = store1.get({ name: 'core-1' })
+  const core2 = store1.get({ name: 'core-2' })
+  await core1.append('hello')
+  await core2.append('world')
+
+  const core3 = store2.get({ key: core1.key })
+  const core4 = store2.get({ key: core2.key })
+
+  t.alike(await core3.get(0), Buffer.from('hello'))
+  t.alike(await core4.get(0), Buffer.from('world'))
+})
+
+test('replicating cores using discovery key hook', async function (t) {
+  const dir = tmpdir()
+  let store1 = new Corestore(dir)
+  const store2 = new Corestore(ram)
+
+  const core = store1.get({ name: 'main' })
+  await core.append('hello')
+  const key = core.key
+
+  await store1.close()
+  store1 = new Corestore(dir)
+
+  const s = store1.replicate(true, { live: true })
+  s.pipe(store2.replicate(false, { live: true })).pipe(s)
+
+  const core2 = store2.get(key)
+  t.alike(await core2.get(0), Buffer.from('hello'))
 })
 
 test('nested namespaces', async function (t) {
@@ -92,65 +127,208 @@ test('nested namespaces', async function (t) {
   const core2 = ns1b.get({ name: 'main' })
   await Promise.all([core1.ready(), core2.ready()])
 
-  t.false(core1.key.equals(core2.key))
-  t.true(core1.writable)
-  t.true(core2.writable)
-  t.same(store.cores.size, 2)
-
-  t.end()
+  t.not(core1.key.equals(core2.key))
+  t.ok(core1.writable)
+  t.ok(core2.writable)
+  t.is(store.cores.size, 2)
 })
 
 test('core uncached when all sessions close', async function (t) {
   const store = new Corestore(ram)
   const core1 = store.get({ name: 'main' })
   await core1.ready()
-  t.same(store.cores.size, 1)
+  t.is(store.cores.size, 1)
   await core1.close()
-  t.same(store.cores.size, 0)
-  t.end()
+  t.is(store.cores.size, 0)
 })
 
 test('writable core loaded from name userData', async function (t) {
-  const dir = await tmp.dir({ unsafeCleanup: true })
+  const dir = tmpdir()
 
-  let store = new Corestore(dir.path)
+  let store = new Corestore(dir)
   let core = store.get({ name: 'main' })
   await core.ready()
   const key = core.key
 
-  t.true(core.writable)
+  t.ok(core.writable)
   await core.append('hello')
-  t.same(core.length, 1)
+  t.is(core.length, 1)
 
   await store.close()
-  store = new Corestore(dir.path)
+  store = new Corestore(dir)
   core = store.get(key)
   await core.ready()
 
-  t.true(core.writable)
+  t.ok(core.writable)
   await core.append('world')
-  t.same(core.length, 2)
-  t.same(await core.get(0), Buffer.from('hello'))
-  t.same(await core.get(1), Buffer.from('world'))
+  t.is(core.length, 2)
+  t.alike(await core.get(0), Buffer.from('hello'))
+  t.alike(await core.get(1), Buffer.from('world'))
+})
 
-  await dir.cleanup()
-  t.end()
+test('writable core loaded from name and namespace userData', async function (t) {
+  const dir = tmpdir()
+
+  let store = new Corestore(dir)
+  let core = store.namespace('ns1').get({ name: 'main' })
+  await core.ready()
+  const key = core.key
+
+  t.ok(core.writable)
+  await core.append('hello')
+  t.is(core.length, 1)
+
+  await store.close()
+  store = new Corestore(dir)
+  core = store.get(key)
+  await core.ready()
+
+  t.ok(core.writable)
+  await core.append('world')
+  t.is(core.length, 2)
+  t.alike(await core.get(0), Buffer.from('hello'))
+  t.alike(await core.get(1), Buffer.from('world'))
 })
 
 test('storage locking', async function (t) {
-  const dir = await tmp.dir({ unsafeCleanup: true })
+  const dir = tmpdir()
 
-  const store1 = new Corestore(dir.path)
+  const store1 = new Corestore(dir)
   await store1.ready()
 
-  const store2 = new Corestore(dir.path)
+  const store2 = new Corestore(dir)
   try {
     await store2.ready()
     t.fail('dir should have been locked')
   } catch {
     t.pass('dir was locked')
   }
+})
 
-  await dir.cleanup()
-  t.end()
+test('closing a namespace does not close cores', async function (t) {
+  const store = new Corestore(ram)
+  const ns1 = store.namespace('ns1')
+  const core1 = ns1.get({ name: 'core-1' })
+  const core2 = ns1.get({ name: 'core-2' })
+  await Promise.all([core1.ready(), core2.ready()])
+
+  await ns1.close()
+
+  t.is(store.cores.size, 2)
+  t.not(core1.closed)
+  t.not(core1.closed)
+
+  await store.close()
+
+  t.is(store.cores.size, 0)
+  t.ok(core1.closed)
+  t.ok(core2.closed)
+})
+
+test('findingPeers', async function (t) {
+  t.plan(6)
+
+  const store = new Corestore(ram)
+
+  const ns1 = store.namespace('ns1')
+  const ns2 = store.namespace('ns2')
+
+  const a = ns1.get(Buffer.alloc(32).fill('a'))
+  const b = ns2.get(Buffer.alloc(32).fill('b'))
+
+  const done = ns1.findingPeers()
+
+  let aUpdated = false
+  let bUpdated = false
+  let cUpdated = false
+
+  const c = ns1.get(Buffer.alloc(32).fill('c'))
+
+  a.update().then(function (bool) {
+    aUpdated = true
+  })
+
+  b.update().then(function (bool) {
+    bUpdated = true
+  })
+
+  c.update().then(function (bool) {
+    cUpdated = true
+  })
+
+  await new Promise(resolve => setImmediate(resolve))
+
+  t.is(aUpdated, false)
+  t.is(bUpdated, true)
+  t.is(cUpdated, false)
+
+  done()
+
+  await new Promise(resolve => setImmediate(resolve))
+
+  t.is(aUpdated, true)
+  t.is(bUpdated, true)
+  t.is(cUpdated, true)
 })
+
+test('different primary keys yield different keypairs', async function (t) {
+  const pk1 = randomBytes(32)
+  const pk2 = randomBytes(32)
+  t.unlike(pk1, pk2)
+
+  const store1 = new Corestore(ram, { primaryKey: pk1 })
+  const store2 = new Corestore(ram, { primaryKey: pk2 })
+
+  const kp1 = await store1.createKeyPair('hello')
+  const kp2 = await store2.createKeyPair('hello')
+
+  t.unlike(kp1.publicKey, kp2.publicKey)
+})
+
+test('keypair auth sign', async function (t) {
+  const store = new Corestore(ram)
+  const keyPair = await store.createKeyPair('foo')
+  const message = b4a.from('hello world')
+
+  const sig = keyPair.auth.sign(message)
+
+  t.is(sig.length, 64)
+  t.ok(crypto.verify(message, sig, keyPair.publicKey))
+  t.absent(crypto.verify(message, b4a.alloc(64), keyPair.publicKey))
+})
+
+test('keypair auth verify', async function (t) {
+  const store = new Corestore(ram)
+  const keyPair = await store.createKeyPair('foo')
+  const message = b4a.from('hello world')
+
+  const sig = crypto.sign(message, keyPair.secretKey)
+
+  t.is(sig.length, 64)
+  t.ok(keyPair.auth.verify(message, sig))
+  t.absent(keyPair.auth.verify(message, b4a.alloc(64)))
+})
+
+test('core caching after reopen regression', async function (t) {
+  const store = new Corestore(ram)
+  const core = store.get({ name: 'test-core' })
+  await core.ready()
+
+  core.close()
+  await core.opening
+
+  const core2 = store.get({ name: 'test-core' })
+  await core2.ready()
+
+  t.pass('did not infinite loop')
+})
+
+function tmpdir () {
+  return path.join(os.tmpdir(), 'corestore-' + Math.random().toString(16).slice(2))
+}
+
+function randomBytes (n) {
+  const buf = b4a.allocUnsafe(n)
+  sodium.randombytes_buf(buf)
+  return buf
+}

package/test/cache.js

@@ -0,0 +1,46 @@
+const test = require('brittle')
+const RAM = require('random-access-memory')
+
+const Corestore = require('..')
+
+test('core cache', async function (t) {
+  const store = new Corestore(RAM, { cache: true })
+
+  const core = store.get({ name: 'core' })
+  await core.append(['a', 'b', 'c'])
+
+  const p = core.get(0)
+  const q = core.get(0)
+
+  t.is(await p, await q)
+})
+
+test('clear cache on truncate', async function (t) {
+  const store = new Corestore(RAM, { cache: true })
+
+  const core = store.get({ name: 'core' })
+  await core.append(['a', 'b', 'c'])
+
+  const p = core.get(0)
+
+  await core.truncate(0)
+  await core.append('d')
+
+  const q = core.get(0)
+
+  t.alike(await p, Buffer.from('a'))
+  t.alike(await q, Buffer.from('d'))
+})
+
+test('core cache on namespace', async function (t) {
+  const store = new Corestore(RAM, { cache: true })
+  const ns1 = store.namespace('test-namespace-1')
+
+  const c1 = store.get({ name: 'test-core' })
+  const c2 = ns1.get({ name: 'test-core' })
+
+  await Promise.all([c1.ready(), c2.ready()])
+
+  t.ok(c1.cache)
+  t.ok(c2.cache)
+})

package/lib/keys.js

@@ -1,104 +0,0 @@
-// TODO: Extract this into a standalone module
-
-const sodium = require('sodium-universal')
-const blake2b = require('blake2b-universal')
-
-const DEFAULT_TOKEN = Buffer.alloc(0)
-const NAMESPACE = Buffer.from('@hyperspace/key-manager')
-
-module.exports = class KeyManager {
-  constructor (storage, profile, opts = {}) {
-    this.storage = storage
-    this.profile = profile
-  }
-
-  _sign (keyPair, message) {
-    if (!keyPair._secretKey) throw new Error('Invalid key pair')
-    const signature = Buffer.allocUnsafe(sodium.crypto_sign_BYTES)
-    sodium.crypto_sign_detached(signature, message, keyPair._secretKey)
-    return signature
-  }
-
-  createSecret (name, token) {
-    return deriveSeed(this.profile, token, name)
-  }
-
-  createHypercoreKeyPair (name, token) {
-    const keyPair = {
-      publicKey: Buffer.allocUnsafe(sodium.crypto_sign_PUBLICKEYBYTES),
-      _secretKey: Buffer.alloc(sodium.crypto_sign_SECRETKEYBYTES),
-      sign: (msg) => this._sign(keyPair, msg)
-    }
-
-    sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair._secretKey, this.createSecret(name, token))
-
-    return keyPair
-  }
-
-  createNetworkIdentity (name, token) {
-    const keyPair = {
-      publicKey: Buffer.alloc(32),
-      secretKey: Buffer.alloc(64)
-    }
-
-    sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair.secretKey, this.createSecret(name, token))
-
-    return keyPair
-  }
-
-  close () {
-    return new Promise((resolve, reject) => {
-      this.storage.close(err => {
-        if (err) return reject(err)
-        return resolve()
-      })
-    })
-  }
-
-  static createToken () {
-    return randomBytes(32)
-  }
-
-  static async fromStorage (storage, opts = {}) {
-    const profileStorage = storage(opts.name || 'default')
-
-    const profile = await new Promise((resolve, reject) => {
-      profileStorage.stat((err, st) => {
-        if (err && err.code !== 'ENOENT') return reject(err)
-        if (err || st.size < 32 || opts.overwrite) {
-          const key = randomBytes(32)
-          return profileStorage.write(0, key, err => {
-            if (err) return reject(err)
-            return resolve(key)
-          })
-        }
-        profileStorage.read(0, 32, (err, key) => {
-          if (err) return reject(err)
-          return resolve(key)
-        })
-      })
-    })
-
-    return new this(profileStorage, profile, opts)
-  }
-}
-
-function deriveSeed (profile, token, name, output) {
-  if (token && token.length < 32) throw new Error('Token must be a Buffer with length >= 32')
-  if (!name || typeof name !== 'string') throw new Error('name must be a String')
-  if (!output) output = Buffer.alloc(32)
-
-  blake2b.batch(output, [
-    NAMESPACE,
-    token || DEFAULT_TOKEN,
-    Buffer.from(Buffer.byteLength(name, 'ascii') + '\n' + name, 'ascii')
-  ], profile)
-
-  return output
-}
-
-function randomBytes (n) {
-  const buf = Buffer.allocUnsafe(n)
-  sodium.randombytes_buf(buf)
-  return buf
-}

package/test/keys.js

@@ -1,74 +0,0 @@
-const p = require('path')
-const fs = require('fs')
-
-const test = require('tape')
-const ram = require('random-access-memory')
-const raf = require('random-access-file')
-
-const KeyManager = require('../lib/keys')
-
-test('can create hypercore keypairs', async t => {
-  const keys = await KeyManager.fromStorage(ram)
-
-  const kp1 = await keys.createHypercoreKeyPair('core1')
-  const kp2 = await keys.createHypercoreKeyPair('core2')
-
-  t.same(kp1.publicKey.length, 32)
-  t.same(kp2.publicKey.length, 32)
-  t.notSame(kp1.publicKey, kp2.publicKey)
-
-  t.end()
-})
-
-test('distinct tokens create distinct hypercore keypairs', async t => {
-  const keys = await KeyManager.fromStorage(ram)
-  const token1 = KeyManager.createToken()
-  const token2 = KeyManager.createToken()
-
-  const kp1 = await keys.createHypercoreKeyPair('core1', token1)
-  const kp2 = await keys.createHypercoreKeyPair('core1', token2)
-
-  t.notSame(kp1.publicKey, kp2.publicKey)
-
-  t.end()
-})
-
-test('short user-provided token will throw', async t => {
-  const keys = await KeyManager.fromStorage(ram)
-
-  try {
-    await keys.createHypercoreKeyPair('core1', Buffer.from('hello'))
-    t.fail('did not throw')
-  } catch {
-    t.pass('threw correctly')
-  }
-
-  t.end()
-})
-
-test('persistent storage regenerates keys correctly', async t => {
-  const testPath = p.resolve(__dirname, 'test-data')
-
-  const keys1 = await KeyManager.fromStorage((name) => raf(testPath, { directory: testPath }))
-  const kp1 = await keys1.createHypercoreKeyPair('core1')
-
-  const keys2 = await KeyManager.fromStorage((name) => raf(testPath, { directory: testPath }))
-  const kp2 = await keys2.createHypercoreKeyPair('core1')
-
-  t.same(kp1.publicKey, kp2.publicKey)
-
-  await fs.promises.rm(testPath, { recursive: true })
-  t.end()
-})
-
-test('different master keys -> different keys', async t => {
-  const keys1 = await KeyManager.fromStorage(ram)
-  const keys2 = await KeyManager.fromStorage(ram)
-
-  const kp1 = await keys1.createHypercoreKeyPair('core1')
-  const kp2 = await keys2.createHypercoreKeyPair('core1')
-
-  t.notSame(kp1.publicKey, kp2.publicKey)
-
-  t.end()
-})