corestore 6.0.1-alpha.5 → 6.0.1-alpha.9

package/index.js

@@ -1,4 +1,5 @@
 const { EventEmitter } = require('events')
+const safetyCatch = require('safety-catch')
 const crypto = require('hypercore-crypto')
 const sodium = require('sodium-universal')
 const Hypercore = require('hypercore')
@@ -37,11 +38,11 @@ module.exports = class Corestore extends EventEmitter {
   }
 
   async _generateKeys (opts) {
-    if (opts.discoveryKey) {
+    if (opts._discoveryKey) {
       return {
         keyPair: null,
         sign: null,
-        discoveryKey: opts.discoveryKey
+        discoveryKey: opts._discoveryKey
       }
     }
     if (!opts.name) {
@@ -65,15 +66,22 @@ module.exports = class Corestore extends EventEmitter {
     }
   }
 
-  async _postload (core) {
-    const name = await core.getUserData(USERDATA_NAME_KEY)
+  _getPrereadyUserData (core, key) {
+    for (const { key: savedKey, value } of core.core.header.userData) {
+      if (key === savedKey) return value
+    }
+    return null
+  }
+
+  async _preready (core) {
+    const name = this._getPrereadyUserData(core, USERDATA_NAME_KEY)
     if (!name) return
 
-    const namespace = await core.getUserData(USERDATA_NAMESPACE_KEY)
+    const namespace = this._getPrereadyUserData(core, USERDATA_NAMESPACE_KEY)
     const { publicKey, sign } = await this.keys.createHypercoreKeyPair(name.toString(), namespace)
     if (!publicKey.equals(core.key)) throw new Error('Stored core key does not match the provided name')
 
-    // TODO: Should Hypercore expose a helper for this, or should postload return keypair/sign?
+    // TODO: Should Hypercore expose a helper for this, or should preready return keypair/sign?
     core.sign = sign
     core.key = publicKey
     core.writable = true
@@ -87,8 +95,10 @@ module.exports = class Corestore extends EventEmitter {
 
     while (this.cores.has(id)) {
       const existing = this.cores.get(id)
-      if (existing) {
-        if (!existing.closing) return { from: existing, keyPair, sign }
+      if (existing.opened && !existing.closing) return { from: existing, keyPair, sign }
+      if (!existing.opened) {
+        await existing.ready().catch(safetyCatch)
+      } else if (existing.closing) {
         await existing.close()
       }
     }
@@ -103,22 +113,28 @@ module.exports = class Corestore extends EventEmitter {
 
     const storageRoot = [CORES_DIR, id.slice(0, 2), id.slice(2, 4), id].join('/')
     const core = new Hypercore(p => this.storage(storageRoot + '/' + p), {
+      _preready: this._preready.bind(this),
       autoClose: true,
       encryptionKey: opts.encryptionKey || null,
-      keyPair: {
-        publicKey: keyPair.publicKey,
-        secretKey: null
-      },
       userData,
       sign: null,
-      postload: this._postload.bind(this),
-      createIfMissing: !!opts.keyPair
+      createIfMissing: !opts._discoveryKey,
+      keyPair: keyPair && keyPair.publicKey
+        ? {
+            publicKey: keyPair.publicKey,
+            secretKey: null
+          }
+        : null
     })
 
     this.cores.set(id, core)
-    for (const stream of this._replicationStreams) {
-      core.replicate(stream)
-    }
+    core.ready().then(() => {
+      for (const { stream } of this._replicationStreams) {
+        core.replicate(stream)
+      }
+    }, () => {
+      this.cores.delete(id)
+    })
     core.once('close', () => {
       this.cores.delete(id)
     })
@@ -136,22 +152,22 @@ module.exports = class Corestore extends EventEmitter {
     return core
   }
 
-  replicate (opts = {}) {
-    const stream = isStream(opts) ? opts : (opts.stream || Hypercore.createProtocolStream(opts))
+  replicate (isInitiator, opts) {
+    const isExternal = isStream(isInitiator) || !!(opts && opts.stream)
+    const stream = Hypercore.createProtocolStream(isInitiator, {
+      ...opts,
+      ondiscoverykey: discoveryKey => {
+        const core = this.get({ _discoveryKey: discoveryKey })
+        return core.ready().catch(safetyCatch)
+      }
+    })
     for (const core of this.cores.values()) {
-      core.replicate(stream)
+      if (core.opened) core.replicate(stream) // If the core is not opened, it will be replicated in preload.
     }
-    stream.on('discovery-key', discoveryKey => {
-      const core = this.get({ discoveryKey })
-      core.ready().then(() => {
-        core.replicate(stream)
-      }, () => {
-        stream.close(discoveryKey)
-      })
-    })
-    this._replicationStreams.push(stream)
+    const streamRecord = { stream, isExternal }
+    this._replicationStreams.push(streamRecord)
     stream.once('close', () => {
-      this._replicationStreams.splice(this._replicationStreams.indexOf(stream), 1)
+      this._replicationStreams.splice(this._replicationStreams.indexOf(streamRecord), 1)
     })
     return stream
   }
@@ -175,8 +191,9 @@ module.exports = class Corestore extends EventEmitter {
       closePromises.push(core.close())
     }
     await Promise.allSettled(closePromises)
-    for (const stream of this._replicationStreams) {
-      stream.destroy()
+    for (const { stream, isExternal } of this._replicationStreams) {
+      // Only close streams that were created by the Corestore
+      if (!isExternal) stream.destroy()
     }
     await this.keys.close()
   }
@@ -206,8 +223,7 @@ function validateGetOptions (opts) {
   if (opts.name && opts.secretKey) throw new Error('Cannot provide both a name and a secret key')
   if (opts.publicKey && !Buffer.isBuffer(opts.publicKey)) throw new Error('publicKey option must be a Buffer')
   if (opts.secretKey && !Buffer.isBuffer(opts.secretKey)) throw new Error('secretKey option must be a Buffer')
-  if (opts.discoveryKey && !Buffer.isBuffer(opts.discoveryKey)) throw new Error('discoveryKey option must be a Buffer')
-  if (!opts.name && !opts.publicKey) throw new Error('Must provide either a name or a publicKey')
+  if (!opts._discoveryKey && (!opts.name && !opts.publicKey)) throw new Error('Must provide either a name or a publicKey')
   return opts
 }
 

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "corestore",
-  "version": "6.0.1-alpha.5",
+  "version": "6.0.1-alpha.9",
   "description": "A Hypercore factory that simplifies managing collections of cores.",
   "main": "index.js",
   "scripts": {
-    "test": "standard && tape test/*.js"
+    "test": "standard && brittle test/*.js"
   },
   "repository": {
     "type": "git",
@@ -20,10 +20,10 @@
   },
   "homepage": "https://github.com/hypercore-protocol/corestore#readme",
   "devDependencies": {
+    "brittle": "^1.6.0",
     "random-access-file": "^2.2.0",
     "random-access-memory": "^3.1.2",
     "standardx": "^7.0.0",
-    "tape": "^5.3.1",
     "tmp-promise": "^3.0.2"
   },
   "dependencies": {
@@ -31,6 +31,7 @@
     "derive-key": "^1.0.1",
     "hypercore": "next",
     "hypercore-crypto": "^2.3.0",
+    "safety-catch": "^1.0.1",
     "sodium-universal": "^3.0.4"
   }
 }

package/test/all.js

@@ -1,4 +1,4 @@
-const test = require('tape')
+const test = require('brittle')
 const crypto = require('hypercore-crypto')
 const ram = require('random-access-memory')
 const tmp = require('tmp-promise')
@@ -13,15 +13,13 @@ test('basic get with caching', async function (t) {
 
   await Promise.all([core1a.ready(), core1b.ready(), core2.ready()])
 
-  t.same(core1a.key, core1b.key)
-  t.notSame(core1a.key, core2.key)
+  t.alike(core1a.key, core1b.key)
+  t.unlike(core1a.key, core2.key)
 
-  t.true(core1a.writable)
-  t.true(core1b.writable)
+  t.ok(core1a.writable)
+  t.ok(core1b.writable)
 
-  t.same(store.cores.size, 2)
-
-  t.end()
+  t.is(store.cores.size, 2)
 })
 
 test('basic get with custom keypair', async function (t) {
@@ -33,12 +31,10 @@ test('basic get with custom keypair', async function (t) {
   const core2 = store.get(kp2)
   await Promise.all([core1.ready(), core2.ready()])
 
-  t.same(core1.key, kp1.publicKey)
-  t.same(core2.key, kp2.publicKey)
-  t.true(core1.writable)
-  t.true(core2.writable)
-
-  t.end()
+  t.alike(core1.key, kp1.publicKey)
+  t.alike(core2.key, kp2.publicKey)
+  t.ok(core1.writable)
+  t.ok(core2.writable)
 })
 
 test('basic namespaces', async function (t) {
@@ -52,12 +48,12 @@ test('basic namespaces', async function (t) {
   const core3 = ns3.get({ name: 'main' })
   await Promise.all([core1.ready(), core2.ready(), core3.ready()])
 
-  t.false(core1.key.equals(core2.key))
-  t.true(core1.key.equals(core3.key))
-  t.true(core1.writable)
-  t.true(core2.writable)
-  t.true(core3.writable)
-  t.same(store.cores.size, 2)
+  t.absent(core1.key.equals(core2.key))
+  t.ok(core1.key.equals(core3.key))
+  t.ok(core1.writable)
+  t.ok(core2.writable)
+  t.ok(core3.writable)
+  t.is(store.cores.size, 2)
 
   t.end()
 })
@@ -77,10 +73,48 @@ test('basic replication', async function (t) {
   const s = store1.replicate(true)
   s.pipe(store2.replicate(false)).pipe(s)
 
-  t.same(await core3.get(0), Buffer.from('hello'))
-  t.same(await core4.get(0), Buffer.from('world'))
+  t.alike(await core3.get(0), Buffer.from('hello'))
+  t.alike(await core4.get(0), Buffer.from('world'))
+})
 
-  t.end()
+test('replicating cores created after replication begins', async function (t) {
+  const store1 = new Corestore(ram)
+  const store2 = new Corestore(ram)
+
+  const s = store1.replicate(true, { live: true })
+  s.pipe(store2.replicate(false, { live: true })).pipe(s)
+
+  const core1 = store1.get({ name: 'core-1' })
+  const core2 = store1.get({ name: 'core-2' })
+  await core1.append('hello')
+  await core2.append('world')
+
+  const core3 = store2.get({ key: core1.key })
+  const core4 = store2.get({ key: core2.key })
+
+  t.alike(await core3.get(0), Buffer.from('hello'))
+  t.alike(await core4.get(0), Buffer.from('world'))
+})
+
+test('replicating cores using discovery key hook', async function (t) {
+  const dir = await tmp.dir({ unsafeCleanup: true })
+  let store1 = new Corestore(dir.path)
+  const store2 = new Corestore(ram)
+
+  const core = store1.get({ name: 'main' })
+  await core.append('hello')
+  const key = core.key
+
+  await store1.close()
+  store1 = new Corestore(dir.path)
+
+  const s = store1.replicate(true, { live: true })
+  s.pipe(store2.replicate(false, { live: true })).pipe(s)
+
+  const core2 = store2.get(key)
+  t.alike(await core2.get(0), Buffer.from('hello'))
+
+  await dir.cleanup()
 })
 
 test('nested namespaces', async function (t) {
@@ -92,22 +126,19 @@ test('nested namespaces', async function (t) {
   const core2 = ns1b.get({ name: 'main' })
   await Promise.all([core1.ready(), core2.ready()])
 
-  t.false(core1.key.equals(core2.key))
-  t.true(core1.writable)
-  t.true(core2.writable)
-  t.same(store.cores.size, 2)
-
-  t.end()
+  t.not(core1.key.equals(core2.key))
+  t.ok(core1.writable)
+  t.ok(core2.writable)
+  t.is(store.cores.size, 2)
 })
 
 test('core uncached when all sessions close', async function (t) {
   const store = new Corestore(ram)
   const core1 = store.get({ name: 'main' })
   await core1.ready()
-  t.same(store.cores.size, 1)
+  t.is(store.cores.size, 1)
   await core1.close()
-  t.same(store.cores.size, 0)
-  t.end()
+  t.is(store.cores.size, 0)
 })
 
 test('writable core loaded from name userData', async function (t) {
@@ -118,23 +149,22 @@ test('writable core loaded from name userData', async function (t) {
   await core.ready()
   const key = core.key
 
-  t.true(core.writable)
+  t.ok(core.writable)
   await core.append('hello')
-  t.same(core.length, 1)
+  t.is(core.length, 1)
 
   await store.close()
   store = new Corestore(dir.path)
   core = store.get(key)
   await core.ready()
 
-  t.true(core.writable)
+  t.ok(core.writable)
   await core.append('world')
-  t.same(core.length, 2)
-  t.same(await core.get(0), Buffer.from('hello'))
-  t.same(await core.get(1), Buffer.from('world'))
+  t.is(core.length, 2)
+  t.alike(await core.get(0), Buffer.from('hello'))
+  t.alike(await core.get(1), Buffer.from('world'))
 
   await dir.cleanup()
-  t.end()
 })
 
 test('storage locking', async function (t) {
@@ -152,5 +182,4 @@ test('storage locking', async function (t) {
   }
 
   await dir.cleanup()
-  t.end()
 })

package/test/keys.js

@@ -1,7 +1,7 @@
 const p = require('path')
 const fs = require('fs')
 
-const test = require('tape')
+const test = require('brittle')
 const ram = require('random-access-memory')
 const raf = require('random-access-file')
 
@@ -13,11 +13,9 @@ test('can create hypercore keypairs', async t => {
   const kp1 = await keys.createHypercoreKeyPair('core1')
   const kp2 = await keys.createHypercoreKeyPair('core2')
 
-  t.same(kp1.publicKey.length, 32)
-  t.same(kp2.publicKey.length, 32)
-  t.notSame(kp1.publicKey, kp2.publicKey)
-
-  t.end()
+  t.is(kp1.publicKey.length, 32)
+  t.is(kp2.publicKey.length, 32)
+  t.unlike(kp1.publicKey, kp2.publicKey)
 })
 
 test('distinct tokens create distinct hypercore keypairs', async t => {
@@ -28,9 +26,7 @@ test('distinct tokens create distinct hypercore keypairs', async t => {
   const kp1 = await keys.createHypercoreKeyPair('core1', token1)
   const kp2 = await keys.createHypercoreKeyPair('core1', token2)
 
-  t.notSame(kp1.publicKey, kp2.publicKey)
-
-  t.end()
+  t.unlike(kp1.publicKey, kp2.publicKey)
 })
 
 test('short user-provided token will throw', async t => {
@@ -42,8 +38,6 @@ test('short user-provided token will throw', async t => {
   } catch {
     t.pass('threw correctly')
   }
-
-  t.end()
 })
 
 test('persistent storage regenerates keys correctly', async t => {
@@ -55,10 +49,9 @@ test('persistent storage regenerates keys correctly', async t => {
   const keys2 = await KeyManager.fromStorage((name) => raf(testPath, { directory: testPath }))
   const kp2 = await keys2.createHypercoreKeyPair('core1')
 
-  t.same(kp1.publicKey, kp2.publicKey)
+  t.alike(kp1.publicKey, kp2.publicKey)
 
   await fs.promises.rm(testPath, { recursive: true })
-  t.end()
 })
 
 test('different master keys -> different keys', async t => {
@@ -68,7 +61,5 @@ test('different master keys -> different keys', async t => {
   const kp1 = await keys1.createHypercoreKeyPair('core1')
   const kp2 = await keys2.createHypercoreKeyPair('core1')
 
-  t.notSame(kp1.publicKey, kp2.publicKey)
-
-  t.end()
+  t.unlike(kp1.publicKey, kp2.publicKey)
 })