corestore 6.0.5 → 6.1.0

package/index.js

@@ -13,25 +13,26 @@ const CORES_DIR = 'cores'
 const PRIMARY_KEY_FILE_NAME = 'primary-key'
 const USERDATA_NAME_KEY = 'corestore/name'
 const USERDATA_NAMESPACE_KEY = 'corestore/namespace'
+const POOL_SIZE = 512 // how many open fds to aim for before cycling them
 
 module.exports = class Corestore extends EventEmitter {
   constructor (storage, opts = {}) {
     super()
 
-    this.storage = Hypercore.defaultStorage(storage, { lock: PRIMARY_KEY_FILE_NAME })
-    this.cores = opts._cores || new Map()
-    this.primaryKey = null
+    const root = opts._root
+
+    this.storage = Hypercore.defaultStorage(storage, { lock: PRIMARY_KEY_FILE_NAME, poolSize: opts.poolSize || POOL_SIZE })
+    this.cores = root ? root.cores : new Map()
     this.cache = !!opts.cache
+    this.primaryKey = opts.primaryKey || null
 
     this._keyStorage = null
-    this._primaryKey = opts.primaryKey
     this._namespace = opts.namespace || DEFAULT_NAMESPACE
 
-    this._root = opts._root || this
-    this._replicationStreams = opts._streams || []
+    this._root = root || this
+    this._replicationStreams = root ? root._replicationStreams : []
     this._overwrite = opts.overwrite === true
 
-    this._streamSessions = opts._streamSessions || new Map()
     this._sessions = new Set() // sessions for THIS namespace
 
     this._findingPeersCount = 0
@@ -40,9 +41,12 @@ module.exports = class Corestore extends EventEmitter {
     if (this._namespace.byteLength !== 32) throw new Error('Namespace must be a 32-byte Buffer or Uint8Array')
 
     this._closing = null
-    this._opening = opts._opening ? opts._opening.then(() => this._open()) : this._open()
+    this._opening = this._open()
     this._opening.catch(safetyCatch)
-    this.ready = () => this._opening
+  }
+
+  ready () {
+    return this._opening
   }
 
   findingPeers () {
@@ -73,16 +77,19 @@ module.exports = class Corestore extends EventEmitter {
   }
 
   async _open () {
-    if (this._primaryKey) {
-      this.primaryKey = await this._primaryKey
-      return this.primaryKey
+    if (this._root !== this) {
+      await this._root._opening
+      this.primaryKey = this._root.primaryKey
+      return
     }
+
     this._keyStorage = this.storage(PRIMARY_KEY_FILE_NAME)
+
     this.primaryKey = await new Promise((resolve, reject) => {
       this._keyStorage.stat((err, st) => {
         if (err && err.code !== 'ENOENT') return reject(err)
         if (err || st.size < 32 || this._overwrite) {
-          const key = crypto.randomBytes(32)
+          const key = this.primaryKey || crypto.randomBytes(32)
           return this._keyStorage.write(0, key, err => {
             if (err) return reject(err)
             return resolve(key)
@@ -90,11 +97,11 @@ module.exports = class Corestore extends EventEmitter {
         }
         this._keyStorage.read(0, 32, (err, key) => {
           if (err) return reject(err)
+          if (this.primaryKey) return resolve(this.primaryKey)
           return resolve(key)
         })
       })
     })
-    return this.primaryKey
   }
 
   async _generateKeys (opts) {
@@ -150,7 +157,7 @@ module.exports = class Corestore extends EventEmitter {
   }
 
   async _preload (opts) {
-    await this.ready()
+    if (!this.primaryKey) await this._opening
 
     const { discoveryKey, keyPair, auth } = await this._generateKeys(opts)
     const id = b4a.toString(discoveryKey, 'hex')
@@ -194,10 +201,7 @@ module.exports = class Corestore extends EventEmitter {
     this.cores.set(id, core)
     core.ready().then(() => {
       for (const { stream } of this._replicationStreams) {
-        const sessions = this._streamSessions.get(stream)
-        const session = core.session()
-        sessions.push(session)
-        core.replicate(stream)
+        core.replicate(stream, { session: true })
       }
     }, () => {
       this.cores.delete(id)
@@ -268,36 +272,26 @@ module.exports = class Corestore extends EventEmitter {
       }
     })
 
-    const sessions = []
     for (const core of this.cores.values()) {
       if (!core.opened) continue // If the core is not opened, it will be replicated in preload.
-      const session = core.session()
-      sessions.push(session)
-      core.replicate(stream)
+      core.replicate(stream, { session: true })
     }
 
     const streamRecord = { stream, isExternal }
     this._replicationStreams.push(streamRecord)
-    this._streamSessions.set(stream, sessions)
 
     stream.once('close', () => {
       this._replicationStreams.splice(this._replicationStreams.indexOf(streamRecord), 1)
-      this._streamSessions.delete(stream)
-      Promise.all(sessions.map(s => s.close())).catch(safetyCatch)
     })
+
     return stream
   }
 
   namespace (name) {
     return new Corestore(this.storage, {
-      primaryKey: this._opening.then(() => this.primaryKey),
       namespace: generateNamespace(this._namespace, name),
       cache: this.cache,
-      _root: this._root,
-      _opening: this._opening,
-      _cores: this.cores,
-      _streams: this._replicationStreams,
-      _streamSessions: this._streamSessions
+      _root: this._root
     })
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "corestore",
-  "version": "6.0.5",
+  "version": "6.1.0",
   "description": "A Hypercore factory that simplifies managing collections of cores.",
   "main": "index.js",
   "scripts": {
@@ -19,14 +19,18 @@
     "url": "https://github.com/hypercore-protocol/corestore/issues"
   },
   "homepage": "https://github.com/hypercore-protocol/corestore#readme",
+  "files": [
+    "index.js",
+    "lib/**.js"
+  ],
   "devDependencies": {
     "brittle": "^3.0.0",
-    "random-access-memory": "^5.0.1",
+    "random-access-memory": "^6.0.0",
     "standardx": "^7.0.0"
   },
   "dependencies": {
     "b4a": "^1.3.1",
-    "hypercore": "^10.2.0",
+    "hypercore": "^10.3.1",
     "hypercore-crypto": "^3.2.1",
     "safety-catch": "^1.0.1",
     "sodium-universal": "^3.0.4",

package/.github/workflows/test-node.yml

@@ -1,23 +0,0 @@
-name: Build Status
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-jobs:
-  build:
-    strategy:
-      matrix:
-        node-version: [lts/*]
-        os: [ubuntu-latest, macos-latest, windows-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v2
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2
-      with:
-        node-version: ${{ matrix.node-version }}
-    - run: npm install
-    - run: npm test

package/test/all.js

@@ -1,411 +0,0 @@
-const test = require('brittle')
-const crypto = require('hypercore-crypto')
-const ram = require('random-access-memory')
-const os = require('os')
-const path = require('path')
-const b4a = require('b4a')
-const sodium = require('sodium-universal')
-
-const Corestore = require('..')
-
-test('basic get with caching', async function (t) {
-  const store = new Corestore(ram)
-  const core1a = store.get({ name: 'core-1' })
-  const core1b = store.get({ name: 'core-1' })
-  const core2 = store.get({ name: 'core-2' })
-
-  await Promise.all([core1a.ready(), core1b.ready(), core2.ready()])
-
-  t.alike(core1a.key, core1b.key)
-  t.unlike(core1a.key, core2.key)
-
-  t.ok(core1a.writable)
-  t.ok(core1b.writable)
-
-  t.is(store.cores.size, 2)
-})
-
-test('basic get with custom keypair', async function (t) {
-  const store = new Corestore(ram)
-  const kp1 = crypto.keyPair()
-  const kp2 = crypto.keyPair()
-
-  const core1 = store.get(kp1)
-  const core2 = store.get(kp2)
-  await Promise.all([core1.ready(), core2.ready()])
-
-  t.alike(core1.key, kp1.publicKey)
-  t.alike(core2.key, kp2.publicKey)
-  t.ok(core1.writable)
-  t.ok(core2.writable)
-})
-
-test('get with createIfMissing=false throws if new core', async function (t) {
-  const store = new Corestore(ram)
-  const core1a = store.get({ name: 'core-1', createIfMissing: false })
-
-  await t.exception(core1a.ready(), 'No Hypercore is stored here')
-})
-
-test('get with createIfMissing=false works if no new core', async function (t) {
-  const store = new Corestore(ram)
-  const name = 'core-1'
-
-  const core1 = store.get({ name })
-  await core1.ready()
-
-  const core1Too = store.get({ name, createIfMissing: false })
-  await t.execution(core1Too.ready())
-})
-
-test('basic namespaces', async function (t) {
-  const store = new Corestore(ram)
-  const ns1 = store.namespace('ns1')
-  const ns2 = store.namespace('ns2')
-  const ns3 = store.namespace('ns1') // Duplicate namespace
-
-  const core1 = ns1.get({ name: 'main' })
-  const core2 = ns2.get({ name: 'main' })
-  const core3 = ns3.get({ name: 'main' })
-  await Promise.all([core1.ready(), core2.ready(), core3.ready()])
-
-  t.absent(core1.key.equals(core2.key))
-  t.ok(core1.key.equals(core3.key))
-  t.ok(core1.writable)
-  t.ok(core2.writable)
-  t.ok(core3.writable)
-  t.is(store.cores.size, 2)
-
-  t.end()
-})
-
-test('basic replication', async function (t) {
-  const store1 = new Corestore(ram)
-  const store2 = new Corestore(ram)
-
-  const core1 = store1.get({ name: 'core-1' })
-  const core2 = store1.get({ name: 'core-2' })
-  await core1.append('hello')
-  await core2.append('world')
-
-  const core3 = store2.get({ key: core1.key })
-  const core4 = store2.get({ key: core2.key })
-
-  const s = store1.replicate(true)
-  s.pipe(store2.replicate(false)).pipe(s)
-
-  t.alike(await core3.get(0), Buffer.from('hello'))
-  t.alike(await core4.get(0), Buffer.from('world'))
-})
-
-test('replicating cores created after replication begins', async function (t) {
-  const store1 = new Corestore(ram)
-  const store2 = new Corestore(ram)
-
-  const s = store1.replicate(true, { live: true })
-  s.pipe(store2.replicate(false, { live: true })).pipe(s)
-
-  const core1 = store1.get({ name: 'core-1' })
-  const core2 = store1.get({ name: 'core-2' })
-  await core1.append('hello')
-  await core2.append('world')
-
-  const core3 = store2.get({ key: core1.key })
-  const core4 = store2.get({ key: core2.key })
-
-  t.alike(await core3.get(0), Buffer.from('hello'))
-  t.alike(await core4.get(0), Buffer.from('world'))
-})
-
-test('replicating cores using discovery key hook', async function (t) {
-  const dir = tmpdir()
-  let store1 = new Corestore(dir)
-  const store2 = new Corestore(ram)
-
-  const core = store1.get({ name: 'main' })
-  await core.append('hello')
-  const key = core.key
-
-  await store1.close()
-  store1 = new Corestore(dir)
-
-  const s = store1.replicate(true, { live: true })
-  s.pipe(store2.replicate(false, { live: true })).pipe(s)
-
-  const core2 = store2.get(key)
-  t.alike(await core2.get(0), Buffer.from('hello'))
-})
-
-test('nested namespaces', async function (t) {
-  const store = new Corestore(ram)
-  const ns1a = store.namespace('ns1').namespace('a')
-  const ns1b = store.namespace('ns1').namespace('b')
-
-  const core1 = ns1a.get({ name: 'main' })
-  const core2 = ns1b.get({ name: 'main' })
-  await Promise.all([core1.ready(), core2.ready()])
-
-  t.not(core1.key.equals(core2.key))
-  t.ok(core1.writable)
-  t.ok(core2.writable)
-  t.is(store.cores.size, 2)
-})
-
-test('core uncached when all sessions close', async function (t) {
-  const store = new Corestore(ram)
-  const core1 = store.get({ name: 'main' })
-  await core1.ready()
-  t.is(store.cores.size, 1)
-  await core1.close()
-  t.is(store.cores.size, 0)
-})
-
-test('writable core loaded from name userData', async function (t) {
-  const dir = tmpdir()
-
-  let store = new Corestore(dir)
-  let core = store.get({ name: 'main' })
-  await core.ready()
-  const key = core.key
-
-  t.ok(core.writable)
-  await core.append('hello')
-  t.is(core.length, 1)
-
-  await store.close()
-  store = new Corestore(dir)
-  core = store.get(key)
-  await core.ready()
-
-  t.ok(core.writable)
-  await core.append('world')
-  t.is(core.length, 2)
-  t.alike(await core.get(0), Buffer.from('hello'))
-  t.alike(await core.get(1), Buffer.from('world'))
-})
-
-test('writable core loaded from name and namespace userData', async function (t) {
-  const dir = tmpdir()
-
-  let store = new Corestore(dir)
-  let core = store.namespace('ns1').get({ name: 'main' })
-  await core.ready()
-  const key = core.key
-
-  t.ok(core.writable)
-  await core.append('hello')
-  t.is(core.length, 1)
-
-  await store.close()
-  store = new Corestore(dir)
-  core = store.get(key)
-  await core.ready()
-
-  t.ok(core.writable)
-  await core.append('world')
-  t.is(core.length, 2)
-  t.alike(await core.get(0), Buffer.from('hello'))
-  t.alike(await core.get(1), Buffer.from('world'))
-})
-
-test('storage locking', async function (t) {
-  const dir = tmpdir()
-
-  const store1 = new Corestore(dir)
-  await store1.ready()
-
-  const store2 = new Corestore(dir)
-  try {
-    await store2.ready()
-    t.fail('dir should have been locked')
-  } catch {
-    t.pass('dir was locked')
-  }
-})
-
-test('cores close when their last referencing namespace closes', async function (t) {
-  const store = new Corestore(ram)
-  const ns1 = store.namespace('ns1')
-  const core1 = ns1.get({ name: 'core-1' })
-  const core2 = ns1.get({ name: 'core-2' })
-  await Promise.all([core1.ready(), core2.ready()])
-
-  const core3 = store.get(core1.key)
-  const core4 = store.get(core2.key)
-  await Promise.all([core3.ready(), core4.ready()])
-
-  await ns1.close()
-
-  t.is(store.cores.size, 2)
-  t.not(core1.closed)
-  t.not(core1.closed)
-
-  await store.close()
-
-  t.is(store.cores.size, 0)
-  t.ok(core1.closed)
-  t.ok(core2.closed)
-})
-
-test('findingPeers', async function (t) {
-  t.plan(6)
-
-  const store = new Corestore(ram)
-
-  const ns1 = store.namespace('ns1')
-  const ns2 = store.namespace('ns2')
-
-  const a = ns1.get(Buffer.alloc(32).fill('a'))
-  const b = ns2.get(Buffer.alloc(32).fill('b'))
-
-  const done = ns1.findingPeers()
-
-  let aUpdated = false
-  let bUpdated = false
-  let cUpdated = false
-
-  const c = ns1.get(Buffer.alloc(32).fill('c'))
-
-  a.update().then(function (bool) {
-    aUpdated = true
-  })
-
-  b.update().then(function (bool) {
-    bUpdated = true
-  })
-
-  c.update().then(function (bool) {
-    cUpdated = true
-  })
-
-  await new Promise(resolve => setImmediate(resolve))
-
-  t.is(aUpdated, false)
-  t.is(bUpdated, true)
-  t.is(cUpdated, false)
-
-  done()
-
-  await new Promise(resolve => setImmediate(resolve))
-
-  t.is(aUpdated, true)
-  t.is(bUpdated, true)
-  t.is(cUpdated, true)
-})
-
-test('different primary keys yield different keypairs', async function (t) {
-  const pk1 = randomBytes(32)
-  const pk2 = randomBytes(32)
-  t.unlike(pk1, pk2)
-
-  const store1 = new Corestore(ram, { primaryKey: pk1 })
-  const store2 = new Corestore(ram, { primaryKey: pk2 })
-
-  const kp1 = await store1.createKeyPair('hello')
-  const kp2 = await store2.createKeyPair('hello')
-
-  t.unlike(kp1.publicKey, kp2.publicKey)
-})
-
-test('keypair auth sign', async function (t) {
-  const store = new Corestore(ram)
-  const keyPair = await store.createKeyPair('foo')
-  const message = b4a.from('hello world')
-
-  const sig = keyPair.auth.sign(message)
-
-  t.is(sig.length, 64)
-  t.ok(crypto.verify(message, sig, keyPair.publicKey))
-  t.absent(crypto.verify(message, b4a.alloc(64), keyPair.publicKey))
-})
-
-test('keypair auth verify', async function (t) {
-  const store = new Corestore(ram)
-  const keyPair = await store.createKeyPair('foo')
-  const message = b4a.from('hello world')
-
-  const sig = crypto.sign(message, keyPair.secretKey)
-
-  t.is(sig.length, 64)
-  t.ok(keyPair.auth.verify(message, sig))
-  t.absent(keyPair.auth.verify(message, b4a.alloc(64)))
-})
-
-test('core caching after reopen regression', async function (t) {
-  const store = new Corestore(ram)
-  const core = store.get({ name: 'test-core' })
-  await core.ready()
-
-  core.close()
-  await core.opening
-
-  const core2 = store.get({ name: 'test-core' })
-  await core2.ready()
-
-  t.pass('did not infinite loop')
-})
-
-test('closing a namespace does not close the root corestore', async function (t) {
-  const store = new Corestore(ram)
-  const core1 = store.get({ name: 'core-1' })
-  await core1.ready()
-
-  const ns = store.namespace('test-namespace')
-  const core2 = ns.get(core1.key)
-  await core2.ready()
-
-  await ns.close()
-  t.is(core1.closed, false)
-  t.is(core2.closed, true)
-})
-
-test('open a new session concurrently with a close should throw', async function (t) {
-  const store = new Corestore(ram)
-  const ns = store.namespace('test-namespace')
-
-  const core1 = ns.get({ name: 'core-1' })
-  store.close()
-
-  try {
-    await core1.ready()
-    t.fail('core1 should not have opened')
-  } catch {
-    t.pass('core1 did not open after corestore close')
-  }
-})
-
-test('closing the root corestore closes all sessions', async function (t) {
-  const store = new Corestore(ram)
-  const ns = store.namespace('test-namespace')
-
-  const core1 = store.get({ name: 'core-1' })
-  const core2 = store.get({ name: 'core-2' })
-  await Promise.all([core1.ready(), core2.ready()])
-
-  const core3 = ns.get(core1.key)
-  const core4 = ns.get(core2.key)
-  await Promise.all([core3.ready(), core4.ready()])
-
-  await store.close()
-
-  t.is(core1.closed, true)
-  t.is(core2.closed, true)
-
-  try {
-    const core5 = ns.get({ name: 'core-3' })
-    await core5.ready()
-    t.fail('core5 should not have opened after corestore close')
-  } catch (err) {
-    t.pass('core5 did not open after corestore close')
-  }
-})
-
-function tmpdir () {
-  return path.join(os.tmpdir(), 'corestore-' + Math.random().toString(16).slice(2))
-}
-
-function randomBytes (n) {
-  const buf = b4a.allocUnsafe(n)
-  sodium.randombytes_buf(buf)
-  return buf
-}

package/test/cache.js

@@ -1,46 +0,0 @@
-const test = require('brittle')
-const RAM = require('random-access-memory')
-
-const Corestore = require('..')
-
-test('core cache', async function (t) {
-  const store = new Corestore(RAM, { cache: true })
-
-  const core = store.get({ name: 'core' })
-  await core.append(['a', 'b', 'c'])
-
-  const p = core.get(0)
-  const q = core.get(0)
-
-  t.is(await p, await q)
-})
-
-test('clear cache on truncate', async function (t) {
-  const store = new Corestore(RAM, { cache: true })
-
-  const core = store.get({ name: 'core' })
-  await core.append(['a', 'b', 'c'])
-
-  const p = core.get(0)
-
-  await core.truncate(0)
-  await core.append('d')
-
-  const q = core.get(0)
-
-  t.alike(await p, Buffer.from('a'))
-  t.alike(await q, Buffer.from('d'))
-})
-
-test('core cache on namespace', async function (t) {
-  const store = new Corestore(RAM, { cache: true })
-  const ns1 = store.namespace('test-namespace-1')
-
-  const c1 = store.get({ name: 'test-core' })
-  const c2 = ns1.get({ name: 'test-core' })
-
-  await Promise.all([c1.ready(), c2.ready()])
-
-  t.ok(c1.cache)
-  t.ok(c2.cache)
-})

package/test/helpers/index.js

@@ -1,19 +0,0 @@
-const fs = require('fs').promises
-
-async function cleanup (dirs) {
-  return Promise.allSettled(dirs.map(dir => fs.rmdir(dir, { recursive: true })))
-}
-
-function delay (ms, cb) {
-  return new Promise(resolve => {
-    setTimeout(() => {
-      if (cb) cb()
-      resolve()
-    }, ms)
-  })
-}
-
-module.exports = {
-  delay,
-  cleanup
-}