corestore 6.0.0-beta2 → 6.0.1-alpha.12

package/test/all.js

@@ -1,302 +1,207 @@
-const p = require('path')
+const { test, configure } = require('brittle')
+const crypto = require('hypercore-crypto')
 const ram = require('random-access-memory')
-const raf = require('random-access-file')
-const test = require('tape')
-const hypercoreCrypto = require('hypercore-crypto')
-const { toPromises } = require('hypercore-promisifier')
-const OldCorestore = require('@andrewosh/corestore-migration')
+const tmp = require('tmp-promise')
 
 const Corestore = require('..')
-const { cleanup } = require('./helpers')
 
-test('ram-based corestore, acceptable get options', async t => {
-  const store = create(ram)
+configure({ serial: true })
 
-  // A name option
-  const core1 = store.get({ name: 'default' })
-  await toPromises(core1).ready()
+test('basic get with caching', async function (t) {
+  const store = new Corestore(ram)
+  const core1a = store.get({ name: 'core-1' })
+  const core1b = store.get({ name: 'core-1' })
+  const core2 = store.get({ name: 'core-2' })
 
-  {
-    // Buffer arg
-    const core = store.get(core1.key)
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
-
-  {
-    // String arg
-    const core = store.get(core1.key.toString('hex'))
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
+  await Promise.all([core1a.ready(), core1b.ready(), core2.ready()])
 
-  {
-    // Object arg
-    const core = store.get({ key: core1.key })
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
+  t.alike(core1a.key, core1b.key)
+  t.unlike(core1a.key, core2.key)
 
-  {
-    // Object arg with string key
-    const core = store.get({ key: core1.key.toString('hex') })
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
-
-  {
-    // Custom keypair
-    const core = store.get({ keyPair: { secretKey: core1.secretKey, publicKey: core1.key } })
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
+  t.ok(core1a.writable)
+  t.ok(core1b.writable)
 
-  t.end()
-})
-
-test('ram-based corestore, unacceptable get options', async t => {
-  const store = create(ram)
-  const badGets = [
-    () => store.get(),
-    () => store.get('abc'),
-    () => store.get({ name: null }),
-    () => store.get({ key: null })
-  ]
-  for (const get of badGets) {
-    try {
-      get()
-      t.fail('get did not throw correctly')
-    } catch (err) {
-      t.true(err)
-    }
-  }
+  t.is(store.cores.size, 2)
 })
 
-test('ram-based corestore, many gets before ready', async t => {
-  const store = create(ram)
-  const core1 = store.get({ name: 'core1' })
-  const core2 = store.get({ name: 'core1' })
-  const core3 = store.get({ name: 'core3' })
-
-  await Promise.all([
-    toPromises(core1).ready(),
-    toPromises(core2).ready(),
-    toPromises(core3).ready()
-  ])
-
-  t.same(core1, core2)
-  t.notSame(core1, core3)
+test('basic get with custom keypair', async function (t) {
+  const store = new Corestore(ram)
+  const kp1 = crypto.keyPair()
+  const kp2 = crypto.keyPair()
 
-  // At this point, the pre-ready cores should've been moved to the main cache.
-  t.same(store.cache.size, 2)
+  const core1 = store.get(kp1)
+  const core2 = store.get(kp2)
+  await Promise.all([core1.ready(), core2.ready()])
 
-  t.end()
+  t.alike(core1.key, kp1.publicKey)
+  t.alike(core2.key, kp2.publicKey)
+  t.ok(core1.writable)
+  t.ok(core2.writable)
 })
 
-test('ram-based corestore, closing a ref decrements refcount', async t => {
-  const store = create(ram)
-  const core = toPromises(store.get({ name: 'core1' }))
-  await core.ready()
+test('basic namespaces', async function (t) {
+  const store = new Corestore(ram)
+  const ns1 = store.namespace('ns1')
+  const ns2 = store.namespace('ns2')
+  const ns3 = store.namespace('ns1') // Duplicate namespace
 
-  const cacheEntry = store.cache.entries.get(core.discoveryKey.toString('hex'))
-  t.same(cacheEntry.refs, 1)
+  const core1 = ns1.get({ name: 'main' })
+  const core2 = ns2.get({ name: 'main' })
+  const core3 = ns3.get({ name: 'main' })
+  await Promise.all([core1.ready(), core2.ready(), core3.ready()])
 
-  await core.close()
-  t.same(cacheEntry.refs, 0)
+  t.absent(core1.key.equals(core2.key))
+  t.ok(core1.key.equals(core3.key))
+  t.ok(core1.writable)
+  t.ok(core2.writable)
+  t.ok(core3.writable)
+  t.is(store.cores.size, 2)
 
   t.end()
 })
 
-test('ram-based corestore, simple replication', async t => {
-  const store1 = create(ram)
-  const store2 = create(ram)
+test('basic replication', async function (t) {
+  const store1 = new Corestore(ram)
+  const store2 = new Corestore(ram)
 
-  const core1 = toPromises(store1.get({ name: 'core1', valueEncoding: 'utf-8' }))
+  const core1 = store1.get({ name: 'core-1' })
+  const core2 = store1.get({ name: 'core-2' })
   await core1.append('hello')
-
-  const s1 = store1.replicate(true, { live: true })
-  s1.pipe(store2.replicate(false, { live: true })).pipe(s1)
-
-  const clone1 = toPromises(store2.get({ key: core1.key, valueEncoding: 'utf-8' }))
-  t.same(await clone1.get(0), 'hello')
-
-  const core2 = toPromises(store1.get({ name: 'core2', valueEncoding: 'utf-8' }))
   await core2.append('world')
 
-  const clone2 = toPromises(store2.get({ key: core2.key, valueEncoding: 'utf-8' }))
-  t.same(await clone2.get(0), 'world')
+  const core3 = store2.get({ key: core1.key })
+  const core4 = store2.get({ key: core2.key })
 
-  t.end()
-})
+  const s = store1.replicate(true)
+  s.pipe(store2.replicate(false)).pipe(s)
 
-test('ram-based corestore, sparse replication', async t => {
-  const store1 = create(ram, { sparse: true })
-  const store2 = create(ram, { sparse: true })
-
-  const core1 = toPromises(store1.get({ name: 'core1', valueEncoding: 'utf-8' }))
-  await core1.append('hello')
+  t.alike(await core3.get(0), Buffer.from('hello'))
+  t.alike(await core4.get(0), Buffer.from('world'))
+})
 
-  const s1 = store1.replicate(true, { live: true })
-  s1.pipe(store2.replicate(false, { live: true })).pipe(s1)
+test('replicating cores created after replication begins', async function (t) {
+  const store1 = new Corestore(ram)
+  const store2 = new Corestore(ram)
 
-  const clone1 = toPromises(store2.get({ key: core1.key, valueEncoding: 'utf-8' }))
-  t.same(await clone1.get(0), 'hello')
+  const s = store1.replicate(true, { live: true })
+  s.pipe(store2.replicate(false, { live: true })).pipe(s)
 
-  const core2 = toPromises(store1.get({ name: 'core2', valueEncoding: 'utf-8' }))
+  const core1 = store1.get({ name: 'core-1' })
+  const core2 = store1.get({ name: 'core-2' })
+  await core1.append('hello')
   await core2.append('world')
 
-  const clone2 = toPromises(store2.get({ key: core2.key, valueEncoding: 'utf-8' }))
-  t.same(await clone2.get(0), 'world')
+  const core3 = store2.get({ key: core1.key })
+  const core4 = store2.get({ key: core2.key })
 
-  t.end()
+  t.alike(await core3.get(0), Buffer.from('hello'))
+  t.alike(await core4.get(0), Buffer.from('world'))
 })
 
-test('raf-based corestore, simple replication', async t => {
-  const store1 = create(path => raf(p.join('store1', path)))
-  const store2 = create(path => raf(p.join('store2', path)))
-
-  const core1 = toPromises(store1.get({ name: 'core1', valueEncoding: 'utf-8' }))
-  await core1.append('hello')
+test('replicating cores using discovery key hook', async function (t) {
+  const dir = await tmp.dir({ unsafeCleanup: true })
+  let store1 = new Corestore(dir.path)
+  const store2 = new Corestore(ram)
 
-  const s1 = store1.replicate(true, { live: true })
-  s1.pipe(store2.replicate(false, { live: true })).pipe(s1)
+  const core = store1.get({ name: 'main' })
+  await core.append('hello')
+  const key = core.key
 
-  const clone1 = toPromises(store2.get({ key: core1.key, valueEncoding: 'utf-8' }))
-  t.same(await clone1.get(0), 'hello')
+  await store1.close()
+  store1 = new Corestore(dir.path)
 
-  const core2 = toPromises(store1.get({ name: 'core2', valueEncoding: 'utf-8' }))
-  await core2.append('world')
+  const s = store1.replicate(true, { live: true })
+  s.pipe(store2.replicate(false, { live: true })).pipe(s)
 
-  const clone2 = toPromises(store2.get({ key: core2.key, valueEncoding: 'utf-8' }))
-  t.same(await clone2.get(0), 'world')
+  const core2 = store2.get(key)
+  t.alike(await core2.get(0), Buffer.from('hello'))
 
-  await cleanup(['store1', 'store2'])
-  t.end()
+  await dir.cleanup()
 })
 
-test('raf-based corestore, close and reopen', async t => {
-  let store = create('test-store')
+test('nested namespaces', async function (t) {
+  const store = new Corestore(ram)
+  const ns1a = store.namespace('ns1').namespace('a')
+  const ns1b = store.namespace('ns1').namespace('b')
 
-  let core1 = toPromises(store.get({ name: 'core1', valueEncoding: 'utf-8' }))
-  await core1.append('hello')
-
-  t.same(await core1.get(0), 'hello')
-
-  await store.close()
-  store = create('test-store')
-  core1 = toPromises(store.get({ name: 'core1', valueEncoding: 'utf-8' }))
+  const core1 = ns1a.get({ name: 'main' })
+  const core2 = ns1b.get({ name: 'main' })
+  await Promise.all([core1.ready(), core2.ready()])
 
-  t.same(await core1.get(0), 'hello')
+  t.not(core1.key.equals(core2.key))
+  t.ok(core1.writable)
+  t.ok(core2.writable)
+  t.is(store.cores.size, 2)
+})
 
-  await cleanup(['test-store'])
-  t.end()
+test('core uncached when all sessions close', async function (t) {
+  const store = new Corestore(ram)
+  const core1 = store.get({ name: 'main' })
+  await core1.ready()
+  t.is(store.cores.size, 1)
+  await core1.close()
+  t.is(store.cores.size, 0)
 })
 
-test('raf-based corestore, close and reopen with keypair option', async t => {
-  let store = create('test-store')
-  const keyPair = hypercoreCrypto.keyPair()
+test('writable core loaded from name userData', async function (t) {
+  const dir = await tmp.dir({ unsafeCleanup: true })
 
-  let core1 = toPromises(store.get({ keyPair, valueEncoding: 'utf-8' }))
-  await core1.append('hello')
+  let store = new Corestore(dir.path)
+  let core = store.get({ name: 'main' })
+  await core.ready()
+  const key = core.key
 
-  t.same(await core1.get(0), 'hello')
+  t.ok(core.writable)
+  await core.append('hello')
+  t.is(core.length, 1)
 
   await store.close()
-  store = create('test-store')
-  core1 = toPromises(store.get({ keyPair, valueEncoding: 'utf-8' }))
-
-  t.same(await core1.get(0), 'hello')
-
-  await cleanup(['test-store'])
-  t.end()
-})
-
-test('namespace method is equivalent to name array', async t => {
-  const store = create(ram)
-
-  const core1 = store.get({ name: ['a', 'b', 'c'] })
-  const core2 = store.namespace('a').namespace('b').get({ name: 'c' })
+  store = new Corestore(dir.path)
+  core = store.get(key)
+  await core.ready()
 
-  await toPromises(core1).ready()
-  await toPromises(core2).ready()
+  t.ok(core.writable)
+  await core.append('world')
+  t.is(core.length, 2)
+  t.alike(await core.get(0), Buffer.from('hello'))
+  t.alike(await core.get(1), Buffer.from('world'))
 
-  t.same(core1, core2)
-  t.end()
+  await dir.cleanup()
 })
 
-test('can backup/restore', async t => {
-  const firstStore = create(ram)
-  const core1 = firstStore.get({ name: 'hello-world' })
-  await toPromises(core1).ready()
-  const manifest = await firstStore.backup()
+test('storage locking', async function (t) {
+  const dir = await tmp.dir({ unsafeCleanup: true })
 
-  const secondStore = await Corestore.restore(manifest, ram)
-  const core2 = secondStore.get({ key: core1.key })
-  await toPromises(core2).ready()
+  const store1 = new Corestore(dir.path)
+  await store1.ready()
 
-  t.true(core2.writable)
+  const store2 = new Corestore(dir.path)
+  try {
+    await store2.ready()
+    t.fail('dir should have been locked')
+  } catch {
+    t.pass('dir was locked')
+  }
 
-  t.end()
+  await dir.cleanup()
 })
 
-test('can migrate an old corestore', async t => {
-  const oldStore = new OldCorestore('test-store')
-  await oldStore.ready()
-
-  const oldCores = [
-    oldStore.default(),
-    oldStore.namespace('hello').default(),
-    oldStore.namespace('hello').namespace('world').default(),
-    oldStore.get({ name: 'test-name' }),
-    // Randomly-generated names should also be migrated.
-    oldStore.get()
-  ]
-  await Promise.all(oldCores.map(c => new Promise(resolve => c.ready(resolve))))
-
-  const names = [
-    'default',
-    ['hello'],
-    ['hello', 'world'],
-    'test-name'
-  ]
-  const keys = oldCores.map(c => c.key)
-  await new Promise(resolve => oldStore.close(resolve))
-
-  {
-    const store = create('test-store', {
-      migrationRoot: 'test-store'
-    })
-
-    // Cores are writable when loaded by name
-    for (let i = 0; i < names.length; i++) {
-      const core = store.get({ name: names[i] })
-      await toPromises(core).ready()
-      t.true(core.writable)
-      t.same(core.key, keys[i])
-    }
-
-    await store.close()
-  }
+test('closing a namespace does not close cores', async function (t) {
+  const store = new Corestore(ram)
+  const ns1 = store.namespace('ns1')
+  const core1 = ns1.get({ name: 'core-1' })
+  const core2 = ns1.get({ name: 'core-2' })
+  await Promise.all([core1.ready(), core2.ready()])
 
-  {
-    // The second time should work without a migration.
-    const store = create('test-store')
+  await ns1.close()
 
-    // Cores are writable when loaded by key
-    for (let i = 0; i < keys.length; i++) {
-      const core = store.get({ key: keys[i] })
-      await toPromises(core).ready()
-      t.true(core.writable)
-    }
+  t.is(store.cores.size, 2)
+  t.not(core1.closed)
+  t.not(core1.closed)
 
-    await store.close()
-  }
+  await store.close()
 
-  await cleanup(['test-store'])
-  t.end()
+  t.is(store.cores.size, 0)
+  t.ok(core1.closed)
+  t.ok(core2.closed)
 })
-
-function create (storage, opts) {
-  const store = new Corestore(storage, opts)
-  return store
-}

package/test/helpers/index.js

@@ -1,12 +1,7 @@
-const fs = require('fs')
+const fs = require('fs').promises
 
 async function cleanup (dirs) {
-  return Promise.all(dirs.map(dir => new Promise((resolve, reject) => {
-    fs.rmdir(dir, { recursive: true }, err => {
-      if (err) return reject(err)
-      return resolve()
-    })
-  })))
+  return Promise.allSettled(dirs.map(dir => fs.rmdir(dir, { recursive: true })))
 }
 
 function delay (ms, cb) {

package/test/keys.js

@@ -0,0 +1,65 @@
+const p = require('path')
+const fs = require('fs')
+
+const test = require('brittle')
+const ram = require('random-access-memory')
+const raf = require('random-access-file')
+
+const KeyManager = require('../lib/keys')
+
+test('can create hypercore keypairs', async t => {
+  const keys = await KeyManager.fromStorage(ram)
+
+  const kp1 = await keys.createHypercoreKeyPair('core1')
+  const kp2 = await keys.createHypercoreKeyPair('core2')
+
+  t.is(kp1.publicKey.length, 32)
+  t.is(kp2.publicKey.length, 32)
+  t.unlike(kp1.publicKey, kp2.publicKey)
+})
+
+test('distinct tokens create distinct hypercore keypairs', async t => {
+  const keys = await KeyManager.fromStorage(ram)
+  const token1 = KeyManager.createToken()
+  const token2 = KeyManager.createToken()
+
+  const kp1 = await keys.createHypercoreKeyPair('core1', token1)
+  const kp2 = await keys.createHypercoreKeyPair('core1', token2)
+
+  t.unlike(kp1.publicKey, kp2.publicKey)
+})
+
+test('short user-provided token will throw', async t => {
+  const keys = await KeyManager.fromStorage(ram)
+
+  try {
+    await keys.createHypercoreKeyPair('core1', Buffer.from('hello'))
+    t.fail('did not throw')
+  } catch {
+    t.pass('threw correctly')
+  }
+})
+
+test('persistent storage regenerates keys correctly', async t => {
+  const testPath = p.resolve(__dirname, 'test-data')
+
+  const keys1 = await KeyManager.fromStorage((name) => raf(testPath, { directory: testPath }))
+  const kp1 = await keys1.createHypercoreKeyPair('core1')
+
+  const keys2 = await KeyManager.fromStorage((name) => raf(testPath, { directory: testPath }))
+  const kp2 = await keys2.createHypercoreKeyPair('core1')
+
+  t.alike(kp1.publicKey, kp2.publicKey)
+
+  await fs.promises.rm(testPath, { recursive: true })
+})
+
+test('different master keys -> different keys', async t => {
+  const keys1 = await KeyManager.fromStorage(ram)
+  const keys2 = await KeyManager.fromStorage(ram)
+
+  const kp1 = await keys1.createHypercoreKeyPair('core1')
+  const kp2 = await keys2.createHypercoreKeyPair('core1')
+
+  t.unlike(kp1.publicKey, kp2.publicKey)
+})

package/lib/buffer-file.js

@@ -1,26 +0,0 @@
-const RAS = require('random-access-storage')
-
-module.exports = class BufferFile extends RAS {
-  constructor (buf) {
-    super()
-    this._buf = buf
-  }
-
-  _open (req) {
-    if (typeof this._buf !== 'function') return req.callback(null)
-    this._buf((err, b) => {
-      if (err) return req.callback(err)
-      this._buf = b
-      req.callback(null)
-    })
-  }
-
-  _stat (req) {
-    req.callback(null, { size: this._buf ? this._buf.length : 0 })
-  }
-
-  _read (req) {
-    if (!this._buf) return req.callback(new Error('Could not satisfy length'))
-    req.callback(null, this._buf.slice(req.offset, req.offset + req.size))
-  }
-}