corestore 6.0.0-beta2 → 6.0.0

package/test/all.js

@@ -1,302 +1,334 @@
-const p = require('path')
+const test = require('brittle')
+const crypto = require('hypercore-crypto')
 const ram = require('random-access-memory')
-const raf = require('random-access-file')
-const test = require('tape')
-const hypercoreCrypto = require('hypercore-crypto')
-const { toPromises } = require('hypercore-promisifier')
-const OldCorestore = require('@andrewosh/corestore-migration')
+const os = require('os')
+const path = require('path')
+const b4a = require('b4a')
+const sodium = require('sodium-universal')
 
 const Corestore = require('..')
-const { cleanup } = require('./helpers')
 
-test('ram-based corestore, acceptable get options', async t => {
-  const store = create(ram)
+test('basic get with caching', async function (t) {
+  const store = new Corestore(ram)
+  const core1a = store.get({ name: 'core-1' })
+  const core1b = store.get({ name: 'core-1' })
+  const core2 = store.get({ name: 'core-2' })
 
-  // A name option
-  const core1 = store.get({ name: 'default' })
-  await toPromises(core1).ready()
+  await Promise.all([core1a.ready(), core1b.ready(), core2.ready()])
 
-  {
-    // Buffer arg
-    const core = store.get(core1.key)
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
+  t.alike(core1a.key, core1b.key)
+  t.unlike(core1a.key, core2.key)
 
-  {
-    // String arg
-    const core = store.get(core1.key.toString('hex'))
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
+  t.ok(core1a.writable)
+  t.ok(core1b.writable)
 
-  {
-    // Object arg
-    const core = store.get({ key: core1.key })
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
-
-  {
-    // Object arg with string key
-    const core = store.get({ key: core1.key.toString('hex') })
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
+  t.is(store.cores.size, 2)
+})
 
-  {
-    // Custom keypair
-    const core = store.get({ keyPair: { secretKey: core1.secretKey, publicKey: core1.key } })
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
+test('basic get with custom keypair', async function (t) {
+  const store = new Corestore(ram)
+  const kp1 = crypto.keyPair()
+  const kp2 = crypto.keyPair()
 
-  t.end()
-})
+  const core1 = store.get(kp1)
+  const core2 = store.get(kp2)
+  await Promise.all([core1.ready(), core2.ready()])
 
-test('ram-based corestore, unacceptable get options', async t => {
-  const store = create(ram)
-  const badGets = [
-    () => store.get(),
-    () => store.get('abc'),
-    () => store.get({ name: null }),
-    () => store.get({ key: null })
-  ]
-  for (const get of badGets) {
-    try {
-      get()
-      t.fail('get did not throw correctly')
-    } catch (err) {
-      t.true(err)
-    }
-  }
+  t.alike(core1.key, kp1.publicKey)
+  t.alike(core2.key, kp2.publicKey)
+  t.ok(core1.writable)
+  t.ok(core2.writable)
 })
 
-test('ram-based corestore, many gets before ready', async t => {
-  const store = create(ram)
-  const core1 = store.get({ name: 'core1' })
-  const core2 = store.get({ name: 'core1' })
-  const core3 = store.get({ name: 'core3' })
-
-  await Promise.all([
-    toPromises(core1).ready(),
-    toPromises(core2).ready(),
-    toPromises(core3).ready()
-  ])
+test('basic namespaces', async function (t) {
+  const store = new Corestore(ram)
+  const ns1 = store.namespace('ns1')
+  const ns2 = store.namespace('ns2')
+  const ns3 = store.namespace('ns1') // Duplicate namespace
 
-  t.same(core1, core2)
-  t.notSame(core1, core3)
+  const core1 = ns1.get({ name: 'main' })
+  const core2 = ns2.get({ name: 'main' })
+  const core3 = ns3.get({ name: 'main' })
+  await Promise.all([core1.ready(), core2.ready(), core3.ready()])
 
-  // At this point, the pre-ready cores should've been moved to the main cache.
-  t.same(store.cache.size, 2)
+  t.absent(core1.key.equals(core2.key))
+  t.ok(core1.key.equals(core3.key))
+  t.ok(core1.writable)
+  t.ok(core2.writable)
+  t.ok(core3.writable)
+  t.is(store.cores.size, 2)
 
   t.end()
 })
 
-test('ram-based corestore, closing a ref decrements refcount', async t => {
-  const store = create(ram)
-  const core = toPromises(store.get({ name: 'core1' }))
-  await core.ready()
+test('basic replication', async function (t) {
+  const store1 = new Corestore(ram)
+  const store2 = new Corestore(ram)
 
-  const cacheEntry = store.cache.entries.get(core.discoveryKey.toString('hex'))
-  t.same(cacheEntry.refs, 1)
+  const core1 = store1.get({ name: 'core-1' })
+  const core2 = store1.get({ name: 'core-2' })
+  await core1.append('hello')
+  await core2.append('world')
 
-  await core.close()
-  t.same(cacheEntry.refs, 0)
+  const core3 = store2.get({ key: core1.key })
+  const core4 = store2.get({ key: core2.key })
 
-  t.end()
+  const s = store1.replicate(true)
+  s.pipe(store2.replicate(false)).pipe(s)
+
+  t.alike(await core3.get(0), Buffer.from('hello'))
+  t.alike(await core4.get(0), Buffer.from('world'))
 })
 
-test('ram-based corestore, simple replication', async t => {
-  const store1 = create(ram)
-  const store2 = create(ram)
+test('replicating cores created after replication begins', async function (t) {
+  const store1 = new Corestore(ram)
+  const store2 = new Corestore(ram)
 
-  const core1 = toPromises(store1.get({ name: 'core1', valueEncoding: 'utf-8' }))
+  const s = store1.replicate(true, { live: true })
+  s.pipe(store2.replicate(false, { live: true })).pipe(s)
+
+  const core1 = store1.get({ name: 'core-1' })
+  const core2 = store1.get({ name: 'core-2' })
   await core1.append('hello')
+  await core2.append('world')
 
-  const s1 = store1.replicate(true, { live: true })
-  s1.pipe(store2.replicate(false, { live: true })).pipe(s1)
+  const core3 = store2.get({ key: core1.key })
+  const core4 = store2.get({ key: core2.key })
 
-  const clone1 = toPromises(store2.get({ key: core1.key, valueEncoding: 'utf-8' }))
-  t.same(await clone1.get(0), 'hello')
+  t.alike(await core3.get(0), Buffer.from('hello'))
+  t.alike(await core4.get(0), Buffer.from('world'))
+})
 
-  const core2 = toPromises(store1.get({ name: 'core2', valueEncoding: 'utf-8' }))
-  await core2.append('world')
+test('replicating cores using discovery key hook', async function (t) {
+  const dir = tmpdir()
+  let store1 = new Corestore(dir)
+  const store2 = new Corestore(ram)
 
-  const clone2 = toPromises(store2.get({ key: core2.key, valueEncoding: 'utf-8' }))
-  t.same(await clone2.get(0), 'world')
+  const core = store1.get({ name: 'main' })
+  await core.append('hello')
+  const key = core.key
 
-  t.end()
+  await store1.close()
+  store1 = new Corestore(dir)
+
+  const s = store1.replicate(true, { live: true })
+  s.pipe(store2.replicate(false, { live: true })).pipe(s)
+
+  const core2 = store2.get(key)
+  t.alike(await core2.get(0), Buffer.from('hello'))
 })
 
-test('ram-based corestore, sparse replication', async t => {
-  const store1 = create(ram, { sparse: true })
-  const store2 = create(ram, { sparse: true })
+test('nested namespaces', async function (t) {
+  const store = new Corestore(ram)
+  const ns1a = store.namespace('ns1').namespace('a')
+  const ns1b = store.namespace('ns1').namespace('b')
 
-  const core1 = toPromises(store1.get({ name: 'core1', valueEncoding: 'utf-8' }))
-  await core1.append('hello')
+  const core1 = ns1a.get({ name: 'main' })
+  const core2 = ns1b.get({ name: 'main' })
+  await Promise.all([core1.ready(), core2.ready()])
 
-  const s1 = store1.replicate(true, { live: true })
-  s1.pipe(store2.replicate(false, { live: true })).pipe(s1)
+  t.not(core1.key.equals(core2.key))
+  t.ok(core1.writable)
+  t.ok(core2.writable)
+  t.is(store.cores.size, 2)
+})
 
-  const clone1 = toPromises(store2.get({ key: core1.key, valueEncoding: 'utf-8' }))
-  t.same(await clone1.get(0), 'hello')
+test('core uncached when all sessions close', async function (t) {
+  const store = new Corestore(ram)
+  const core1 = store.get({ name: 'main' })
+  await core1.ready()
+  t.is(store.cores.size, 1)
+  await core1.close()
+  t.is(store.cores.size, 0)
+})
 
-  const core2 = toPromises(store1.get({ name: 'core2', valueEncoding: 'utf-8' }))
-  await core2.append('world')
+test('writable core loaded from name userData', async function (t) {
+  const dir = tmpdir()
 
-  const clone2 = toPromises(store2.get({ key: core2.key, valueEncoding: 'utf-8' }))
-  t.same(await clone2.get(0), 'world')
+  let store = new Corestore(dir)
+  let core = store.get({ name: 'main' })
+  await core.ready()
+  const key = core.key
 
-  t.end()
+  t.ok(core.writable)
+  await core.append('hello')
+  t.is(core.length, 1)
+
+  await store.close()
+  store = new Corestore(dir)
+  core = store.get(key)
+  await core.ready()
+
+  t.ok(core.writable)
+  await core.append('world')
+  t.is(core.length, 2)
+  t.alike(await core.get(0), Buffer.from('hello'))
+  t.alike(await core.get(1), Buffer.from('world'))
 })
 
-test('raf-based corestore, simple replication', async t => {
-  const store1 = create(path => raf(p.join('store1', path)))
-  const store2 = create(path => raf(p.join('store2', path)))
+test('writable core loaded from name and namespace userData', async function (t) {
+  const dir = tmpdir()
 
-  const core1 = toPromises(store1.get({ name: 'core1', valueEncoding: 'utf-8' }))
-  await core1.append('hello')
+  let store = new Corestore(dir)
+  let core = store.namespace('ns1').get({ name: 'main' })
+  await core.ready()
+  const key = core.key
 
-  const s1 = store1.replicate(true, { live: true })
-  s1.pipe(store2.replicate(false, { live: true })).pipe(s1)
+  t.ok(core.writable)
+  await core.append('hello')
+  t.is(core.length, 1)
 
-  const clone1 = toPromises(store2.get({ key: core1.key, valueEncoding: 'utf-8' }))
-  t.same(await clone1.get(0), 'hello')
+  await store.close()
+  store = new Corestore(dir)
+  core = store.get(key)
+  await core.ready()
 
-  const core2 = toPromises(store1.get({ name: 'core2', valueEncoding: 'utf-8' }))
-  await core2.append('world')
+  t.ok(core.writable)
+  await core.append('world')
+  t.is(core.length, 2)
+  t.alike(await core.get(0), Buffer.from('hello'))
+  t.alike(await core.get(1), Buffer.from('world'))
+})
 
-  const clone2 = toPromises(store2.get({ key: core2.key, valueEncoding: 'utf-8' }))
-  t.same(await clone2.get(0), 'world')
+test('storage locking', async function (t) {
+  const dir = tmpdir()
 
-  await cleanup(['store1', 'store2'])
-  t.end()
+  const store1 = new Corestore(dir)
+  await store1.ready()
+
+  const store2 = new Corestore(dir)
+  try {
+    await store2.ready()
+    t.fail('dir should have been locked')
+  } catch {
+    t.pass('dir was locked')
+  }
 })
 
-test('raf-based corestore, close and reopen', async t => {
-  let store = create('test-store')
+test('closing a namespace does not close cores', async function (t) {
+  const store = new Corestore(ram)
+  const ns1 = store.namespace('ns1')
+  const core1 = ns1.get({ name: 'core-1' })
+  const core2 = ns1.get({ name: 'core-2' })
+  await Promise.all([core1.ready(), core2.ready()])
 
-  let core1 = toPromises(store.get({ name: 'core1', valueEncoding: 'utf-8' }))
-  await core1.append('hello')
+  await ns1.close()
 
-  t.same(await core1.get(0), 'hello')
+  t.is(store.cores.size, 2)
+  t.not(core1.closed)
+  t.not(core1.closed)
 
   await store.close()
-  store = create('test-store')
-  core1 = toPromises(store.get({ name: 'core1', valueEncoding: 'utf-8' }))
-
-  t.same(await core1.get(0), 'hello')
 
-  await cleanup(['test-store'])
-  t.end()
+  t.is(store.cores.size, 0)
+  t.ok(core1.closed)
+  t.ok(core2.closed)
 })
 
-test('raf-based corestore, close and reopen with keypair option', async t => {
-  let store = create('test-store')
-  const keyPair = hypercoreCrypto.keyPair()
+test('findingPeers', async function (t) {
+  t.plan(6)
 
-  let core1 = toPromises(store.get({ keyPair, valueEncoding: 'utf-8' }))
-  await core1.append('hello')
+  const store = new Corestore(ram)
 
-  t.same(await core1.get(0), 'hello')
+  const ns1 = store.namespace('ns1')
+  const ns2 = store.namespace('ns2')
 
-  await store.close()
-  store = create('test-store')
-  core1 = toPromises(store.get({ keyPair, valueEncoding: 'utf-8' }))
+  const a = ns1.get(Buffer.alloc(32).fill('a'))
+  const b = ns2.get(Buffer.alloc(32).fill('b'))
 
-  t.same(await core1.get(0), 'hello')
+  const done = ns1.findingPeers()
 
-  await cleanup(['test-store'])
-  t.end()
-})
+  let aUpdated = false
+  let bUpdated = false
+  let cUpdated = false
 
-test('namespace method is equivalent to name array', async t => {
-  const store = create(ram)
+  const c = ns1.get(Buffer.alloc(32).fill('c'))
 
-  const core1 = store.get({ name: ['a', 'b', 'c'] })
-  const core2 = store.namespace('a').namespace('b').get({ name: 'c' })
+  a.update().then(function (bool) {
+    aUpdated = true
+  })
 
-  await toPromises(core1).ready()
-  await toPromises(core2).ready()
+  b.update().then(function (bool) {
+    bUpdated = true
+  })
 
-  t.same(core1, core2)
-  t.end()
+  c.update().then(function (bool) {
+    cUpdated = true
+  })
+
+  await new Promise(resolve => setImmediate(resolve))
+
+  t.is(aUpdated, false)
+  t.is(bUpdated, true)
+  t.is(cUpdated, false)
+
+  done()
+
+  await new Promise(resolve => setImmediate(resolve))
+
+  t.is(aUpdated, true)
+  t.is(bUpdated, true)
+  t.is(cUpdated, true)
 })
 
-test('can backup/restore', async t => {
-  const firstStore = create(ram)
-  const core1 = firstStore.get({ name: 'hello-world' })
-  await toPromises(core1).ready()
-  const manifest = await firstStore.backup()
+test('different primary keys yield different keypairs', async function (t) {
+  const pk1 = randomBytes(32)
+  const pk2 = randomBytes(32)
+  t.unlike(pk1, pk2)
 
-  const secondStore = await Corestore.restore(manifest, ram)
-  const core2 = secondStore.get({ key: core1.key })
-  await toPromises(core2).ready()
+  const store1 = new Corestore(ram, { primaryKey: pk1 })
+  const store2 = new Corestore(ram, { primaryKey: pk2 })
 
-  t.true(core2.writable)
+  const kp1 = await store1.createKeyPair('hello')
+  const kp2 = await store2.createKeyPair('hello')
 
-  t.end()
+  t.unlike(kp1.publicKey, kp2.publicKey)
 })
 
-test('can migrate an old corestore', async t => {
-  const oldStore = new OldCorestore('test-store')
-  await oldStore.ready()
-
-  const oldCores = [
-    oldStore.default(),
-    oldStore.namespace('hello').default(),
-    oldStore.namespace('hello').namespace('world').default(),
-    oldStore.get({ name: 'test-name' }),
-    // Randomly-generated names should also be migrated.
-    oldStore.get()
-  ]
-  await Promise.all(oldCores.map(c => new Promise(resolve => c.ready(resolve))))
-
-  const names = [
-    'default',
-    ['hello'],
-    ['hello', 'world'],
-    'test-name'
-  ]
-  const keys = oldCores.map(c => c.key)
-  await new Promise(resolve => oldStore.close(resolve))
-
-  {
-    const store = create('test-store', {
-      migrationRoot: 'test-store'
-    })
-
-    // Cores are writable when loaded by name
-    for (let i = 0; i < names.length; i++) {
-      const core = store.get({ name: names[i] })
-      await toPromises(core).ready()
-      t.true(core.writable)
-      t.same(core.key, keys[i])
-    }
-
-    await store.close()
-  }
+test('keypair auth sign', async function (t) {
+  const store = new Corestore(ram)
+  const keyPair = await store.createKeyPair('foo')
+  const message = b4a.from('hello world')
 
-  {
-    // The second time should work without a migration.
-    const store = create('test-store')
+  const sig = keyPair.auth.sign(message)
 
-    // Cores are writable when loaded by key
-    for (let i = 0; i < keys.length; i++) {
-      const core = store.get({ key: keys[i] })
-      await toPromises(core).ready()
-      t.true(core.writable)
-    }
+  t.is(sig.length, 64)
+  t.ok(crypto.verify(message, sig, keyPair.publicKey))
+  t.absent(crypto.verify(message, b4a.alloc(64), keyPair.publicKey))
+})
 
-    await store.close()
-  }
+test('keypair auth verify', async function (t) {
+  const store = new Corestore(ram)
+  const keyPair = await store.createKeyPair('foo')
+  const message = b4a.from('hello world')
 
-  await cleanup(['test-store'])
-  t.end()
+  const sig = crypto.sign(message, keyPair.secretKey)
+
+  t.is(sig.length, 64)
+  t.ok(keyPair.auth.verify(message, sig))
+  t.absent(keyPair.auth.verify(message, b4a.alloc(64)))
 })
 
-function create (storage, opts) {
-  const store = new Corestore(storage, opts)
-  return store
+test('core caching after reopen regression', async function (t) {
+  const store = new Corestore(ram)
+  const core = store.get({ name: 'test-core' })
+  await core.ready()
+
+  core.close()
+  await core.opening
+
+  const core2 = store.get({ name: 'test-core' })
+  await core2.ready()
+
+  t.pass('did not infinite loop')
+})
+
+function tmpdir () {
+  return path.join(os.tmpdir(), 'corestore-' + Math.random().toString(16).slice(2))
+}
+
+function randomBytes (n) {
+  const buf = b4a.allocUnsafe(n)
+  sodium.randombytes_buf(buf)
+  return buf
 }

package/test/cache.js

@@ -0,0 +1,46 @@
+const test = require('brittle')
+const RAM = require('random-access-memory')
+
+const Corestore = require('..')
+
+test('core cache', async function (t) {
+  const store = new Corestore(RAM, { cache: true })
+
+  const core = store.get({ name: 'core' })
+  await core.append(['a', 'b', 'c'])
+
+  const p = core.get(0)
+  const q = core.get(0)
+
+  t.is(await p, await q)
+})
+
+test('clear cache on truncate', async function (t) {
+  const store = new Corestore(RAM, { cache: true })
+
+  const core = store.get({ name: 'core' })
+  await core.append(['a', 'b', 'c'])
+
+  const p = core.get(0)
+
+  await core.truncate(0)
+  await core.append('d')
+
+  const q = core.get(0)
+
+  t.alike(await p, Buffer.from('a'))
+  t.alike(await q, Buffer.from('d'))
+})
+
+test('core cache on namespace', async function (t) {
+  const store = new Corestore(RAM, { cache: true })
+  const ns1 = store.namespace('test-namespace-1')
+
+  const c1 = store.get({ name: 'test-core' })
+  const c2 = ns1.get({ name: 'test-core' })
+
+  await Promise.all([c1.ready(), c2.ready()])
+
+  t.ok(c1.cache)
+  t.ok(c2.cache)
+})

package/test/helpers/index.js

@@ -1,12 +1,7 @@
-const fs = require('fs')
+const fs = require('fs').promises
 
 async function cleanup (dirs) {
-  return Promise.all(dirs.map(dir => new Promise((resolve, reject) => {
-    fs.rmdir(dir, { recursive: true }, err => {
-      if (err) return reject(err)
-      return resolve()
-    })
-  })))
+  return Promise.allSettled(dirs.map(dir => fs.rmdir(dir, { recursive: true })))
 }
 
 function delay (ms, cb) {

package/lib/buffer-file.js

@@ -1,26 +0,0 @@
-const RAS = require('random-access-storage')
-
-module.exports = class BufferFile extends RAS {
-  constructor (buf) {
-    super()
-    this._buf = buf
-  }
-
-  _open (req) {
-    if (typeof this._buf !== 'function') return req.callback(null)
-    this._buf((err, b) => {
-      if (err) return req.callback(err)
-      this._buf = b
-      req.callback(null)
-    })
-  }
-
-  _stat (req) {
-    req.callback(null, { size: this._buf ? this._buf.length : 0 })
-  }
-
-  _read (req) {
-    if (!this._buf) return req.callback(new Error('Could not satisfy length'))
-    req.callback(null, this._buf.slice(req.offset, req.offset + req.size))
-  }
-}