corestore 6.0.0-beta1 → 6.0.1-alpha.7

package/test/all.js

@@ -1,302 +1,145 @@
-const p = require('path')
+const test = require('brittle')
+const crypto = require('hypercore-crypto')
 const ram = require('random-access-memory')
-const raf = require('random-access-file')
-const test = require('tape')
-const hypercoreCrypto = require('hypercore-crypto')
-const { toPromises } = require('hypercore-promisifier')
-const OldCorestore = require('@andrewosh/corestore-migration')
+const tmp = require('tmp-promise')
 
 const Corestore = require('..')
-const { cleanup } = require('./helpers')
 
-test('ram-based corestore, acceptable get options', async t => {
-  const store = create(ram)
+test('basic get with caching', async function (t) {
+  const store = new Corestore(ram)
+  const core1a = store.get({ name: 'core-1' })
+  const core1b = store.get({ name: 'core-1' })
+  const core2 = store.get({ name: 'core-2' })
 
-  // A name option
-  const core1 = store.get({ name: 'default' })
-  await toPromises(core1).ready()
+  await Promise.all([core1a.ready(), core1b.ready(), core2.ready()])
 
-  {
-    // Buffer arg
-    const core = store.get(core1.key)
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
+  t.alike(core1a.key, core1b.key)
+  t.unlike(core1a.key, core2.key)
 
-  {
-    // String arg
-    const core = store.get(core1.key.toString('hex'))
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
+  t.ok(core1a.writable)
+  t.ok(core1b.writable)
 
-  {
-    // Object arg
-    const core = store.get({ key: core1.key })
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
-
-  {
-    // Object arg with string key
-    const core = store.get({ key: core1.key.toString('hex') })
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
-
-  {
-    // Custom keypair
-    const core = store.get({ keyPair: { secretKey: core1.secretKey, publicKey: core1.key } })
-    await toPromises(core).ready()
-    t.same(core, core1)
-  }
-
-  t.end()
+  t.is(store.cores.size, 2)
 })
 
-test('ram-based corestore, unacceptable get options', async t => {
-  const store = create(ram)
-  const badGets = [
-    () => store.get(),
-    () => store.get('abc'),
-    () => store.get({ name: null }),
-    () => store.get({ key: null })
-  ]
-  for (const get of badGets) {
-    try {
-      get()
-      t.fail('get did not throw correctly')
-    } catch (err) {
-      t.true(err)
-    }
-  }
-})
-
-test('ram-based corestore, many gets before ready', async t => {
-  const store = create(ram)
-  const core1 = store.get({ name: 'core1' })
-  const core2 = store.get({ name: 'core1' })
-  const core3 = store.get({ name: 'core3' })
-
-  await Promise.all([
-    toPromises(core1).ready(),
-    toPromises(core2).ready(),
-    toPromises(core3).ready()
-  ])
+test('basic get with custom keypair', async function (t) {
+  const store = new Corestore(ram)
+  const kp1 = crypto.keyPair()
+  const kp2 = crypto.keyPair()
 
-  t.same(core1, core2)
-  t.notSame(core1, core3)
+  const core1 = store.get(kp1)
+  const core2 = store.get(kp2)
+  await Promise.all([core1.ready(), core2.ready()])
 
-  // At this point, the pre-ready cores should've been moved to the main cache.
-  t.same(store.cache.size, 2)
-
-  t.end()
+  t.alike(core1.key, kp1.publicKey)
+  t.alike(core2.key, kp2.publicKey)
+  t.ok(core1.writable)
+  t.ok(core2.writable)
 })
 
-test('ram-based corestore, closing a ref decrements refcount', async t => {
-  const store = create(ram)
-  const core = toPromises(store.get({ name: 'core1' }))
-  await core.ready()
+test('basic namespaces', async function (t) {
+  const store = new Corestore(ram)
+  const ns1 = store.namespace('ns1')
+  const ns2 = store.namespace('ns2')
+  const ns3 = store.namespace('ns1') // Duplicate namespace
 
-  const cacheEntry = store.cache.entries.get(core.discoveryKey.toString('hex'))
-  t.same(cacheEntry.refs, 1)
+  const core1 = ns1.get({ name: 'main' })
+  const core2 = ns2.get({ name: 'main' })
+  const core3 = ns3.get({ name: 'main' })
+  await Promise.all([core1.ready(), core2.ready(), core3.ready()])
 
-  await core.close()
-  t.same(cacheEntry.refs, 0)
+  t.absent(core1.key.equals(core2.key))
+  t.ok(core1.key.equals(core3.key))
+  t.ok(core1.writable)
+  t.ok(core2.writable)
+  t.ok(core3.writable)
+  t.is(store.cores.size, 2)
 
   t.end()
 })
 
-test('ram-based corestore, simple replication', async t => {
-  const store1 = create(ram)
-  const store2 = create(ram)
+test('basic replication', async function (t) {
+  const store1 = new Corestore(ram)
+  const store2 = new Corestore(ram)
 
-  const core1 = toPromises(store1.get({ name: 'core1', valueEncoding: 'utf-8' }))
+  const core1 = store1.get({ name: 'core-1' })
+  const core2 = store1.get({ name: 'core-2' })
   await core1.append('hello')
-
-  const s1 = store1.replicate(true, { live: true })
-  s1.pipe(store2.replicate(false, { live: true })).pipe(s1)
-
-  const clone1 = toPromises(store2.get({ key: core1.key, valueEncoding: 'utf-8' }))
-  t.same(await clone1.get(0), 'hello')
-
-  const core2 = toPromises(store1.get({ name: 'core2', valueEncoding: 'utf-8' }))
   await core2.append('world')
 
-  const clone2 = toPromises(store2.get({ key: core2.key, valueEncoding: 'utf-8' }))
-  t.same(await clone2.get(0), 'world')
-
-  t.end()
-})
-
-test('ram-based corestore, sparse replication', async t => {
-  const store1 = create(ram, { sparse: true })
-  const store2 = create(ram, { sparse: true })
-
-  const core1 = toPromises(store1.get({ name: 'core1', valueEncoding: 'utf-8' }))
-  await core1.append('hello')
-
-  const s1 = store1.replicate(true, { live: true })
-  s1.pipe(store2.replicate(false, { live: true })).pipe(s1)
-
-  const clone1 = toPromises(store2.get({ key: core1.key, valueEncoding: 'utf-8' }))
-  t.same(await clone1.get(0), 'hello')
-
-  const core2 = toPromises(store1.get({ name: 'core2', valueEncoding: 'utf-8' }))
-  await core2.append('world')
+  const core3 = store2.get({ key: core1.key })
+  const core4 = store2.get({ key: core2.key })
 
-  const clone2 = toPromises(store2.get({ key: core2.key, valueEncoding: 'utf-8' }))
-  t.same(await clone2.get(0), 'world')
+  const s = store1.replicate(true)
+  s.pipe(store2.replicate(false)).pipe(s)
 
-  t.end()
+  t.alike(await core3.get(0), Buffer.from('hello'))
+  t.alike(await core4.get(0), Buffer.from('world'))
 })
 
-test('raf-based corestore, simple replication', async t => {
-  const store1 = create(path => raf(p.join('store1', path)))
-  const store2 = create(path => raf(p.join('store2', path)))
-
-  const core1 = toPromises(store1.get({ name: 'core1', valueEncoding: 'utf-8' }))
-  await core1.append('hello')
-
-  const s1 = store1.replicate(true, { live: true })
-  s1.pipe(store2.replicate(false, { live: true })).pipe(s1)
+test('nested namespaces', async function (t) {
+  const store = new Corestore(ram)
+  const ns1a = store.namespace('ns1').namespace('a')
+  const ns1b = store.namespace('ns1').namespace('b')
 
-  const clone1 = toPromises(store2.get({ key: core1.key, valueEncoding: 'utf-8' }))
-  t.same(await clone1.get(0), 'hello')
+  const core1 = ns1a.get({ name: 'main' })
+  const core2 = ns1b.get({ name: 'main' })
+  await Promise.all([core1.ready(), core2.ready()])
 
-  const core2 = toPromises(store1.get({ name: 'core2', valueEncoding: 'utf-8' }))
-  await core2.append('world')
-
-  const clone2 = toPromises(store2.get({ key: core2.key, valueEncoding: 'utf-8' }))
-  t.same(await clone2.get(0), 'world')
-
-  await cleanup(['store1', 'store2'])
-  t.end()
+  t.not(core1.key.equals(core2.key))
+  t.ok(core1.writable)
+  t.ok(core2.writable)
+  t.is(store.cores.size, 2)
 })
 
-test('raf-based corestore, close and reopen', async t => {
-  let store = create('test-store')
-
-  let core1 = toPromises(store.get({ name: 'core1', valueEncoding: 'utf-8' }))
-  await core1.append('hello')
-
-  t.same(await core1.get(0), 'hello')
-
-  await store.close()
-  store = create('test-store')
-  core1 = toPromises(store.get({ name: 'core1', valueEncoding: 'utf-8' }))
-
-  t.same(await core1.get(0), 'hello')
-
-  await cleanup(['test-store'])
-  t.end()
+test('core uncached when all sessions close', async function (t) {
+  const store = new Corestore(ram)
+  const core1 = store.get({ name: 'main' })
+  await core1.ready()
+  t.is(store.cores.size, 1)
+  await core1.close()
+  t.is(store.cores.size, 0)
 })
 
-test('raf-based corestore, close and reopen with keypair option', async t => {
-  let store = create('test-store')
-  const keyPair = hypercoreCrypto.keyPair()
+test('writable core loaded from name userData', async function (t) {
+  const dir = await tmp.dir({ unsafeCleanup: true })
 
-  let core1 = toPromises(store.get({ keyPair, valueEncoding: 'utf-8' }))
-  await core1.append('hello')
+  let store = new Corestore(dir.path)
+  let core = store.get({ name: 'main' })
+  await core.ready()
+  const key = core.key
 
-  t.same(await core1.get(0), 'hello')
+  t.ok(core.writable)
+  await core.append('hello')
+  t.is(core.length, 1)
 
   await store.close()
-  store = create('test-store')
-  core1 = toPromises(store.get({ keyPair, valueEncoding: 'utf-8' }))
-
-  t.same(await core1.get(0), 'hello')
-
-  await cleanup(['test-store'])
-  t.end()
-})
-
-test('namespace method is equivalent to name array', async t => {
-  const store = create(ram)
-
-  const core1 = store.get({ name: ['a', 'b', 'c'] })
-  const core2 = store.namespace('a').namespace('b').get({ name: 'c' })
-
-  await toPromises(core1).ready()
-  await toPromises(core2).ready()
-
-  t.same(core1, core2)
-  t.end()
-})
-
-test('can backup/restore', async t => {
-  const firstStore = create(ram)
-  const core1 = firstStore.get({ name: 'hello-world' })
-  await toPromises(core1).ready()
-  const manifest = await firstStore.backup()
-
-  const secondStore = await Corestore.restore(manifest, ram)
-  const core2 = secondStore.get({ key: core1.key })
-  await toPromises(core2).ready()
+  store = new Corestore(dir.path)
+  core = store.get(key)
+  await core.ready()
 
-  t.true(core2.writable)
+  t.ok(core.writable)
+  await core.append('world')
+  t.is(core.length, 2)
+  t.alike(await core.get(0), Buffer.from('hello'))
+  t.alike(await core.get(1), Buffer.from('world'))
 
-  t.end()
+  await dir.cleanup()
 })
 
-test('can migrate an old corestore', async t => {
-  const oldStore = new OldCorestore('test-store')
-  await oldStore.ready()
-
-  const oldCores = [
-    oldStore.default(),
-    oldStore.namespace('hello').default(),
-    oldStore.namespace('hello').namespace('world').default(),
-    oldStore.get({ name: 'test-name' }),
-    // Randomly-generated names should also be migrated.
-    oldStore.get()
-  ]
-  await Promise.all(oldCores.map(c => new Promise(resolve => c.ready(resolve))))
-
-  const names = [
-    'default',
-    ['hello'],
-    ['hello', 'world'],
-    'test-name'
-  ]
-  const keys = oldCores.map(c => c.key)
-  await new Promise(resolve => oldStore.close(resolve))
-
-  {
-    const store = create('test-store', {
-      migrationRoot: 'test-store'
-    })
-
-    // Cores are writable when loaded by name
-    for (let i = 0; i < names.length; i++) {
-      const core = store.get({ name: names[i] })
-      await toPromises(core).ready()
-      t.true(core.writable)
-      t.same(core.key, keys[i])
-    }
-
-    await store.close()
-  }
-
-  {
-    // The second time should work without a migration.
-    const store = create('test-store')
+test('storage locking', async function (t) {
+  const dir = await tmp.dir({ unsafeCleanup: true })
 
-    // Cores are writable when loaded by key
-    for (let i = 0; i < keys.length; i++) {
-      const core = store.get({ key: keys[i] })
-      await toPromises(core).ready()
-      t.true(core.writable)
-    }
+  const store1 = new Corestore(dir.path)
+  await store1.ready()
 
-    await store.close()
+  const store2 = new Corestore(dir.path)
+  try {
+    await store2.ready()
+    t.fail('dir should have been locked')
+  } catch {
+    t.pass('dir was locked')
   }
 
-  await cleanup(['test-store'])
-  t.end()
+  await dir.cleanup()
 })
-
-function create (storage, opts) {
-  const store = new Corestore(storage, opts)
-  return store
-}

package/test/helpers/index.js

@@ -1,12 +1,7 @@
-const fs = require('fs')
+const fs = require('fs').promises
 
 async function cleanup (dirs) {
-  return Promise.all(dirs.map(dir => new Promise((resolve, reject) => {
-    fs.rmdir(dir, { recursive: true }, err => {
-      if (err) return reject(err)
-      return resolve()
-    })
-  })))
+  return Promise.allSettled(dirs.map(dir => fs.rmdir(dir, { recursive: true })))
 }
 
 function delay (ms, cb) {

package/test/keys.js

@@ -0,0 +1,65 @@
+const p = require('path')
+const fs = require('fs')
+
+const test = require('brittle')
+const ram = require('random-access-memory')
+const raf = require('random-access-file')
+
+const KeyManager = require('../lib/keys')
+
+test('can create hypercore keypairs', async t => {
+  const keys = await KeyManager.fromStorage(ram)
+
+  const kp1 = await keys.createHypercoreKeyPair('core1')
+  const kp2 = await keys.createHypercoreKeyPair('core2')
+
+  t.is(kp1.publicKey.length, 32)
+  t.is(kp2.publicKey.length, 32)
+  t.unlike(kp1.publicKey, kp2.publicKey)
+})
+
+test('distinct tokens create distinct hypercore keypairs', async t => {
+  const keys = await KeyManager.fromStorage(ram)
+  const token1 = KeyManager.createToken()
+  const token2 = KeyManager.createToken()
+
+  const kp1 = await keys.createHypercoreKeyPair('core1', token1)
+  const kp2 = await keys.createHypercoreKeyPair('core1', token2)
+
+  t.unlike(kp1.publicKey, kp2.publicKey)
+})
+
+test('short user-provided token will throw', async t => {
+  const keys = await KeyManager.fromStorage(ram)
+
+  try {
+    await keys.createHypercoreKeyPair('core1', Buffer.from('hello'))
+    t.fail('did not throw')
+  } catch {
+    t.pass('threw correctly')
+  }
+})
+
+test('persistent storage regenerates keys correctly', async t => {
+  const testPath = p.resolve(__dirname, 'test-data')
+
+  const keys1 = await KeyManager.fromStorage((name) => raf(testPath, { directory: testPath }))
+  const kp1 = await keys1.createHypercoreKeyPair('core1')
+
+  const keys2 = await KeyManager.fromStorage((name) => raf(testPath, { directory: testPath }))
+  const kp2 = await keys2.createHypercoreKeyPair('core1')
+
+  t.alike(kp1.publicKey, kp2.publicKey)
+
+  await fs.promises.rm(testPath, { recursive: true })
+})
+
+test('different master keys -> different keys', async t => {
+  const keys1 = await KeyManager.fromStorage(ram)
+  const keys2 = await KeyManager.fromStorage(ram)
+
+  const kp1 = await keys1.createHypercoreKeyPair('core1')
+  const kp2 = await keys2.createHypercoreKeyPair('core1')
+
+  t.unlike(kp1.publicKey, kp2.publicKey)
+})

package/lib/buffer-file.js

@@ -1,26 +0,0 @@
-const RAS = require('random-access-storage')
-
-module.exports = class BufferFile extends RAS {
-  constructor (buf) {
-    super()
-    this._buf = buf
-  }
-
-  _open (req) {
-    if (typeof this._buf !== 'function') return req.callback(null)
-    this._buf((err, b) => {
-      if (err) return req.callback(err)
-      this._buf = b
-      req.callback(null)
-    })
-  }
-
-  _stat (req) {
-    req.callback(null, { size: this._buf ? this._buf.length : 0 })
-  }
-
-  _read (req) {
-    if (!this._buf) return req.callback(new Error('Could not satisfy length'))
-    req.callback(null, this._buf.slice(req.offset, req.offset + req.size))
-  }
-}

package/lib/db.js

@@ -1,160 +0,0 @@
-const p = require('path')
-const fs = require('fs').promises
-
-const varint = require('varint')
-const Hyperbee = require('hyperbee')
-const hypercore = require('hypercore')
-const hypercoreCrypto = require('hypercore-crypto')
-const readdir = require('@jsdevtools/readdir-enhanced')
-const { NanoresourcePromise: Nanoresource } = require('nanoresource-promise/emitter')
-
-const INDEX_PATH = 'index'
-const INDEX_VERSION = '@corestore/v1'
-const MIGRATION_KEY = '@migrated'
-const KEYS_NAMESPACE = 'by-key'
-const DKEYS_NAMESPACE = 'by-dkey'
-
-module.exports = class Index extends Nanoresource {
-  constructor (storage, opts = {}) {
-    super()
-
-    this.storage = storage
-    this._migrationRoot = opts.migrationRoot
-    this._core = hypercore(p => this.storage(INDEX_PATH + '/' + p))
-    this._db = new Hyperbee(this._core, {
-      keyEncoding: 'utf-8',
-      valueEncoding: 'json'
-    }).sub(INDEX_VERSION)
-
-    this._byKey = this._db.sub(KEYS_NAMESPACE)
-    this._byDKey = this._db.sub(DKEYS_NAMESPACE)
-
-    this._core.on('error', err => this.emit('error', err))
-  }
-
-  // Nanoresource Methods
-
-  async _open () {
-    await new Promise((resolve, reject) => {
-      this._core.open(err => {
-        if (err) return reject(err)
-        return resolve()
-      })
-    })
-    return this._maybeMigrate()
-  }
-
-  _close () {
-    return new Promise((resolve, reject) => {
-      this._core.close(err => {
-        if (err) return reject(err)
-        return resolve()
-      })
-    })
-  }
-
-  // Private Methods
-
-  async _maybeMigrate () {
-    if (!this._migrationRoot) return
-    const isMigrated = await this._db.get(MIGRATION_KEY)
-    if (isMigrated) return
-
-    const basePath = p.resolve(this._migrationRoot)
-    const nameIterator = readdir.iterator(basePath, {
-      basePath,
-      deep: true,
-      filter: /.*name$/
-    })
-
-    const b = this._db.batch()
-    await b.put(MIGRATION_KEY, { migrated: true })
-
-    for await (const namePath of nameIterator) {
-      let name = await fs.readFile(namePath)
-      if (!varint.decode(name)) continue
-      name = name.slice(varint.decode.bytes).toString('utf-8')
-
-      const keyPath = p.join(p.dirname(namePath), 'key')
-      const publicKey = await loadKey(keyPath)
-      if (!publicKey) continue
-
-      const discoveryKey = hypercoreCrypto.discoveryKey(publicKey)
-      await this._putCoreBatch({
-        name,
-        publicKey,
-        discoveryKey
-      }, b)
-    }
-
-    return b.flush()
-
-    async function loadKey (path) {
-      try {
-        const key = await fs.readFile(path)
-        return key
-      } catch (err) {
-        if (err.code !== 'ENOENT') throw err
-        return null
-      }
-    }
-  }
-
-  async _putCoreBatch (keys, batch) {
-    const record = {
-      name: keys.name,
-      publicKey: toString(keys.publicKey),
-      discoveryKey: toString(keys.discoveryKey)
-    }
-    const b = batch || this._db.batch()
-    await b.put(KEYS_NAMESPACE + this._db.sep + record.publicKey, record)
-    await b.put(DKEYS_NAMESPACE + this._db.sep + record.discoveryKey, record)
-    if (!batch) await b.flush()
-  }
-
-  // Public Methods
-
-  async getName (keys) {
-    const node = await this._byKey.get(toString(keys.publicKey))
-    return node && node.value.name
-  }
-
-  async getPassiveCoreKeys (dkey) {
-    const node = await this._byDKey.get(toString(dkey))
-    return node && toKeys(node.value)
-  }
-
-  async getAllCores () {
-    const allCores = []
-    for await (const { value } of this._byKey.createReadStream()) {
-      allCores.push(value)
-    }
-    return allCores
-  }
-
-  async saveKeys (keys) {
-    const existing = await this.getName(keys)
-    if (existing) return
-    return this._putCoreBatch(keys)
-  }
-
-  async restore (manifest) {
-    if (!manifest.cores || !Array.isArray(manifest.cores)) throw new Error('Malformed manifest.')
-    for (const keys of manifest.cores) {
-      await this.saveKeys(keys)
-    }
-  }
-}
-
-function toString (buf) {
-  if (typeof buf === 'string') return buf
-  return buf.toString('hex')
-}
-
-function toKeys (record) {
-  return {
-    name: record.name,
-    publicKey: Buffer.from(record.publicKey, 'hex'),
-    discoveryKey: Buffer.from(record.discoveryKey, 'hex')
-  }
-}