corestore 6.18.4 → 7.0.1

package/README.md

@@ -13,6 +13,11 @@ Corestore provides:
 ### Installation
 `npm install corestore`
 
+> [!NOTE]
+> This readme reflects Corestore 7, our latest major version that is backed by RocksDB for storage and atomicity.
+> Whilst we are fully validating that, the npm dist-tag for latest is set to latest version of Corestore 7, the previous major, to avoid too much disruption.
+> It will be updated to 11 in a few weeks.
+
 ### Usage
 A corestore instance can be constructed with a random-access-storage module, a function that returns a random-access-storage module given a path, or a string. If a string is specified, it will be assumed to be a path to a local storage directory:
 ```js
@@ -24,27 +29,17 @@ const core2 = store.get({ name: 'core-2' })
 ```
 
 ### API
-#### `const store = new Corestore(storage, opts = {})`
+#### `const store = new Corestore(storage)`
 Create a new Corestore instance.
 
 `storage` can be either a random-access-storage module, a string, or a function that takes a path and returns an random-access-storage instance.
 
-Opts include:
-```js
-{
-  inflightRange: null // Advanced option. Forwarded to the Hypercores created by corestore.get(...)
-}
-```
-
-#### `const core = store.get(key | { name: 'a-name', exclusive, ...hypercoreOpts})`
+#### `const core = store.get(key | { name: 'a-name', ...hypercoreOpts})`
 Loads a Hypercore, either by name (if the `name` option is provided), or from the provided key (if the first argument is a Buffer or String with hex/z32 key, or if the `key` options is set).
 
 If that Hypercore has previously been loaded, subsequent calls to `get` will return a new Hypercore session on the existing core.
 
-If you set the `exclusive` option and you are opening a writable session it will wait for all other exclusive writable to close before
-opening the Hypercore effectively meaning any op on the core will wait until its exclusive.
-
-All other options besides `name` and `key` and `exclusive` will be forwarded to the Hypercore constructor.
+All other options besides `name` and `key` will be forwarded to the Hypercore constructor.
 
 #### `const stream = store.replicate(optsOrStream)`
 Creates a replication stream that's capable of replicating all Hypercores that are managed by the Corestore, assuming the remote peer has the correct capabilities.
@@ -67,8 +62,11 @@ swarm.join(...)
 swarm.on('connection', (connection) => store.replicate(connection))
 ```
 
+#### `const storeB = storeA.session()`
+Create a new Corestore session. Closing a session will close all cores made from this session.
+
 #### `const store = store.namespace(name)`
-Create a new namespaced Corestore. Namespacing is useful if you're going to be sharing a single Corestore instance between many applications or components, as it prevents name collisions.
+Create a new namespaced Corestore session. Namespacing is useful if you're going to be sharing a single Corestore instance between many applications or components, as it prevents name collisions.
 
 Namespaces can be chained:
 ```js
@@ -78,32 +76,8 @@ const core1 = ns1.get({ name: 'main' }) // These will load different Hypercores
 const core2 = ns2.get({ name: 'main' })
 ```
 
-#### `const storeB = storeA.session(opts)`
-Create a new Corestore that shares resources with the original, like cache, cores, replication streams, and storage, while optionally resetting the namespace, overriding `primaryKey`.
-Useful when an application wants to accept an optional Corestore, but needs to maintain a predictable key derivation.
-
-`opts` are the same as the constructor options: 
-
-```js
-{
-  primaryKey, // Overrides the primaryKey for this session
-  namespace, // If set to null it will reset to the DEFAULT_NAMESPACE
-  detach: true, // By disabling this, closing the session will also close the store that created the session
-  inflightRange: null // Advanced option. Forwarded to the Hypercores created by `session.get(...)
-}
-```
-
 #### `await store.close()`
 Fully close this Corestore instance.
 
-#### `store.on('core-open', core)`
-Emitted when the first session for a core is opened.
-
-*Note: This core may close at any time, so treat it as a weak reference*
-
-#### `store.on('core-close', core)`
-Emitted when the last session for a core is closed.
-
 ### License
 MIT
-

package/index.js

@@ -1,562 +1,415 @@
-const safetyCatch = require('safety-catch')
-const crypto = require('hypercore-crypto')
-const sodium = require('sodium-universal')
-const Hypercore = require('hypercore')
-const hypercoreId = require('hypercore-id-encoding')
-const Xache = require('xache')
 const b4a = require('b4a')
+const Hypercore = require('hypercore')
 const ReadyResource = require('ready-resource')
-const RW = require('read-write-mutexify')
+const EventEmitter = require('events')
+const sodium = require('sodium-universal')
+const crypto = require('hypercore-crypto')
+const ID = require('hypercore-id-encoding')
 
 const [NS] = crypto.namespace('corestore', 1)
 const DEFAULT_NAMESPACE = b4a.alloc(32) // This is meant to be 32 0-bytes
 
-const CORES_DIR = 'cores'
-const PRIMARY_KEY_FILE_NAME = 'primary-key'
-const USERDATA_NAME_KEY = 'corestore/name'
-const USERDATA_NAMESPACE_KEY = 'corestore/namespace'
-const POOL_SIZE = 512 // how many open fds to aim for before cycling them
-const DEFAULT_MANIFEST = 0 // bump to 1 when this is more widely deployed
-const DEFAULT_COMPAT = true
-
-module.exports = class Corestore extends ReadyResource {
-  constructor (storage, opts = {}) {
-    super()
-
-    const root = opts._root
-
-    this.storage = Hypercore.defaultStorage(storage, { lock: PRIMARY_KEY_FILE_NAME, poolSize: opts.poolSize || POOL_SIZE, rmdir: true })
-    this.cores = root ? root.cores : new Map()
-    this.cache = !!opts.cache
-    this.primaryKey = opts.primaryKey || null
-    this.passive = !!opts.passive
-    this.manifestVersion = typeof opts.manifestVersion === 'number' ? opts.manifestVersion : (root ? root.manifestVersion : DEFAULT_MANIFEST)
-    this.compat = typeof opts.compat === 'boolean' ? opts.compat : (root ? root.compat : DEFAULT_COMPAT)
-    this.inflightRange = opts.inflightRange || null
-    this.globalCache = opts.globalCache || null
-
-    this._keyStorage = null
-    this._bootstrap = opts._bootstrap || null
-    this._namespace = opts.namespace || DEFAULT_NAMESPACE
-    this._noCoreCache = root ? root._noCoreCache : new Xache({ maxSize: 65536 })
-
-    this._root = root || this
-    this._replicationStreams = root ? root._replicationStreams : []
-    this._overwrite = opts.overwrite === true
-    this._readonly = opts.writable === false
-    this._attached = opts._attached || null
-    this._notDownloadingLinger = opts.notDownloadingLinger
-
-    this._sessions = new Set() // sessions for THIS namespace
-    this._rootStoreSessions = new Set()
-    this._locks = root ? root._locks : new Map()
+class StreamTracker {
+  constructor () {
+    this.records = []
+  }
 
-    this._findingPeersCount = 0
-    this._findingPeers = []
-    this._isCorestore = true
+  add (stream, isExternal) {
+    const record = { index: 0, stream, isExternal }
+    this.records.push(record)
+    return record
+  }
 
-    if (this._namespace.byteLength !== 32) throw new Error('Namespace must be a 32-byte Buffer or Uint8Array')
-    this.ready().catch(safetyCatch)
+  remove (record) {
+    const popped = this.records.pop()
+    if (popped === record) return
+    this.records[popped.index = record.index] = popped
   }
 
-  static isCorestore (obj) {
-    return !!(typeof obj === 'object' && obj && obj._isCorestore)
+  attachAll (core) {
+    for (let i = 0; i < this.records.length; i++) {
+      const record = this.records[i]
+      const muxer = record.stream.noiseStream.userData
+      if (!core.replicator.attached(muxer)) core.replicator.attachTo(muxer)
+    }
   }
 
-  static from (storage, opts) {
-    return this.isCorestore(storage) ? storage : new this(storage, opts)
+  destroy () {
+    // reverse is safer cause we delete mb
+    for (let i = this.records.length - 1; i >= 0; i--) {
+      const record = this.records[i]
+      if (!record.isExternal) record.stream.destroy()
+    }
   }
+}
 
-  // for now just release the lock...
-  async suspend () {
-    if (this._root !== this) return this._root.suspend()
+class SessionTracker {
+  constructor () {
+    this.map = new Map()
+  }
 
-    await this.ready()
+  get size () {
+    return this.map.size
+  }
 
-    if (this._keyStorage !== null) {
-      await new Promise((resolve, reject) => {
-        this._keyStorage.suspend((err) => {
-          if (err) return reject(err)
-          resolve()
-        })
-      })
-    }
+  get (id) {
+    const existing = this.map.get(id)
+    if (existing !== undefined) return existing
+    const fresh = []
+    this.map.set(id, fresh)
+    return fresh
   }
 
-  async resume () {
-    if (this._root !== this) return this._root.resume()
+  gc (id) {
+    this.map.delete(id)
+  }
 
-    await this.ready()
+  list (id) {
+    return id ? (this.map.get(id) || []) : [...this]
+  }
 
-    if (this._keyStorage !== null) {
-      await new Promise((resolve, reject) => {
-        this._keyStorage.open((err) => {
-          if (err) return reject(err)
-          resolve()
-        })
-      })
+  * [Symbol.iterator] () {
+    for (const sessions of this.map.values()) {
+      yield * sessions[Symbol.iterator]()
     }
   }
+}
 
-  findingPeers () {
-    let done = false
-    this._incFindingPeers()
+class CoreTracker extends EventEmitter {
+  constructor () {
+    super()
+    this.map = new Map()
+    this.watching = []
+  }
 
-    return () => {
-      if (done) return
-      done = true
-      this._decFindingPeers()
-    }
+  get size () {
+    return this.map.size
   }
 
-  _emitCore (name, core) {
-    this.emit(name, core)
-    for (const session of this._root._rootStoreSessions) {
-      if (session !== this) {
-        session.emit(name, core)
-      }
-    }
-    if (this !== this._root) this._root.emit(name, core)
+  watch (store) {
+    if (store.watchIndex !== -1) return
+    store.watchIndex = this.watching.push(store) - 1
   }
 
-  _incFindingPeers () {
-    if (++this._findingPeersCount !== 1) return
+  unwatch (store) {
+    if (store.watchIndex === -1) return
+    const head = this.watching.pop()
+    if (head !== store) this.watching[(head.watchIndex = store.watchIndex)] = head
+    store.watchIndex = -1
+  }
 
-    for (const core of this._sessions) {
-      this._findingPeers.push(core.findingPeers())
-    }
+  get (id) {
+    // we allow you do call this from the outside, so support normal buffers also
+    if (b4a.isBuffer(id)) id = b4a.toString(id, 'hex')
+    return this.map.get(id) || null
   }
 
-  _decFindingPeers () {
-    if (--this._findingPeersCount !== 0) return
+  set (id, core) {
+    this.map.set(id, core)
+    this.emit('add', core) // TODO: will be removed
+    if (this.watching.length > 0) this._emit(core)
+  }
 
-    while (this._findingPeers.length > 0) {
-      this._findingPeers.pop()()
+  _emit (core) {
+    for (let i = this.watching.length - 1; i >= 0; i--) {
+      const store = this.watching[i]
+      for (const fn of store.watchers) fn(core)
     }
   }
 
-  async _openNamespaceFromBootstrap () {
-    const ns = await this._bootstrap.getUserData(USERDATA_NAMESPACE_KEY)
-    if (ns) {
-      this._namespace = ns
-    }
+  delete (id, core) {
+    this.map.delete(id)
+    this.emit('remove', core) // TODO: will be removed
   }
 
-  async _open () {
-    if (this._root !== this) {
-      await this._root.ready()
-      if (!this.primaryKey) this.primaryKey = this._root.primaryKey
-      if (this._bootstrap) await this._openNamespaceFromBootstrap()
-      return
-    }
+  [Symbol.iterator] () {
+    return this.map.values()
+  }
+}
 
-    this._keyStorage = this.storage(PRIMARY_KEY_FILE_NAME)
-
-    this.primaryKey = await new Promise((resolve, reject) => {
-      this._keyStorage.stat((err, st) => {
-        if (err && err.code !== 'ENOENT') return reject(err)
-        if (err || st.size < 32 || this._overwrite) {
-          const key = this.primaryKey || crypto.randomBytes(32)
-          return this._keyStorage.write(0, key, err => {
-            if (err) return reject(err)
-            return resolve(key)
-          })
-        }
-        this._keyStorage.read(0, 32, (err, key) => {
-          if (err) return reject(err)
-          if (this.primaryKey) return resolve(this.primaryKey)
-          return resolve(key)
-        })
-      })
-    })
+class Corestore extends ReadyResource {
+  constructor (storage, opts = {}) {
+    super()
 
-    if (this._bootstrap) await this._openNamespaceFromBootstrap()
-  }
+    this.root = opts.root || null
+    this.storage = this.root ? this.root.storage : Hypercore.defaultStorage(storage)
+    this.streamTracker = this.root ? this.root.streamTracker : new StreamTracker()
+    this.cores = this.root ? this.root.cores : new CoreTracker()
+    this.sessions = new SessionTracker()
+    this.globalCache = this.root ? this.root.globalCache : (opts.globalCache || null)
+    this.primaryKey = this.root ? this.root.primaryKey : (opts.primaryKey || null)
+    this.ns = opts.namespace || DEFAULT_NAMESPACE
 
-  async _exists (discoveryKey) {
-    const id = b4a.toString(discoveryKey, 'hex')
-    const storageRoot = getStorageRoot(id)
+    this.watchers = null
+    this.watchIndex = -1
 
-    const st = this.storage(storageRoot + '/oplog')
+    this.manifestVersion = 1 // just compat
 
-    const exists = await new Promise((resolve) => st.stat((err, st) => resolve(!err && st.size > 0)))
-    await new Promise(resolve => st.close(resolve))
+    this._ongcBound = this._ongc.bind(this)
 
-    return exists
+    this.ready().catch(noop)
   }
 
-  async _generateKeys (opts) {
-    if (opts._discoveryKey) {
-      return {
-        manifest: null,
-        keyPair: null,
-        key: null,
-        discoveryKey: opts._discoveryKey
-      }
+  watch (fn) {
+    if (this.watchers === null) {
+      this.watchers = new Set()
+      this.cores.watch(this)
     }
 
-    const keyPair = opts.name
-      ? await this.createKeyPair(opts.name)
-      : (opts.secretKey)
-          ? { secretKey: opts.secretKey, publicKey: opts.publicKey }
-          : null
+    this.watchers.add(fn)
+  }
 
-    if (opts.manifest) {
-      const key = Hypercore.key(opts.manifest)
+  unwatch (fn) {
+    if (this.watchers === null) return
 
-      return {
-        manifest: opts.manifest,
-        keyPair,
-        key,
-        discoveryKey: crypto.discoveryKey(key)
-      }
-    }
+    this.watchers.delete(fn)
 
-    if (opts.key) {
-      return {
-        manifest: null,
-        keyPair,
-        key: opts.key,
-        discoveryKey: crypto.discoveryKey(opts.key)
-      }
+    if (this.watchers.size === 0) {
+      this.watchers = null
+      this.cores.unwatch(this)
     }
+  }
 
-    const publicKey = opts.publicKey || keyPair.publicKey
+  session (opts) {
+    const root = this.root || this
+    return new Corestore(null, { ...opts, root })
+  }
 
-    if (opts.compat === false || (opts.compat !== true && !this.compat)) {
-      let manifest = { version: this.manifestVersion, signers: [{ publicKey }] } // default manifest
-      let key = Hypercore.key(manifest)
-      let discoveryKey = crypto.discoveryKey(key)
+  namespace (name, opts) {
+    return this.session({ ...opts, namespace: generateNamespace(this.ns, name) })
+  }
 
-      if (!(await this._exists(discoveryKey)) && manifest.version !== 0) {
-        const manifestV0 = { version: 0, signers: [{ publicKey }] }
-        const keyV0 = Hypercore.key(manifestV0)
-        const discoveryKeyV0 = crypto.discoveryKey(keyV0)
+  _ongc (session) {
+    if (session.sessions.length === 0) this.sessions.gc(session.id)
+  }
 
-        if (await this._exists(discoveryKeyV0)) {
-          manifest = manifestV0
-          key = keyV0
-          discoveryKey = discoveryKeyV0
-        }
-      }
+  async _getOrSetSeed () {
+    const seed = await this.storage.getSeed()
+    if (seed !== null) return seed
+    return await this.storage.setSeed(this.primaryKey || crypto.randomBytes(32))
+  }
 
-      return {
-        manifest,
-        keyPair,
-        key,
-        discoveryKey
-      }
+  async _open () {
+    if (this.root !== null) {
+      if (this.root.opened === false) await this.root.ready()
+      this.primaryKey = this.root.primaryKey
+      return
     }
 
-    return {
-      manifest: null,
-      keyPair,
-      key: publicKey,
-      discoveryKey: crypto.discoveryKey(publicKey)
+    const primaryKey = await this._getOrSetSeed()
+
+    if (this.primaryKey === null) {
+      this.primaryKey = primaryKey
+      return
     }
-  }
 
-  _getPrereadyUserData (core, key) {
-    // Need to manually read the header values before the Hypercore is ready, hence the ugliness.
-    for (const { key: savedKey, value } of core.core.header.userData) {
-      if (key === savedKey) return value
+    if (!b4a.equals(primaryKey, this.primaryKey)) {
+      throw new Error('Another corestore is stored here')
     }
-    return null
   }
 
-  async _preready (core) {
-    const name = this._getPrereadyUserData(core, USERDATA_NAME_KEY)
-    if (!name) return
+  async _close () {
+    const sessions = []
+    const hanging = [...this.sessions]
+    for (const sess of hanging) sessions.push(sess.close())
 
-    const namespace = this._getPrereadyUserData(core, USERDATA_NAMESPACE_KEY)
-    const keyPair = await this.createKeyPair(b4a.toString(name), namespace)
-    core.setKeyPair(keyPair)
-  }
+    if (this.watchers !== null) this.cores.unwatch(this)
 
-  _getLock (id) {
-    let rw = this._locks.get(id)
+    await Promise.all(sessions)
+    if (this.root !== null) return
 
-    if (!rw) {
-      rw = new RW()
-      this._locks.set(id, rw)
-    }
-
-    return rw
+    const cores = []
+    for (const core of this.cores) cores.push(core.close())
+    await Promise.all(cores)
+    await this.storage.close()
   }
 
-  async _preload (id, keys, opts) {
-    const { manifest, keyPair, key } = keys
+  async _attachMaybe (muxer, discoveryKey) {
+    if (this.opened === false) await this.ready()
+    if (this.cores.get(toHex(discoveryKey)) === null && !(await this.storage.has(discoveryKey))) return
 
-    while (this.cores.has(id)) {
-      const existing = this.cores.get(id)
-      if (existing.opened && !existing.closing) return { from: existing, keyPair, manifest, cache: !!opts.cache }
-      if (existing.closing) {
-        await existing.close()
-      } else {
-        await existing.ready().catch(safetyCatch)
-      }
-    }
+    const core = this._getCore(discoveryKey, { createIfMissing: false })
 
-    const hasKeyPair = !!(keyPair && keyPair.secretKey)
-    const userData = {}
-    if (opts.name) {
-      userData[USERDATA_NAME_KEY] = b4a.from(opts.name)
-      userData[USERDATA_NAMESPACE_KEY] = this._namespace
-    }
+    if (!core) return
+    if (!core.opened) await core.ready()
 
-    // No more async ticks allowed after this point -- necessary for caching
-
-    const storageRoot = getStorageRoot(id)
-    const core = new Hypercore(p => this.storage(storageRoot + '/' + p), {
-      _preready: this._preready.bind(this),
-      notDownloadingLinger: this._notDownloadingLinger,
-      inflightRange: this.inflightRange,
-      autoClose: true,
-      active: false,
-      encryptionKey: opts.encryptionKey || null,
-      isBlockKey: !!opts.isBlockKey,
-      userData,
-      manifest,
-      key,
-      compat: opts.compat,
-      cache: opts.cache,
-      globalCache: this.globalCache,
-      createIfMissing: opts.createIfMissing === false ? false : !opts._discoveryKey,
-      keyPair: hasKeyPair ? keyPair : null
-    })
-
-    if (this._root.closing) {
-      try {
-        await core.close()
-      } catch {}
-      throw new Error('The corestore is closed')
+    if (!core.replicator.attached(muxer)) {
+      core.replicator.attachTo(muxer)
     }
+  }
 
-    this.cores.set(id, core)
-    this._noCoreCache.delete(id)
-    core.ready().then(() => {
-      if (core.closing) return // extra safety here as ready is a tick after open
-      if (hasKeyPair) core.setKeyPair(keyPair)
-      this._emitCore('core-open', core)
-      if (this.passive) return
-
-      const ondownloading = () => {
-        for (const { stream } of this._replicationStreams) {
-          core.replicate(stream, { session: true })
-        }
+  replicate (isInitiator, opts) {
+    const isExternal = isStream(isInitiator)
+    const stream = Hypercore.createProtocolStream(isInitiator, {
+      ...opts,
+      ondiscoverykey: discoveryKey => {
+        if (this.closing) return
+        const muxer = stream.noiseStream.userData
+        return this._attachMaybe(muxer, discoveryKey)
       }
-      // when the replicator says we are downloading, answer the call
-      core.replicator.ondownloading = ondownloading
-      // trigger once if the condition is already true
-      if (core.replicator.downloading) ondownloading()
-    }, () => {
-      this._noCoreCache.set(id, true)
-      this.cores.delete(id)
-    })
-    core.once('close', () => {
-      this._emitCore('core-close', core)
-      this.cores.delete(id)
-    })
-    core.on('conflict', (len, fork, proof) => {
-      this.emit('conflict', core, len, fork, proof)
     })
 
-    return { from: core, keyPair, manifest, cache: !!opts.cache }
-  }
+    if (this.cores.size > 0) {
+      const muxer = stream.noiseStream.userData
+      const uncork = muxer.uncork.bind(muxer)
+      muxer.cork()
 
-  async createKeyPair (name, namespace = this._namespace) {
-    if (!this.opened) await this.ready()
+      for (const core of this.cores) {
+        if (!core.replicator.downloading || core.replicator.attached(muxer) || !core.opened) continue
+        core.replicator.attachTo(muxer)
+      }
 
-    const keyPair = {
-      publicKey: b4a.allocUnsafeSlow(sodium.crypto_sign_PUBLICKEYBYTES),
-      secretKey: b4a.alloc(sodium.crypto_sign_SECRETKEYBYTES)
+      stream.noiseStream.opened.then(uncork)
     }
 
-    const seed = deriveSeed(this.primaryKey, namespace, name)
-    sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair.secretKey, seed)
-
-    return keyPair
+    const record = this.streamTracker.add(stream, isExternal)
+    stream.once('close', () => this.streamTracker.remove(record))
+    return stream
   }
 
-  get (opts = {}) {
-    if (this.closing || this._root.closing) throw new Error('The corestore is closed')
-    opts = validateGetOptions(opts)
-
-    if (opts.cache !== false) {
-      opts.cache = opts.cache === true || (this.cache && !opts.cache) ? defaultCache() : opts.cache
-    }
-    if (this._readonly && opts.writable !== false) {
-      opts.writable = false
+  get (opts) {
+    if (b4a.isBuffer(opts) || typeof opts === 'string') opts = { key: opts }
+    if (!opts) opts = {}
+
+    const conf = {
+      preload: null,
+      parent: opts.parent || null,
+      sessions: null,
+      ongc: null,
+      core: null,
+      active: opts.active !== false,
+      encryptionKey: opts.encryptionKey || null,
+      isBlockKey: !!opts.isBlockKey,
+      valueEncoding: opts.valueEncoding || null,
+      exclusive: !!opts.exclusive,
+      manifest: opts.manifest || null,
+      keyPair: opts.keyPair || null,
+      onwait: opts.onwait || null,
+      wait: opts.wait !== false,
+      timeout: opts.timeout || 0,
+      draft: !!opts.draft,
+      writable: opts.writable
     }
 
-    let rw = null
-    let id = null
-
-    const core = new Hypercore(null, {
-      ...opts,
-      globalCache: this.globalCache,
-      name: null,
-      preload: async () => {
-        if (opts.preload) opts = { ...opts, ...(await opts.preload()) }
-        if (!this.opened) await this.ready()
-
-        const keys = await this._generateKeys(opts)
-
-        id = b4a.toString(keys.discoveryKey, 'hex')
-        rw = (opts.exclusive && opts.writable !== false) ? this._getLock(id) : null
-
-        if (rw) await rw.write.lock()
-        return await this._preload(id, keys, opts)
-      }
-    })
-
-    this._sessions.add(core)
-    if (this._findingPeersCount > 0) {
-      this._findingPeers.push(core.findingPeers())
+    // name requires us to rt to storage + ready, so needs preload
+    // same goes if user has defined async preload obvs
+    if (opts.name || opts.preload) {
+      conf.preload = this._preload(opts)
+      return new Hypercore(null, null, conf)
     }
 
-    const gc = () => {
-      // technically better to also clear _findingPeers if we added it,
-      // but the lifecycle for those are pretty short so prob not worth the complexity
-      // as _decFindingPeers clear them all.
-      this._sessions.delete(core)
+    // if not not we can sync create it, which just is easier for the
+    // upstream user in terms of guarantees (key is there etc etc)
+    const core = this._getCore(null, opts)
 
-      if (!rw) return
-      rw.write.unlock()
-      if (!rw.write.locked) this._locks.delete(id)
-    }
+    conf.core = core
+    conf.sessions = this.sessions.get(core.id)
+    conf.ongc = this._ongcBound
 
-    core.ready().catch(gc)
-    core.once('close', gc)
+    return new Hypercore(null, null, conf)
+  }
 
-    return core
+  async createKeyPair (name, ns = this.ns) {
+    if (this.opened === false) await this.ready()
+    return createKeyPair(this.primaryKey, ns, name)
   }
 
-  replicate (isInitiator, opts) {
-    const isExternal = isStream(isInitiator) || !!(opts && opts.stream)
-    const stream = Hypercore.createProtocolStream(isInitiator, {
-      ...opts,
-      ondiscoverykey: async discoveryKey => {
-        if (this.closing) return
+  async _preload (opts) {
+    if (opts.preload) opts = { ...opts, ...(await opts.preload) }
+    if (this.opened === false) await this.ready()
 
-        const id = b4a.toString(discoveryKey, 'hex')
-        if (this._noCoreCache.get(id)) return
+    const discoveryKey = opts.name ? await this.storage.getAlias({ name: opts.name, namespace: this.ns }) : null
+    const core = this._getCore(discoveryKey, opts)
 
-        const core = this.get({ _discoveryKey: discoveryKey, active: false })
+    return {
+      parent: opts.parent || null,
+      core,
+      sessions: this.sessions.get(core.id),
+      ongc: this._ongcBound,
+      encryptionKey: opts.encryptionKey || null,
+      isBlockKey: !!opts.isBlockKey
+    }
+  }
 
-        try {
-          await core.ready()
-        } catch {
-          return
-        }
+  _auth (discoveryKey, opts) {
+    const result = {
+      keyPair: null,
+      key: null,
+      discoveryKey,
+      manifest: null
+    }
 
-        // remote is asking for the core so we HAVE to answer even if not downloading
-        if (!core.closing) core.replicate(stream, { session: true })
-        await core.close()
-      }
-    })
+    if (opts.name) {
+      result.keyPair = createKeyPair(this.primaryKey, this.ns, opts.name)
+    } else if (opts.keyPair) {
+      result.keyPair = opts.keyPair
+    }
 
-    if (!this.passive) {
-      const muxer = stream.noiseStream.userData
-      muxer.cork()
-      for (const core of this.cores.values()) {
-        // If the core is not opened, it will be replicated in preload.
-        if (!core.opened || core.closing || !core.replicator.downloading) continue
-        core.replicate(stream, { session: true })
-      }
-      stream.noiseStream.opened.then(() => muxer.uncork())
+    if (opts.manifest) {
+      result.manifest = opts.manifest
+    } else if (result.keyPair && !result.discoveryKey) {
+      result.manifest = { version: 1, signers: [{ publicKey: result.keyPair.publicKey }] }
     }
 
-    const streamRecord = { stream, isExternal }
-    this._replicationStreams.push(streamRecord)
+    if (opts.key) result.key = ID.decode(opts.key)
+    else if (result.manifest) result.key = Hypercore.key(result.manifest)
 
-    stream.once('close', () => {
-      this._replicationStreams.splice(this._replicationStreams.indexOf(streamRecord), 1)
-    })
+    if (result.discoveryKey) return result
 
-    return stream
+    if (opts.discoveryKey) result.discoveryKey = ID.decode(opts.discoveryKey)
+    else if (result.key) result.discoveryKey = crypto.discoveryKey(result.key)
+    else throw new Error('Could not derive discovery from input')
+
+    return result
   }
 
-  namespace (name, opts) {
-    if (name instanceof Hypercore) {
-      return this.session({ ...opts, _bootstrap: name })
-    }
-    return this.session({ ...opts, namespace: generateNamespace(this._namespace, name) })
+  _hasCore (discoveryKey) {
+    return this.cores.get(toHex(discoveryKey)) !== null
   }
 
-  session (opts) {
-    const session = new Corestore(this.storage, {
-      namespace: this._namespace,
-      cache: this.cache,
-      writable: !this._readonly,
-      _attached: opts && opts.detach === false ? this : null,
-      _root: this._root,
-      inflightRange: this.inflightRange,
-      globalCache: this.globalCache,
-      ...opts
+  _getCore (discoveryKey, opts) {
+    const auth = this._auth(discoveryKey, opts)
+
+    const id = toHex(auth.discoveryKey)
+    const existing = this.cores.get(id)
+    if (existing) return existing
+
+    const core = Hypercore.createCore(this.storage, {
+      eagerUpgrade: true,
+      notDownloadingLinger: opts.notDownloadingLinger,
+      allowFork: opts.allowFork !== false,
+      inflightRange: opts.inflightRange,
+      compat: false, // no compat for now :)
+      force: opts.force,
+      createIfMissing: opts.createIfMissing,
+      discoveryKey: auth.discoveryKey,
+      overwrite: opts.overwrite,
+      key: auth.key,
+      keyPair: auth.keyPair,
+      legacy: opts.legacy,
+      manifest: auth.manifest,
+      globalCache: opts.globalCache || this.globalCache || null,
+      alias: opts.name ? { name: opts.name, namespace: this.ns } : null
     })
-    if (this === this._root) this._rootStoreSessions.add(session)
-    return session
-  }
 
-  _closeNamespace () {
-    const closePromises = []
-    for (const session of this._sessions) {
-      closePromises.push(session.close())
+    core.onidle = () => {
+      core.destroy()
+      this.cores.delete(id, core)
     }
-    return Promise.allSettled(closePromises)
-  }
 
-  async _closePrimaryNamespace () {
-    const closePromises = []
-    // At this point, the primary namespace is closing.
-    for (const { stream, isExternal } of this._replicationStreams) {
-      // Only close streams that were created by the Corestore
-      if (!isExternal) stream.destroy()
+    core.replicator.ondownloading = () => {
+      this.streamTracker.attachAll(core)
     }
-    for (const core of this.cores.values()) {
-      closePromises.push(forceClose(core))
-    }
-    await Promise.allSettled(closePromises)
-    await new Promise((resolve, reject) => {
-      this._keyStorage.close(err => {
-        if (err) return reject(err)
-        return resolve(null)
-      })
-    })
-  }
-
-  async _close () {
-    this._root._rootStoreSessions.delete(this)
 
-    await this._closeNamespace()
-
-    if (this._root === this) {
-      await this._closePrimaryNamespace()
-    } else if (this._attached) {
-      await this._attached.close()
-    }
+    this.cores.set(id, core)
+    return core
   }
 }
 
-function validateGetOptions (opts) {
-  const key = (b4a.isBuffer(opts) || typeof opts === 'string') ? hypercoreId.decode(opts) : null
-  if (key) return { key }
-
-  if (opts.key) {
-    opts.key = hypercoreId.decode(opts.key)
-  }
-  if (opts.keyPair) {
-    opts.publicKey = opts.keyPair.publicKey
-    opts.secretKey = opts.keyPair.secretKey
-  }
+module.exports = Corestore
 
-  if (opts.name && typeof opts.name !== 'string') throw new Error('name option must be a String')
-  if (opts.name && opts.secretKey) throw new Error('Cannot provide both a name and a secret key')
-  if (opts.publicKey && !b4a.isBuffer(opts.publicKey)) throw new Error('publicKey option must be a Buffer or Uint8Array')
-  if (opts.secretKey && !b4a.isBuffer(opts.secretKey)) throw new Error('secretKey option must be a Buffer or Uint8Array')
-  if (!opts._discoveryKey && (!opts.name && !opts.publicKey && !opts.manifest && !opts.key && !opts.preload)) throw new Error('Must provide either a name or a publicKey')
-  return opts
+function isStream (s) {
+  return typeof s === 'object' && s && typeof s.pipe === 'function'
 }
 
 function generateNamespace (namespace, name) {
@@ -573,19 +426,19 @@ function deriveSeed (primaryKey, namespace, name) {
   return out
 }
 
-function defaultCache () {
-  return new Xache({ maxSize: 65536, maxAge: 0 })
-}
-
-function isStream (s) {
-  return typeof s === 'object' && s && typeof s.pipe === 'function'
+function createKeyPair (primaryKey, namespace, name) {
+  const seed = deriveSeed(primaryKey, namespace, name)
+  const buf = b4a.alloc(sodium.crypto_sign_PUBLICKEYBYTES + sodium.crypto_sign_SECRETKEYBYTES)
+  const keyPair = {
+    publicKey: buf.subarray(0, sodium.crypto_sign_PUBLICKEYBYTES),
+    secretKey: buf.subarray(sodium.crypto_sign_PUBLICKEYBYTES)
+  }
+  sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair.secretKey, seed)
+  return keyPair
 }
 
-async function forceClose (core) {
-  await core.ready()
-  return Promise.all(core.sessions.map(s => s.close()))
-}
+function noop () {}
 
-function getStorageRoot (id) {
-  return CORES_DIR + '/' + id.slice(0, 2) + '/' + id.slice(2, 4) + '/' + id
+function toHex (discoveryKey) {
+  return b4a.toString(discoveryKey, 'hex')
 }

package/package.json

@@ -1,44 +1,43 @@
 {
   "name": "corestore",
-  "version": "6.18.4",
+  "version": "7.0.1",
   "description": "A Hypercore factory that simplifies managing collections of cores.",
   "main": "index.js",
+  "files": [
+    "index.js"
+  ],
+  "dependencies": {
+    "b4a": "^1.6.7",
+    "bare-events": "^2.5.0",
+    "hypercore": "^11.0.0",
+    "hypercore-crypto": "^3.4.2",
+    "hypercore-id-encoding": "^1.3.0",
+    "ready-resource": "^1.1.1",
+    "sodium-universal": "^4.0.1"
+  },
+  "devDependencies": {
+    "brittle": "^3.7.0",
+    "rache": "^1.0.0",
+    "standard": "^17.1.2",
+    "test-tmp": "^1.3.0"
+  },
   "scripts": {
     "test": "standard && brittle test/*.js"
   },
+  "imports": {
+    "events": {
+      "bare": "bare-events",
+      "default": "events"
+    }
+  },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/holepunchto/corestore.git"
+    "url": "https://github.com/holepunchto/corestore2.git"
   },
-  "keywords": [
-    "corestore"
-  ],
-  "author": "Andrew Osheroff <andrewosh@gmail.com>",
+  "author": "Holepunch Inc",
   "license": "MIT",
   "bugs": {
-    "url": "https://github.com/holepunchto/corestore/issues"
+    "url": "https://github.com/holepunchto/corestore2/issues"
   },
-  "homepage": "https://github.com/holepunchto/corestore#readme",
-  "files": [
-    "index.js",
-    "lib/**.js"
-  ],
-  "devDependencies": {
-    "brittle": "^3.2.2",
-    "rache": "^1.0.0",
-    "random-access-memory": "^6.2.0",
-    "standard": "^17.1.0",
-    "test-tmp": "^1.0.2"
-  },
-  "dependencies": {
-    "b4a": "^1.6.4",
-    "hypercore": "^10.37.10",
-    "hypercore-crypto": "^3.4.0",
-    "hypercore-id-encoding": "^1.2.0",
-    "read-write-mutexify": "^2.1.0",
-    "ready-resource": "^1.0.0",
-    "safety-catch": "^1.0.1",
-    "sodium-universal": "^4.0.0",
-    "xache": "^1.1.0"
-  }
+  "homepage": "https://github.com/holepunchto/corestore2"
 }

package/LICENSE

@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2023 Holepunch Inc
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.