hypercore-storage 0.0.40 → 1.0.0

package/index.js

@@ -1,1009 +1,697 @@
 const RocksDB = require('rocksdb-native')
-const c = require('compact-encoding')
-const { UINT } = require('index-encoder')
-const RW = require('read-write-mutexify')
-const b4a = require('b4a')
-const flat = require('flat-tree')
 const rrp = require('resolve-reject-promise')
-const queueTick = require('queue-tick')
-const assert = require('nanoassert')
-
-const m = require('./lib/messages')
-const DependencyStream = require('./lib/dependency-stream')
-const MemoryOverlay = require('./lib/memory-overlay')
-
-const INF = b4a.from([0xff])
-
-// <TL_INFO> = { version, free, total }
-// <TL_LOCAL_SEED> = seed
-// <TL_CORE_INFO><discovery-key-32-bytes> = { version, owner, core, data }
-
-// <core><CORE_MANIFEST>        = { key, manifest? }
-// <core><CORE_LOCAL_SEED>      = seed
-// <core><CORE_ENCRYPTION_KEY>  = encryptionKey // should come later, not important initially
-// <core><CORE_HEAD><data>      = { fork, length, byteLength, signature }
-// <core><CORE_BATCHES><name>   = <data>
-
-// <data><CORE_INFO>            = { version }
-// <data><CORE_UPDATES>         = { contiguousLength, blocks }
-// <data><CORE_DEPENDENCY       = { data, length, roots }
-// <data><CORE_HINTS>           = { reorg } // should come later, not important initially
-// <data><CORE_TREE><index>     = { index, size, hash }
-// <data><CORE_BITFIELD><index> = <4kb buffer>
-// <data><CORE_BLOCKS><index>   = <buffer>
-// <data><CORE_USER_DATA><key>  = <value>
-
-// top level prefixes
-const TL = {
-  STORAGE_INFO: 0,
-  LOCAL_SEED: 1,
-  DKEYS: 2,
-  CORE: 3,
-  DATA: 4,
-  DEFAULT_KEY: 5
-}
-
-// core prefixes
-const CORE = {
-  MANIFEST: 0,
-  LOCAL_SEED: 1,
-  ENCRYPTION_KEY: 2,
-  HEAD: 3,
-  BATCHES: 4
-}
-
-// data prefixes
-const DATA = {
-  INFO: 0,
-  UPDATES: 1,
-  DEPENDENCY: 2,
-  HINTS: 3,
-  TREE: 4,
-  BITFIELD: 5,
-  BLOCK: 6,
-  USER_DATA: 7
-}
-
-const SLAB = {
-  start: 0,
-  end: 65536,
-  buffer: b4a.allocUnsafe(65536)
-}
-
-// PREFIX + BATCH + TYPE + INDEX
-
-class WriteBatch {
-  constructor (storage, write, atom) {
-    this.storage = storage
-    this.write = write
-    this.atom = atom
-  }
-
-  setCoreHead (head) {
-    this.write.tryPut(encodeCoreIndex(this.storage.corePointer, CORE.HEAD, this.storage.dataPointer), c.encode(m.CoreHead, head))
-  }
-
-  setCoreAuth ({ key, manifest }) {
-    this.write.tryPut(encodeCoreIndex(this.storage.corePointer, CORE.MANIFEST), c.encode(m.CoreAuth, { key, manifest }))
-  }
-
-  setBatchPointer (name, pointer) {
-    this.write.tryPut(encodeBatch(this.storage.corePointer, CORE.BATCHES, name), encode(m.DataPointer, pointer))
-  }
+const ScopeLock = require('scope-lock')
+const View = require('./lib/view.js')
+
+const VERSION = 1
+const COLUMN_FAMILY = 'corestore'
+
+const {
+  CorestoreRX,
+  CorestoreTX,
+  CoreTX,
+  CoreRX
+} = require('./lib/tx.js')
+
+const {
+  createCoreStream,
+  createAliasStream,
+  createBlockStream,
+  createBitfieldStream,
+  createUserDataStream,
+  createTreeNodeStream
+} = require('./lib/streams.js')
+
+const EMPTY = new View()
 
-  setDataDependency ({ data, length }) {
-    this.write.tryPut(encodeDataIndex(this.storage.dataPointer, DATA.DEPENDENCY), encode(m.DataDependency, { data, length }))
-  }
-
-  setLocalKeyPair (keyPair) {
-    this.write.tryPut(encodeCoreIndex(this.storage.corePointer, CORE.LOCAL_SEED), encode(m.KeyPair, keyPair))
-  }
-
-  setEncryptionKey (encryptionKey) {
-    this.write.tryPut(encodeCoreIndex(this.storage.corePointer, CORE.ENCRYPTION_KEY), encryptionKey)
-  }
-
-  setDataInfo (info) {
-    if (info.version !== 0) throw new Error('Version > 0 is not supported')
-    this.write.tryPut(encodeDataIndex(this.storage.dataPointer, DATA.INFO), encode(m.DataInfo, info))
+class Atom {
+  constructor (db) {
+    this.db = db
+    this.view = new View()
+    this.flushes = []
   }
 
-  setUserData (key, value) {
-    this.write.tryPut(encodeUserDataIndex(this.storage.dataPointer, DATA.USER_DATA, key), value)
+  onflush (fn) {
+    this.flushes.push(fn)
   }
 
-  putBlock (index, data) {
-    this.write.tryPut(encodeDataIndex(this.storage.dataPointer, DATA.BLOCK, index), data)
-  }
+  async flush () {
+    await View.flush(this.view.changes, this.db)
+    this.view.reset()
 
-  deleteBlock (index) {
-    this.write.tryDelete(encodeDataIndex(this.storage.dataPointer, DATA.BLOCK, index))
-  }
+    const promises = []
+    while (this.flushes.length) promises.push(this.flushes.pop()())
 
-  deleteBlockRange (start, end) {
-    return this._deleteRange(DATA.BLOCK, start, end)
+    await Promise.all(promises)
   }
+}
 
-  putTreeNode (node) {
-    this.write.tryPut(encodeDataIndex(this.storage.dataPointer, DATA.TREE, node.index), encode(m.TreeNode, node))
-  }
+class HypercoreStorage {
+  constructor (store, db, core, view, atomic) {
+    this.store = store
+    this.db = db
+    this.core = core
+    this.view = view
+    this.atomic = atomic
 
-  deleteTreeNode (index) {
-    this.write.tryDelete(encodeDataIndex(this.storage.dataPointer, DATA.TREE, index))
+    this.view.readStart()
   }
 
-  deleteTreeNodeRange (start, end) {
-    return this._deleteRange(DATA.TREE, start, end)
+  get dependencies () {
+    return this.core.dependencies
   }
 
-  putBitfieldPage (index, page) {
-    this.write.tryPut(encodeDataIndex(this.storage.dataPointer, DATA.BITFIELD, index), page)
+  getDependencyLength () {
+    return this.core.dependencies.length
+      ? this.core.dependencies[this.core.dependencies.length - 1].length
+      : -1
   }
 
-  deleteBitfieldPage (index) {
-    this.write.tryDelete(encodeDataIndex(this.storage.dataPointer, DATA.BITFIELD, index))
-  }
+  getDependency (length) {
+    for (let i = this.core.dependencies.length - 1; i >= 0; i--) {
+      const dep = this.core.dependencies[i]
+      if (dep.length < length) return dep
+    }
 
-  deleteBitfieldPageRange (start, end) {
-    return this._deleteRange(DATA.BITFIELD, start, end)
+    return null
   }
 
-  _deleteRange (type, start, end) {
-    const s = encodeDataIndex(this.storage.dataPointer, type, start)
-    const e = encodeDataIndex(this.storage.dataPointer, type, end === -1 ? Infinity : end)
+  // TODO: this might have to be async if the dependents have changed, but prop ok for now
+  updateDependencyLength (length) {
+    const deps = this.core.dependencies
 
-    return this.write.tryDeleteRange(s, e)
-  }
+    for (let i = deps.length - 1; i >= 0; i--) {
+      if (deps[i].length >= length) continue
+      deps[i].length = length
+      this.core.dependencies = deps.slice(0, i + 1)
+      return
+    }
 
-  destroy () {
-    if (this.atom) this.atom.destroy()
-    else this.write.destroy()
+    throw new Error('Dependency not found')
   }
 
-  flush () {
-    if (this.atom) return this.atom.flush()
-    return flushAndDestroy(this.write)
+  get snapshotted () {
+    return this.db._snapshot !== null
   }
-}
 
-class ReadBatch {
-  constructor (storage, read) {
-    this.storage = storage
-    this.read = read
+  snapshot () {
+    return new HypercoreStorage(this.store, this.db.snapshot(), this.core, this.view.snapshot(), this.atomic)
   }
 
-  async getCoreHead () {
-    return this._get(encodeCoreIndex(this.storage.corePointer, CORE.HEAD, this.storage.dataPointer), m.CoreHead)
+  atomize (atom) {
+    return new HypercoreStorage(this.store, this.db.session(), this.core, atom.view, true)
   }
 
-  async getCoreAuth () {
-    return this._get(encodeCoreIndex(this.storage.corePointer, CORE.MANIFEST), m.CoreAuth)
+  atom () {
+    return this.store.atom()
   }
 
-  async getLocalKeyPair () {
-    return this._get(encodeCoreIndex(this.storage.corePointer, CORE.LOCAL_SEED), m.KeyPair)
+  createBlockStream (opts) {
+    return createBlockStream(this.core, this.db, this.view, opts)
   }
 
-  async getEncryptionKey () {
-    return this._get(encodeCoreIndex(this.storage.corePointer, CORE.ENCRYPTION_KEY), null)
+  createTreeNodeStream (opts) {
+    return createTreeNodeStream(this.core, this.db, this.view, opts)
   }
 
-  getDataDependency () {
-    return this._get(encodeDataIndex(this.storage.dataPointer, DATA.DEPENDENCY), m.DataDependency)
+  createBitfieldStream (opts) {
+    return createBitfieldStream(this.core, this.db, this.view, opts)
   }
 
-  getDataInfo (info) {
-    return this._get(encodeDataIndex(this.storage.dataPointer, DATA.INFO), m.DataInfo)
+  createUserDataStream (opts) {
+    return createUserDataStream(this.core, this.db, this.view, opts)
   }
 
-  getUserData (key) {
-    return this._get(encodeUserDataIndex(this.storage.dataPointer, DATA.USER_DATA, key), null)
-  }
+  async resumeSession (name) {
+    const rx = this.read()
+    const existingSessionsPromise = rx.getSessions()
 
-  async hasBlock (index) {
-    return this._has(encodeDataIndex(this.storage.dataPointer, DATA.BLOCK, index))
-  }
+    rx.tryFlush()
+    const existingSessions = await existingSessionsPromise
 
-  async getBlock (index, error) {
-    const dependency = findBlockDependency(this.storage.dependencies, index)
-    const dataPointer = dependency !== null ? dependency : this.storage.dataPointer
+    const sessions = existingSessions || []
+    const session = getBatch(sessions, name, false)
 
-    const key = encodeDataIndex(dataPointer, DATA.BLOCK, index)
-    const block = await this._get(key, null)
+    if (session === null) return null
 
-    if (block === null && error === true) {
-      throw new Error('Node not found: ' + index)
+    const core = {
+      version: this.core.version,
+      corePointer: this.core.corePointer,
+      dataPointer: session.dataPointer,
+      dependencies: []
     }
 
-    return block
-  }
-
-  async hasTreeNode (index) {
-    return this._has(encodeDataIndex(this.storage.dataPointer, DATA.TREE, index))
-  }
-
-  async getTreeNode (index, error) {
-    const dependency = findTreeDependency(this.storage.dependencies, index)
-    const dataPointer = dependency !== null ? dependency : this.storage.dataPointer
+    const coreRx = new CoreRX(core, this.db, this.view)
 
-    const key = encodeDataIndex(dataPointer, DATA.TREE, index)
-    const node = await this._get(key, m.TreeNode)
+    const dependencyPromise = coreRx.getDependency()
+    coreRx.tryFlush()
 
-    if (node === null && error === true) {
-      throw new Error('Node not found: ' + index)
-    }
+    const dependency = await dependencyPromise
+    if (dependency) core.dependencies = this._addDependency(dependency)
 
-    return node
+    return new HypercoreStorage(this.store, this.db.session(), core, this.atomic ? this.view : new View(), this.atomic)
   }
 
-  async getBitfieldPage (index) {
-    const key = encodeDataIndex(this.storage.dataPointer, DATA.BITFIELD, index)
-    return this._get(key, null)
-  }
+  async createSession (name, head, atom) {
+    const rx = this.read(atom)
 
-  async _has (key) {
-    return (await this.read.get(key)) !== null
-  }
+    const existingSessionsPromise = rx.getSessions()
+    const existingHeadPromise = rx.getHead()
 
-  async _get (key, enc) {
-    const buffer = await this.read.get(key)
-    if (buffer === null) return null
+    rx.tryFlush()
 
-    if (enc) return c.decode(enc, buffer)
+    const [existingSessions, existingHead] = await Promise.all([existingSessionsPromise, existingHeadPromise])
+    if (head === null) head = existingHead
 
-    return buffer
-  }
-
-  destroy () {
-    this.read.destroy()
-  }
-
-  flush () {
-    return flushAndDestroy(this.read)
-  }
+    if (existingHead !== null && head.length > existingHead.length) {
+      throw new Error('Invalid head passed, ahead of core')
+    }
 
-  tryFlush () {
-    tryFlushAndDestroy(this.read)
-  }
-}
+    const sessions = existingSessions || []
+    const session = getBatch(sessions, name, true)
 
-class Lock {
-  constructor (mutex) {
-    this.mutex = mutex
-    this.refs = 0
-    this.promise = null
-  }
+    if (session.dataPointer === -1) {
+      session.dataPointer = await this.store._allocData()
+    }
 
-  acquire () {
-    if (this.refs++ === 0) this.promise = this.mutex.lock()
-    return this
-  }
+    const tx = this.write(atom)
 
-  release () {
-    if (--this.refs === 0) return this.mutex.unlock()
-  }
-}
+    tx.setSessions(sessions)
 
-class Atom {
-  constructor (db) {
-    this.db = db
-    this.batch = null
-    this.refs = 0
-    this.destroyed = false
-    this.flushing = null
-    this.resolve = null
-    this.reject = null
+    const length = head === null ? 0 : head.length
+    const core = {
+      version: this.core.version,
+      corePointer: this.core.corePointer,
+      dataPointer: session.dataPointer,
+      dependencies: this._addDependency({ dataPointer: this.core.dataPointer, length })
+    }
 
-    this._executing = null
-    this._waiting = []
-    this._queue = []
-    this._enqueue = (lock, resolve, reject) => this._queue.push({ lock, resolve, reject })
-  }
+    const coreTx = new CoreTX(core, this.db, tx.view, tx.changes)
 
-  enter () {
-    this.refs++
-  }
+    if (length > 0) coreTx.setHead(head)
+    coreTx.setDependency(core.dependencies[core.dependencies.length - 1])
 
-  exit () {
-    if (--this.refs === 0) this._commit()
-  }
+    await tx.flush()
 
-  acquire (mutex) {
-    const lock = this._lock(mutex)
-    return new Promise(this._enqueue.bind(this, lock))
+    return new HypercoreStorage(this.store, this.db.session(), core, atom ? atom.view : this.atomic ? this.view : new View(), !!atom || this.atomic)
   }
 
-  _lock (mutex) {
-    for (const lock of this._waiting) {
-      if (lock.mutex === mutex) return lock.acquire()
+  async createAtomicSession (atom, head) {
+    const length = head === null ? 0 : head.length
+    const core = {
+      version: this.core.version,
+      corePointer: this.core.corePointer,
+      dataPointer: this.core.dataPointer,
+      dependencies: this._addDependency({ dataPointer: this.core.dataPointer, length })
     }
 
-    const lock = new Lock(mutex)
-    this._waiting.push(lock)
-
-    if (this._executing === null) this._executing = this._execute()
-
-    return lock.acquire()
-  }
+    const coreTx = new CoreTX(core, this.db, atom.view, [])
 
-  async _execute () {
-    this.enter()
-    for (const { promise } of this._waiting) await promise
+    if (length > 0) coreTx.setHead(head)
+    coreTx.setDependency(core.dependencies[core.dependencies.length - 1])
 
-    const queue = this._queue
+    await coreTx.flush()
 
-    this._waiting = []
-    this._queue = []
-    this._executing = null
-
-    for (const { lock, resolve, reject } of queue) {
-      if (this.destroyed) reject(new Error('Atomizer destroyed'))
-      else resolve(lock)
-    }
-
-    this._ensureTick() // allow caller to enter
-    this.exit()
-  }
-
-  createBatch () {
-    if (this.refs === 0) this._ensureTick()
-    this.enter()
-    if (this.batch === null) this.batch = this.db.write()
-    return this.batch
-  }
-
-  _ensureTick () {
-    this.enter()
-    queueTick(() => this.exit())
+    return this.atomize(atom)
   }
 
-  async _commit () {
-    if (this.batch === null) return
+  _addDependency (dep) {
+    const deps = []
 
-    const batch = this.batch
-    const resolve = this.resolve
-    const reject = this.reject
+    for (let i = 0; i < this.core.dependencies.length; i++) {
+      const d = this.core.dependencies[i]
 
-    this.batch = null
-    this.flushing = null
-    this.resolve = this.reject = null
+      if (d.length > dep.length) {
+        deps.push({ dataPointer: d.dataPointer, length: dep.length })
+        return deps
+      }
 
-    if (this.destroyed) {
-      this.destroyed = false
-      batch.destroy()
-      if (reject !== null) reject(new Error('Atomic batch destroyed'))
-      return
+      deps.push(d)
     }
 
-    try {
-      await batch.flush()
-    } catch (err) {
-      batch.destroy()
-      reject(err)
-      return
-    }
-
-    batch.destroy()
-    resolve()
-  }
-
-  _createFlushing () {
-    if (this.flushing !== null) return this.flushing
-
-    const { resolve, reject, promise } = rrp()
-
-    this.flushing = promise
-    this.resolve = resolve
-    this.reject = reject
-
-    return this.flushing
-  }
-
-  destroy () {
-    this.destroyed = true
-    this.exit()
-  }
-
-  flush () {
-    const flushing = this._createFlushing()
-    this.exit()
-    return flushing
+    deps.push(dep)
+    return deps
   }
-}
 
-module.exports = class CoreStorage {
-  constructor (dir) {
-    this.db = typeof dir === 'object' ? dir : new RocksDB(dir)
-    this.mutex = new RW()
+  read (atom) {
+    return new CoreRX(this.core, this.db, atom ? atom.view : this.view)
   }
 
-  // just a helper to make tests easier
-  static async clear (dir) {
-    const s = new this(dir)
-    await s.clear()
-    return s
+  write (atom) {
+    return new CoreTX(this.core, this.db, atom ? atom.view : this.atomic ? this.view : null, [])
   }
 
-  async setLocalSeed (seed, overwrite) {
-    if (!overwrite) {
-      const existing = await getLocalSeed(this.db)
-      if (existing) return b4a.equals(existing, seed)
+  close () {
+    if (this.view !== null) {
+      this.view.readStop()
+      this.view = null
     }
 
-    await this.mutex.write.lock()
-
-    try {
-      const b = this.db.write()
-      b.tryPut(b4a.from([TL.LOCAL_SEED]), seed)
-      await flushAndDestroy(b)
-
-      return true
-    } finally {
-      this.mutex.write.unlock()
-    }
-  }
-
-  getLocalSeed () {
-    return getLocalSeed(this.db)
-  }
-
-  info () {
-    return getStorageInfo(this.db)
+    return this.db.close()
   }
+}
 
-  list () {
-    const s = this.db.iterator({
-      gt: b4a.from([TL.DKEYS]),
-      lt: b4a.from([TL.DKEYS + 1])
-    })
-
-    s._readableState.map = mapOnlyDiscoveryKey
-    return s
+class CorestoreStorage {
+  constructor (db) {
+    this.path = typeof db === 'string' ? db : db.path
+    this.rocks = typeof db === 'string' ? new RocksDB(db) : db
+    this.db = createColumnFamily(this.rocks)
+    this.view = null
+    this.enters = 0
+    this.lock = new ScopeLock()
+    this.flushing = null
+    this.version = 0
+    this.migrating = null
   }
 
-  async idle () {
-    if (this.isIdle()) return
-
-    do {
-      await new Promise(setImmediate)
-      await this.db.idle()
-      await new Promise(setImmediate)
-    } while (!this.isIdle())
+  get opened () {
+    return this.db.opened
   }
 
-  isIdle () {
-    return this.db.isIdle()
+  get closed () {
+    return this.db.closed
   }
 
-  ready () {
+  async ready () {
+    if (this.version === 0) await this._migrateStore()
     return this.db.ready()
   }
 
-  close () {
-    return this.db.close()
-  }
-
-  atom () {
-    return new Atom(this.db)
-  }
-
-  async clear () {
-    const b = this.db.write()
-    b.tryDeleteRange(b4a.from([TL.STORAGE_INFO]), INF)
-    await flushAndDestroy(b)
+  static isCoreStorage (db) {
+    return isCorestoreStorage(db)
   }
 
-  async has (discoveryKey) {
-    return !!(await this.db.get(encodeDiscoveryKey(discoveryKey)))
+  static from (db) {
+    if (isCorestoreStorage(db)) return db
+    return new this(db)
   }
 
-  async resume (discoveryKey) {
-    if (!discoveryKey) {
-      discoveryKey = await getDefaultKey(this.db)
-      if (!discoveryKey) return null
+  async _flush () {
+    while (this.enters > 0) {
+      await this.lock.lock()
+      await this.lock.unlock()
     }
-
-    const val = await this.db.get(encodeDiscoveryKey(discoveryKey))
-    if (val === null) return null
-
-    const { core, data } = c.decode(m.CorePointer, val)
-
-    return new HypercoreStorage(this, discoveryKey, core, data, null)
   }
 
-  async create ({ key, manifest, keyPair, encryptionKey, discoveryKey, userData }) {
-    await this.mutex.write.lock()
+  // runs pre any other mutation and read
+  async _migrateStore () {
+    const view = await this._enter()
 
     try {
-      const existing = await this.resume(discoveryKey)
+      if (this.version === VERSION) return
 
-      if (existing) {
-        // todo: verify key/manifest etc.
-        return existing
-      }
+      const rx = new CorestoreRX(this.db, view)
+      const headPromise = rx.getHead()
 
-      if (!key) throw new Error('No key was provided')
+      rx.tryFlush()
+      const head = await headPromise
 
-      let info = await getStorageInfo(this.db)
+      const version = head === null ? 0 : head.version
+      if (version === VERSION) return
 
-      const write = this.db.write()
+      const target = { version: VERSION, dryRun: false }
 
-      if (!info) {
-        write.tryPut(b4a.from([TL.DEFAULT_KEY]), discoveryKey)
-        info = { free: 0, total: 0 }
+      switch (version) {
+        case 0: {
+          await require('./migrations/0').store(this, target)
+          break
+        }
+        default: {
+          throw new Error('Unsupported version: ' + version + ' - you should probably upgrade your dependencies')
+        }
       }
 
-      const core = info.total++
-      const data = info.free++
-
-      write.tryPut(encodeDiscoveryKey(discoveryKey), encode(m.CorePointer, { core, data }))
-      write.tryPut(b4a.from([TL.STORAGE_INFO]), encode(m.StorageInfo, info))
-
-      const storage = new HypercoreStorage(this, discoveryKey, core, data, null)
-      const batch = new WriteBatch(storage, write, null)
-
-      initialiseCoreInfo(batch, { key, manifest, keyPair, encryptionKey })
-      initialiseCoreData(batch, { userData })
-
-      await batch.flush()
-      return storage
+      this.version = VERSION
     } finally {
-      this.mutex.write.unlock()
+      await this._exit()
     }
   }
-}
-
-class HypercoreStorage {
-  constructor (root, discoveryKey, core, data, snapshot) {
-    this.root = root
-    this.db = root.db
-    this.dbSnapshot = snapshot
-    this.dbRead = snapshot || this.db
-    this.mutex = root.mutex
-
-    this.discoveryKey = discoveryKey
-
-    this.dependencies = []
-
-    // pointers
-    this.corePointer = core
-    this.dataPointer = data
-
-    this.destroyed = false
-  }
-
-  get snapshotted () {
-    return this.dbSnapshot !== null
-  }
-
-  atom () {
-    return this.root.atom()
-  }
-
-  dependencyLength () {
-    return this.dependencies.length
-      ? this.dependencies[this.dependencies.length - 1].length
-      : -1
-  }
-
-  async openBatch (name) {
-    const existing = await this.db.get(encodeBatch(this.corePointer, CORE.BATCHES, name))
-    if (!existing) return null
-
-    const storage = new HypercoreStorage(this.root, this.discoveryKey, this.corePointer, this.dataPointer, this.dbSnapshot)
-    const dataPointer = c.decode(m.DataPointer, existing)
-
-    storage.dataPointer = dataPointer
-    storage.dependencies = await addDependencies(this.db, storage.dataPointer, -1)
-
-    return storage
-  }
 
-  async registerBatch (name, head, atom) {
-    await this.mutex.write.lock()
+  // runs pre the core is returned to the user
+  async _migrateCore (core, discoveryKey, locked) {
+    const view = locked ? this.view : await this._enter()
 
-    const storage = new HypercoreStorage(this.root, this.discoveryKey, this.corePointer, this.dataPointer, null)
+    const version = core.core.version
 
     try {
-      const info = await getStorageInfo(this.db)
-      const write = atom ? atom.createBatch() : this.db.write()
-
-      storage.dataPointer = info.free++
+      if (version === VERSION) return
+
+      const target = { version: VERSION, dryRun: false }
+
+      switch (version) {
+        case 0: {
+          await require('./migrations/0').core(core, target)
+          break
+        }
+        default: {
+          throw new Error('Unsupported version: ' + version + ' - you should probably upgrade your dependencies')
+        }
+      }
 
-      write.tryPut(b4a.from([TL.STORAGE_INFO]), encode(m.StorageInfo, info))
+      core.core.version = VERSION
 
-      const batch = new WriteBatch(storage, write, null)
+      if (locked === false) return
 
-      initialiseCoreData(batch)
+      // if its locked, then move the core state into the memview
+      // in case the core is reopened from the memview, pre flush
 
-      batch.setDataDependency({ data: this.dataPointer, length: head.length })
-      batch.setBatchPointer(name, storage.dataPointer)
-      if (head.rootHash) batch.setCoreHead(head) // if no root hash its the empty core - no head yet
+      const rx = new CorestoreRX(this.db, EMPTY)
+      const tx = new CorestoreTX(view)
 
-      if (atom) await atom.flush()
-      else await flushAndDestroy(write)
+      const corePromise = rx.getCore(discoveryKey)
+      rx.tryFlush()
 
-      storage.dependencies = await addDependencies(this.db, storage.dataPointer, head.length)
-      return storage
+      tx.putCore(discoveryKey, await corePromise)
+      tx.apply()
     } finally {
-      this.mutex.write.unlock()
+      if (!locked) await this._exit()
     }
   }
 
-  async registerOverlay (head) {
-    const storage = new MemoryOverlay(this)
-    const batch = storage.createWriteBatch()
-
-    batch.setDataDependency({ data: this.dataPointer, length: head.length })
-    if (head.rootHash) batch.setCoreHead(head) // if no root hash its the empty core - no head yet
-
-    await batch.flush()
-
-    return storage
+  async _enter () {
+    this.enters++
+    await this.lock.lock()
+    if (this.view === null) this.view = new View()
+    return this.view
   }
 
-  createMemoryOverlay () {
-    return new MemoryOverlay(this)
-  }
+  async _exit () {
+    this.enters--
 
-  snapshot () {
-    assert(this.destroyed === false)
-    const s = new HypercoreStorage(this.root, this.discoveryKey, this.corePointer, this.dataPointer, this.dbRead.snapshot())
+    if (this.flushing === null) this.flushing = rrp()
+    const flushed = this.flushing.promise
 
-    for (const { data, length } of this.dependencies) s.dependencies.push({ data, length })
+    if (this.enters === 0 || this.view.size() > 128) {
+      try {
+        await View.flush(this.view.changes, this.db)
+        this.flushing.resolve()
+      } catch (err) {
+        this.flushing.reject(err)
+      } finally {
+        this.flushing = null
+        this.view = null
+      }
+    }
 
-    return s
+    this.lock.unlock()
+    return flushed
   }
 
-  findDependency (length) {
-    for (let i = this.dependencies.length - 1; i >= 0; i--) {
-      const dep = this.dependencies[i]
-      if (dep.length < length) return dep
-    }
+  // when used with core catches this isnt transactional for simplicity, HOWEVER, its just a number
+  // so worth the tradeoff
+  async _allocData () {
+    let dataPointer = 0
 
-    return null
-  }
+    const view = await this._enter()
+    const tx = new CorestoreTX(view)
 
-  updateDependencies (length) {
-    const deps = this.dependencies
+    try {
+      const head = await this._getHead(view)
 
-    for (let i = deps.length - 1; i >= 0; i--) {
-      if (deps[i].length < length) {
-        deps[i].length = length
-        this.dependencies = deps.slice(0, i + 1)
-        return
-      }
+      dataPointer = head.allocated.datas++
+
+      tx.setHead(head)
+      tx.apply()
+    } finally {
+      await this._exit()
     }
 
-    throw new Error('Dependency not found')
+    return dataPointer
   }
 
-  createReadBatch (opts) {
-    assert(this.destroyed === false)
+  // exposes here so migrations can easily access the head in an init state
+  async _getHead (view) {
+    const rx = new CorestoreRX(this.db, view)
+    const headPromise = rx.getHead()
+    rx.tryFlush()
 
-    return new ReadBatch(this, this.dbRead.read())
+    const head = await headPromise
+    return head === null ? initStoreHead() : head
   }
 
-  createWriteBatch (atom) {
-    assert(this.destroyed === false)
-
-    if (atom) return new WriteBatch(this, atom.createBatch(), atom)
-
-    return new WriteBatch(this, this.db.write(), null)
+  atom () {
+    return new Atom(this.db)
   }
 
-  // helper for atomic flows
-  async createWriteBatchAndLock (atomizer, lock) {
-    assert(this.destroyed === false)
-    if (atomizer) atomizer.enter()
+  async close () {
+    if (this.db.closed) return
+    await this._flush()
+    await this.db.close()
+    await this.rocks.close()
+  }
 
-    try {
-      await lock.lock()
-    } catch (err) {
-      if (atomizer) atomizer.exit()
-      throw err
-    }
+  async clear () {
+    if (this.version === 0) await this._migrateStore()
 
-    const batch = this.createWriteBatch(atomizer)
-    if (atomizer) atomizer.exit()
+    const view = await this._enter()
+    const tx = new CorestoreTX(view)
 
-    return batch
-  }
+    tx.clear()
+    tx.apply()
 
-  createBlockStream (opts = {}) {
-    assert(this.destroyed === false)
-    return createStream(this, createBlockStream, opts)
+    await this._exit()
   }
 
-  createUserDataStream (opts = {}) {
-    assert(this.destroyed === false)
-
-    const r = encodeUserDataRange(this.dataPointer, DATA.USER_DATA, opts)
-    const s = this.dbRead.iterator(r)
-    s._readableState.map = mapStreamUserData
-    return s
+  createCoreStream () {
+    // TODO: be nice to run the mgiration here also, but too much plumbing atm
+    return createCoreStream(this.db, EMPTY)
   }
 
-  createTreeNodeStream (opts = {}) {
-    assert(this.destroyed === false)
-
-    const r = encodeIndexRange(this.dataPointer, DATA.TREE, opts)
-    const s = this.dbRead.iterator(r)
-    s._readableState.map = mapStreamTreeNode
-    return s
+  createAliasStream (namespace) {
+    // TODO: be nice to run the mgiration here also, but too much plumbing atm
+    return createAliasStream(this.db, EMPTY, namespace)
   }
 
-  createBitfieldPageStream (opts = {}) {
-    assert(this.destroyed === false)
+  async getAlias (alias) {
+    if (this.version === 0) await this._migrateStore()
 
-    const r = encodeIndexRange(this.dataPointer, DATA.BITFIELD, opts)
-    const s = this.dbRead.iterator(r)
-    s._readableState.map = mapStreamBitfieldPage
-    return s
+    const rx = new CorestoreRX(this.db, EMPTY)
+    const discoveryKeyPromise = rx.getCoreByAlias(alias)
+    rx.tryFlush()
+    return discoveryKeyPromise
   }
 
-  async peekLastTreeNode () {
-    assert(this.destroyed === false)
+  async getSeed () {
+    if (this.version === 0) await this._migrateStore()
 
-    const last = await this.dbRead.peek(encodeIndexRange(this.dataPointer, DATA.TREE, { reverse: true }))
-    if (last === null) return null
-    return c.decode(m.TreeNode, last.value)
-  }
+    const rx = new CorestoreRX(this.db, EMPTY)
+    const headPromise = rx.getHead()
 
-  async peekLastBitfieldPage () {
-    assert(this.destroyed === false)
+    rx.tryFlush()
 
-    const last = await this.dbRead.peek(encodeIndexRange(this.dataPointer, DATA.BITFIELD, { reverse: true }))
-    if (last === null) return null
-    return mapStreamBitfieldPage(last)
+    const head = await headPromise
+    return head === null ? null : head.seed
   }
 
-  destroy () {
-    if (this.destroyed) return
-    this.destroyed = true
+  async setSeed (seed, { overwrite = true } = {}) {
+    if (this.version === 0) await this._migrateStore()
 
-    if (this.dbSnapshot) this.dbSnapshot.close().catch(noop)
-    this.dbSnapshot = null
-  }
-}
+    const view = await this._enter()
+    const tx = new CorestoreTX(view)
 
-function createStream (storage, createStreamType, opts) {
-  return storage.dependencies.length === 0
-    ? createStreamType(storage.db, storage.dataPointer, opts)
-    : new DependencyStream(storage, createStreamType, opts)
-}
+    try {
+      const rx = new CorestoreRX(this.db, view)
+      const headPromise = rx.getHead()
 
-function createBlockStream (db, data, opts) {
-  const r = encodeIndexRange(data, DATA.BLOCK, opts)
-  const s = db.iterator(r)
-  s._readableState.map = mapStreamBlock
-  return s
-}
+      rx.tryFlush()
 
-function mapStreamUserData (data) {
-  const state = { start: 0, end: data.key.byteLength, buffer: data.key }
+      const head = (await headPromise) || initStoreHead()
 
-  UINT.decode(state) // TL.DATA
-  UINT.decode(state) // pointer
-  UINT.decode(state) // DATA.USER_DATA
+      if (head.seed === null || overwrite) head.seed = seed
+      tx.setHead(head)
+      tx.apply()
 
-  const key = c.string.decode(state)
+      return head.seed
+    } finally {
+      await this._exit()
+    }
+  }
 
-  if (data.value.byteLength === 0) return null
+  async getDefaultDiscoveryKey () {
+    if (this.version === 0) await this._migrateStore()
 
-  return { key, value: data.value }
-}
+    const rx = new CorestoreRX(this.db, EMPTY)
+    const headPromise = rx.getHead()
 
-function mapStreamTreeNode (data) {
-  return c.decode(m.TreeNode, data.value)
-}
+    rx.tryFlush()
 
-function mapStreamBitfieldPage (data) {
-  const state = { start: 0, end: data.key.byteLength, buffer: data.key }
+    const head = await headPromise
+    return head === null ? null : head.defaultDiscoveryKey
+  }
 
-  UINT.decode(state) // TL.DATA
-  UINT.decode(state) // pointer
-  UINT.decode(state) // DATA.BITFIELD
+  async setDefaultDiscoveryKey (discoveryKey, { overwrite = true } = {}) {
+    if (this.version === 0) await this._migrateStore()
 
-  const index = UINT.decode(state)
+    const view = await this._enter()
+    const tx = new CorestoreTX(view)
 
-  return { index, page: data.value }
-}
+    try {
+      const rx = new CorestoreRX(this.db, view)
+      const headPromise = rx.getHead()
 
-function mapStreamBlock (data) {
-  const state = { start: 0, end: data.key.byteLength, buffer: data.key }
+      rx.tryFlush()
 
-  UINT.decode(state) // TL.DATA
-  UINT.decode(state) // pointer
-  UINT.decode(state) // DATA.BITFIELD
+      const head = (await headPromise) || initStoreHead()
 
-  const index = UINT.decode(state)
-  return { index, value: data.value }
-}
+      if (head.defaultDiscoveryKey === null || overwrite) head.defaultDiscoveryKey = discoveryKey
+      tx.setHead(head)
+      tx.apply()
 
-function mapOnlyDiscoveryKey (data) {
-  return data.key.subarray(1)
-}
+      return head.defaultDiscoveryKey
+    } finally {
+      await this._exit()
+    }
+  }
 
-async function getDefaultKey (db) {
-  return db.get(b4a.from([TL.DEFAULT_KEY]))
-}
+  async has (discoveryKey) {
+    if (this.version === 0) await this._migrateStore()
 
-async function getLocalSeed (db) {
-  return db.get(b4a.from([TL.LOCAL_SEED]))
-}
+    const rx = new CorestoreRX(this.db, EMPTY)
+    const promise = rx.getCore(discoveryKey)
 
-async function getStorageInfo (db) {
-  const value = await db.get(b4a.from([TL.STORAGE_INFO]))
-  if (value === null) return null
-  return c.decode(m.StorageInfo, value)
-}
+    rx.tryFlush()
 
-function ensureSlab (size) {
-  if (SLAB.buffer.byteLength - SLAB.start < size) {
-    SLAB.buffer = b4a.allocUnsafe(SLAB.end)
-    SLAB.start = 0
+    return (await promise) !== null
   }
 
-  return SLAB
-}
+  async resume (discoveryKey) {
+    if (this.version === 0) await this._migrateStore()
 
-function encodeIndexRange (pointer, type, opts) {
-  const bounded = { gt: null, gte: null, lte: null, lt: null, reverse: !!opts.reverse, limit: toLimit(opts.limit) }
+    if (!discoveryKey) {
+      discoveryKey = await this.getDefaultDiscoveryKey()
+      if (!discoveryKey) return null
+    }
 
-  if (opts.gt || opts.gt === 0) bounded.gt = encodeDataIndex(pointer, type, opts.gt)
-  else if (opts.gte) bounded.gte = encodeDataIndex(pointer, type, opts.gte)
-  else bounded.gte = encodeDataIndex(pointer, type, 0)
+    const rx = new CorestoreRX(this.db, EMPTY)
+    const corePromise = rx.getCore(discoveryKey)
 
-  if (opts.lt || opts.lt === 0) bounded.lt = encodeDataIndex(pointer, type, opts.lt)
-  else if (opts.lte) bounded.lte = encodeDataIndex(pointer, type, opts.lte)
-  else bounded.lte = encodeDataIndex(pointer, type, Infinity) // infinity
+    rx.tryFlush()
+    const core = await corePromise
 
-  return bounded
-}
+    if (core === null) return null
+    return this._resumeFromPointers(EMPTY, discoveryKey, false, core)
+  }
 
-function encodeUserDataRange (pointer, type, opts) {
-  const bounded = { gt: null, gte: null, lte: null, lt: null, reverse: !!opts.reverse, limit: toLimit(opts.limit) }
+  async _resumeFromPointers (view, discoveryKey, create, { version, corePointer, dataPointer }) {
+    const core = { version, corePointer, dataPointer, dependencies: [] }
 
-  if (opts.gt || opts.gt === 0) bounded.gt = encodeUserDataIndex(pointer, type, opts.gt)
-  else if (opts.gte) bounded.gte = encodeUserDataIndex(pointer, type, opts.gte)
-  else bounded.gte = encodeDataIndex(pointer, type, 0)
+    while (true) {
+      const rx = new CoreRX({ version, dataPointer, corePointer: 0, dependencies: [] }, this.db, view)
+      const dependencyPromise = rx.getDependency()
+      rx.tryFlush()
+      const dependency = await dependencyPromise
+      if (!dependency) break
+      core.dependencies.push(dependency)
+      dataPointer = dependency.dataPointer
+    }
 
-  if (opts.lt || opts.lt === 0) bounded.lt = encodeUserDataIndex(pointer, type, opts.lt)
-  else if (opts.lte) bounded.lte = encodeUserDataIndex(pointer, type, opts.lte)
-  else bounded.lte = encodeDataIndex(pointer, type, Infinity)
+    const result = new HypercoreStorage(this, this.db.session(), core, EMPTY, false)
 
-  return bounded
-}
+    if (result.core.version === 0) await this._migrateCore(result, discoveryKey, create)
+    return result
+  }
 
-function toLimit (n) {
-  return n === 0 ? 0 : (n || Infinity)
-}
+  // not allowed to throw validation errors as its a shared tx!
+  async _create (view, { key, manifest, keyPair, encryptionKey, discoveryKey, alias, userData }) {
+    const rx = new CorestoreRX(this.db, view)
+    const tx = new CorestoreTX(view)
 
-function encode (encoding, value) {
-  const state = ensureSlab(128)
-  const start = state.start
-  encoding.encode(state, value)
+    const corePromise = rx.getCore(discoveryKey)
+    const headPromise = rx.getHead()
 
-  assert(state.start <= state.end)
+    rx.tryFlush()
 
-  return state.buffer.subarray(start, state.start)
-}
+    let [core, head] = await Promise.all([corePromise, headPromise])
+    if (core) return this._resumeFromPointers(view, discoveryKey, true, core)
 
-function encodeBatch (pointer, type, name) {
-  const end = 128 + name.length
-  const state = { start: 0, end, buffer: b4a.allocUnsafe(end) }
-  const start = state.start
-  UINT.encode(state, TL.CORE)
-  UINT.encode(state, pointer)
-  UINT.encode(state, type)
-  c.string.encode(state, name)
+    if (head === null) head = initStoreHead()
+    if (head.defaultDiscoveryKey === null) head.defaultDiscoveryKey = discoveryKey
 
-  return state.buffer.subarray(start, state.start)
-}
+    const corePointer = head.allocated.cores++
+    const dataPointer = head.allocated.datas++
 
-function encodeCoreIndex (pointer, type, index) {
-  const state = ensureSlab(128)
-  const start = state.start
-  UINT.encode(state, TL.CORE)
-  UINT.encode(state, pointer)
-  UINT.encode(state, type)
-  if (index !== undefined) UINT.encode(state, index)
+    core = { version: VERSION, corePointer, dataPointer, alias }
 
-  return state.buffer.subarray(start, state.start)
-}
+    tx.setHead(head)
+    tx.putCore(discoveryKey, core)
+    if (alias) tx.putCoreByAlias(alias, discoveryKey)
 
-function encodeDataIndex (pointer, type, index) {
-  const state = ensureSlab(128)
-  const start = state.start
-  UINT.encode(state, TL.DATA)
-  UINT.encode(state, pointer)
-  UINT.encode(state, type)
-  if (index !== undefined) UINT.encode(state, index)
+    const ptr = { corePointer, dataPointer, dependencies: [] }
+    const ctx = new CoreTX(ptr, this.db, view, tx.changes)
 
-  return state.buffer.subarray(start, state.start)
-}
+    ctx.setAuth({
+      key,
+      discoveryKey,
+      manifest,
+      keyPair,
+      encryptionKey
+    })
 
-function encodeUserDataIndex (pointer, type, key) {
-  const end = 128 + key.length
-  const state = { start: 0, end, buffer: b4a.allocUnsafe(end) }
-  const start = state.start
-  UINT.encode(state, TL.DATA)
-  UINT.encode(state, pointer)
-  UINT.encode(state, type)
-  c.string.encode(state, key)
+    if (userData) {
+      for (const { key, value } of userData) {
+        ctx.putUserData(key, value)
+      }
+    }
 
-  return state.buffer.subarray(start, state.start)
-}
+    tx.apply()
 
-function encodeDiscoveryKey (discoveryKey) {
-  const state = ensureSlab(128)
-  const start = state.start
-  UINT.encode(state, TL.DKEYS)
-  c.fixed32.encode(state, discoveryKey)
-  return state.buffer.subarray(start, state.start)
-}
+    return new HypercoreStorage(this, this.db.session(), ptr, EMPTY, false)
+  }
 
-async function addDependencies (db, dataPointer, treeLength) {
-  const dependencies = []
+  async create (data) {
+    if (this.version === 0) await this._migrateStore()
 
-  let dep = await db.get(encodeDataIndex(dataPointer, DATA.DEPENDENCY))
-  while (dep) {
-    const { data, length } = c.decode(m.DataDependency, dep)
-    if (treeLength === -1 || length <= treeLength) dependencies.push({ data, length })
+    const view = await this._enter()
 
-    dep = await db.get(encodeDataIndex(data, DATA.DEPENDENCY))
+    try {
+      return await this._create(view, data)
+    } finally {
+      await this._exit()
+    }
   }
-
-  return dependencies
 }
 
-function findBlockDependency (dependencies, index) {
-  for (const { data, length } of dependencies) {
-    if (index < length) return data
-  }
-  return null
-}
+module.exports = CorestoreStorage
 
-function findTreeDependency (dependencies, index) {
-  for (const { data, length } of dependencies) {
-    if (flat.rightSpan(index) <= (length - 1) * 2) return data
+function initStoreHead () {
+  return {
+    version: 0, // cause we wanna run the migration
+    allocated: {
+      datas: 0,
+      cores: 0
+    },
+    seed: null,
+    defaultDiscoveryKey: null
   }
-  return null
-}
-
-function initialiseCoreInfo (db, { key, manifest, keyPair, encryptionKey }) {
-  db.setCoreAuth({ key, manifest })
-  if (keyPair) db.setLocalKeyPair(keyPair)
-  if (encryptionKey) db.setEncryptionKey(encryptionKey)
 }
 
-function initialiseCoreData (db, { userData } = {}) {
-  db.setDataInfo({ version: 0 })
-  if (userData) {
-    for (const { key, value } of userData) {
-      db.setUserData(key, value)
-    }
+function getBatch (sessions, name, alloc) {
+  for (let i = 0; i < sessions.length; i++) {
+    if (sessions[i].name === name) return sessions[i]
   }
-}
 
-function noop () {}
+  if (!alloc) return null
 
-async function tryFlushAndDestroy (batch) {
-  try {
-    await batch.flush()
-  } catch {}
+  const result = { name, dataPointer: -1 }
+  sessions.push(result)
+  return result
+}
 
-  batch.destroy()
+function isCorestoreStorage (s) {
+  return typeof s === 'object' && !!s && typeof s.setDefaultDiscoveryKey === 'function'
 }
 
-async function flushAndDestroy (batch) {
-  try {
-    await batch.flush()
-  } catch (err) {
-    batch.destroy()
-    throw err
-  }
+function createColumnFamily (db) {
+  const col = new RocksDB.ColumnFamily(COLUMN_FAMILY, {
+    // tuning! atm just the default tuning from rocks, TODO: tweak
+    enableBlobFiles: false,
+    minBlobSize: 0,
+    blobFileSize: 0,
+    enableBlobGarbageCollection: true,
+    tableBlockSize: 16384,
+    tableCacheIndexAndFilterBlocks: true,
+    tableFormatVersion: 4
+  })
 
-  batch.destroy()
+  return db.columnFamily(col)
 }