autobee 0.0.0 → 1.0.2

package/lib/writers.js

@@ -0,0 +1,501 @@
+const b4a = require('b4a')
+const encoding = require('./encoding.js')
+const Triggers = require('./triggers.js')
+const { WriterEncryption } = require('autobee-encryption')
+
+const WRITER_PREFETCH = 10
+
+const BATCH_WRITER_SANITY = 1024
+const BATCH_OPTIMISTIC_SANITY = 32
+
+const MAX_FORWARD_TIME_DRIFT = 2 * 3600 * 1000
+
+class ActiveWriters {
+  constructor(auto) {
+    const local = auto.local
+
+    this.auto = auto
+    this.active = new Map()
+    this.triggers = new Triggers()
+    this.localWriter = new Writer(this, auto.system, true, local, b4a.toString(local.key, 'hex'))
+    this.writable = false
+    this.pending = []
+
+    this.localWriter.attach()
+
+    this._bootstrap()
+  }
+
+  [Symbol.iterator]() {
+    return this.active.values()
+  }
+
+  *external() {
+    for (const w of this.active.values()) {
+      if (w !== this.localWriter) yield w
+    }
+  }
+
+  bump() {
+    this.auto._bump()
+  }
+
+  async wakeup(key, length) {
+    const id = b4a.toString(key, 'hex')
+    let w = this.active.get(id)
+
+    if (w === this.localWriter) return
+    if (!w) w = await this.add(key)
+
+    // TODO: signal that we've seen length on w for gc
+
+    this.bump()
+  }
+
+  getLatestLocalOplog() {
+    return this.localWriter.getLatest()
+  }
+
+  appendLocal(value, timestamp, batch, links, optimistic) {
+    return this.localWriter.append(value, timestamp, batch, links, optimistic)
+  }
+
+  async rotateLocalWriter(core) {
+    this.clearLocal()
+    await this.localWriter.detachAndClose()
+
+    this.localWriter = new Writer(this, this.auto.system, true, core, b4a.toString(core.key, 'hex'))
+    this.localWriter.addPending()
+  }
+
+  clearLocal() {
+    return this.localWriter.clear()
+  }
+
+  flushLocal(head) {
+    return this.localWriter.flush(head)
+  }
+
+  async updateLocalState() {
+    const w = this.localWriter
+    const bootstrapping = w.core.length === 0 && b4a.equals(this.auto.key, w.core.key)
+    const info = await this.auto.system.get(this.auto.local.key)
+    const writable = this.writable
+
+    w.weight = info ? info.weight : 0
+    this.writable = info ? !info.isRemoved : bootstrapping
+
+    if (writable === this.writable) return
+
+    this.auto.emit(this.writable ? 'writable' : 'unwritable')
+    this.auto.emit('update')
+  }
+
+  has(id) {
+    return !!this.active.get(id)
+  }
+
+  async add(key) {
+    if (b4a.equals(key, this.auto.local.key)) await this.updateLocalState()
+
+    const id = b4a.toString(key, 'hex')
+    let w = this.active.get(id)
+    if (w) {
+      w.couple() // couple if we weren't ready first time
+      return w
+    }
+
+    const encryption = this.auto.encryptionKey ? new WriterEncryption(this.auto) : null
+
+    const core = this.auto.store.get({
+      key,
+      encryption
+    })
+    await core.ready()
+    await core.setUserData('referrer', this.auto.key)
+
+    w = new Writer(this, this.auto.system, false, core, id)
+    w.attach()
+    w.couple()
+
+    return w
+  }
+
+  async remove(key) {
+    if (b4a.equals(key, this.auto.local.key)) await this.updateLocalState()
+
+    const id = b4a.toString(key, 'hex')
+    const w = this.active.get(id)
+    if (!w) return
+
+    w.detach()
+    w.decouple()
+    if (w !== this.localWriter) await w.close()
+  }
+
+  _bootstrap() {
+    const bootstrap = this.auto.bootstrap
+
+    bootstrap.setEncryption(this.auto._getEncryptionProvider())
+    bootstrap.setActive(true)
+
+    const id = b4a.toString(bootstrap.key, 'hex')
+    let w = this.active.get(id)
+    if (w) {
+      w.couple() // couple if we weren't ready first time
+      return
+    }
+
+    w = new Writer(this, this.auto.system, false, bootstrap, id)
+    w.attach()
+    w.couple()
+  }
+}
+
+class Writer {
+  constructor(writers, system, local, core, id) {
+    this.writers = writers
+    this.system = system
+
+    this.core = core
+    this.id = id
+    this.index = 0
+    this.weight = 0
+    this.pending = null
+    this.waiting = null
+    this.download = null
+
+    this.isPending = false
+    this.isClosed = false
+    this.isAttached = false
+    this.isAdded = false
+    this.isRemoved = false
+    this.isCoupled = false
+
+    this._onchangeBound = this._onchange.bind(this)
+
+    if (!local) this.core.on('append', this._onchangeBound)
+    if (!local) this.core.on('download', this._onchangeBound)
+
+    this.couple()
+  }
+
+  get isIndexer() {
+    return this.weight === 2
+  }
+
+  couple() {
+    if (this.isCoupled) return
+    // if we don't have coupler yet we can't add
+    if (this.writers.auto._wakeup.addCore(this.core)) {
+      this.isCoupled = true
+    }
+  }
+
+  decouple() {
+    if (!this.isCoupled) return
+    // if we don't have coupler yet we can't remove
+    if (this.writers.auto._wakeup.removeCore(this.core)) {
+      this.isCoupled = false
+    }
+  }
+
+  hasReferrals() {
+    const entries = this.writers.triggers.get(this.id)
+    if (!entries) return false
+    // TODO: should implement a system where we KNOW that a future writers IS not removed
+    // to avoid accidental spam, for now kept simple as the algo still runs
+    return true
+  }
+
+  addPending() {
+    if (this.isPending) return
+    this.isPending = true
+    this.index = this.writers.pending.push(this) - 1
+  }
+
+  removePending() {
+    if (!this.isPending) return
+    this.isPending = false
+
+    const head = this.writers.pending.pop()
+    if (head !== this) {
+      this.writers.pending[this.index] = head
+      head.index = this.index
+    }
+  }
+
+  _onchange() {
+    if (this.waiting === null) this.bump()
+  }
+
+  bump() {
+    this.waiting = null
+    if (this.isAttached) this.addPending() // TODO: remove the isAttach guard
+    this.writers.bump()
+  }
+
+  attach() {
+    if (this.isAttached) return
+    this.isAttached = true
+    this.writers.active.set(this.id, this)
+    this.addPending()
+  }
+
+  detach() {
+    if (!this.isAttached) return
+    this.isAttached = false
+    this.download.destroy()
+    this.download = null
+    this.writers.active.delete(this.id)
+    this.removePending()
+  }
+
+  close() {
+    this.isClosed = true
+
+    if (this.waiting) {
+      this.writers.triggers.remove(this.waiting)
+      this.waiting = null
+    }
+
+    this.removePending()
+
+    return this.core.close()
+  }
+
+  detachAndClose() {
+    this.detach()
+    return this.close()
+  }
+
+  async views() {
+    const latest = await this.getLatest()
+    if (!latest || !latest.views) return []
+
+    // signedLength for autobase compat
+    return [
+      { ...latest.views.system, signedLength: latest.views.system.length },
+      { ...latest.views.view, signedLength: latest.views.view.length }
+    ]
+  }
+
+  async getLatest() {
+    if (this.core.length === 0) return null
+    const block = await this.core.get(this.core.length - 1)
+    const oplog = encoding.decodeOplog(block)
+    return oplog
+  }
+
+  async update() {
+    const info = await this.system.get(this.core.key)
+
+    if (!info) {
+      this.isAdded = this.system.isGenesis()
+    } else {
+      this.isAdded = true
+      this.isRemoved = info.isRemoved
+      this.weight = info.weight
+    }
+
+    return info ? info.length : 0
+  }
+
+  async next() {
+    if (this.waiting !== null) return null
+
+    const batch = []
+    const genesis = this.system.isGenesis()
+
+    let length = await this.update()
+    let optimistic = false
+    let timestamp = 0
+
+    if (this.download) this.download.destroy()
+    this.download = this.core.download({ start: length, end: length + WRITER_PREFETCH })
+
+    while (length < this.core.length) {
+      // @todo aim to prefetch when writer added, with read-ahead
+      // download the block if it's not available locally
+      // writer will be added back as pending automatically
+      if (!(await this.core.has(length))) {
+        this.removePending()
+        return null
+      }
+
+      const block = await this.core.get(length++)
+      const oplog = encoding.decodeOplog(block)
+      const node = createNode(this.core, length, this.weight, oplog)
+      const b = oplog.batch
+
+      // auto correct some batch invariants
+      node.timestamp = timestamp = Math.max(node.timestamp, timestamp)
+      node.optimistic = optimistic = optimistic || node.optimistic
+
+      if (!this.isAdded && !node.optimistic && !genesis) {
+        return null
+      }
+
+      // an optimistic writer cannot be genesis
+      if (node.optimistic && genesis) {
+        return null
+      }
+
+      if (node.optimistic && !this.isAdded && b.end >= BATCH_OPTIMISTIC_SANITY) {
+        return null
+      }
+
+      if (node.timestamp - Date.now() > MAX_FORWARD_TIME_DRIFT) {
+        return null
+      }
+
+      if (!node.optimistic && b.start === 0 && b.end > 1 && b.end < BATCH_WRITER_SANITY) {
+        this.core.download({ start: length, end: length + oplog.batch.end })
+      }
+
+      if (!(await this._isLinked(node))) {
+        this.removePending()
+        return null
+      }
+
+      batch.push(node)
+      if (b.end === 0) return batch
+
+      // TODO: mark as dead
+      if (batch.length >= BATCH_WRITER_SANITY) return null
+    }
+
+    // handle local writer
+    if (this.core.writable && this.pending !== null) {
+      for (let i = 0; i < this.pending.length; i++) {
+        const node = this.pending[i]
+
+        node.batch = { start: i, end: this.pending.length - 1 - i }
+
+        if (node.length <= length) continue
+
+        if (!this.writers.writable && !node.optimistic) {
+          this.writers.clearLocal()
+          throw new Error('Not writable')
+        }
+
+        // an optimistic writer cannot be genesis
+        if (node.optimistic && genesis) {
+          return null
+        }
+
+        if (node.optimistic && !this.isAdded && this.pending.length >= BATCH_OPTIMISTIC_SANITY) {
+          return null
+        }
+
+        if (node.timestamp - Date.now() > MAX_FORWARD_TIME_DRIFT) {
+          return null
+        }
+
+        if (!(await this._isLinked(node))) return null
+
+        this.removePending()
+
+        return this.pending.slice(i)
+      }
+    }
+
+    return null
+  }
+
+  notify(batch) {
+    this.writers.triggers.trigger(this.id, batch[batch.length - 1].length)
+  }
+
+  async _isLinked(node) {
+    const promises = new Array(node.links.length)
+
+    for (let i = 0; i < node.links.length; i++) {
+      promises[i] = this.system.has(node.links[i])
+    }
+
+    const has = await Promise.all(promises)
+
+    for (let i = 0; i < node.links.length; i++) {
+      if (has[i]) continue
+      const link = node.links[i]
+      const id = b4a.toString(link.key, 'hex')
+      this.waiting = this.writers.triggers.add(id, link.length, this)
+      this.writers.wakeup(link.key, link.length).catch(noop)
+      return false
+    }
+
+    return true
+  }
+
+  get appending() {
+    return this.pending !== null
+  }
+
+  append(value, timestamp, batch, links, optimistic) {
+    if (this.pending === null) this.pending = []
+
+    const oplog = {
+      version: encoding.OPLOG_VERSION,
+      timestamp,
+      links,
+      batch,
+      views: null,
+      optimistic,
+      value
+    }
+
+    const node = createNode(
+      this.core,
+      this.core.length + this.pending.length + 1,
+      this.weight,
+      oplog
+    )
+
+    this.pending.push(node)
+    this.addPending()
+
+    return node
+  }
+
+  clear() {
+    this.pending = null
+  }
+
+  flush(view) {
+    if (this.pending === null) return Promise.resolve()
+    const buffers = []
+    const s = this.system.bee.head()
+    const flushes = this.system.flushes
+    this.pending[this.pending.length - 1].views = { system: s, flushes, view }
+    for (let i = 0; i < this.pending.length; i++) {
+      const node = this.pending[i]
+      node.batch = { start: i, end: this.pending.length - 1 - i }
+      buffers.push(encoding.encodeOplog(node))
+    }
+    this.pending = null
+    return this.core.append(buffers)
+  }
+}
+
+exports.ActiveWriters = ActiveWriters
+exports.Writer = Writer
+exports.createNode = createNode
+
+function createNode(core, length, weight, oplog) {
+  return {
+    core,
+    key: core.key,
+    length,
+    weight,
+    version: oplog.version,
+    timestamp: oplog.timestamp,
+    links: oplog.links,
+    batch: oplog.batch,
+    views: oplog.views,
+    optimistic: oplog.optimistic,
+    value: oplog.value
+  }
+}
+
+function noop() {}

package/package.json

@@ -1,12 +1,45 @@
 {
   "name": "autobee",
-  "version": "0.0.0",
-  "description": "",
+  "version": "1.0.2",
+  "description": "Bee based autobase",
+  "license": "Apache-2.0",
+  "author": "Holepunch Inc.",
   "main": "index.js",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/holepunchto/autobee.git"
+  },
+  "files": [
+    "index.js",
+    "lib/*.js",
+    "spec"
+  ],
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "format": "prettier . --write",
+    "lint": "lunte && prettier . --check",
+    "test": "node test/all.js",
+    "test:bare": "bare test/all.js",
+    "test:encrypted": "bare test/all.js --encrypt-all",
+    "test:generate": "brittle -r test/all.js test/*.js"
+  },
+  "dependencies": {
+    "autobee-encryption": "^0.0.5",
+    "autobee-wakeup": "^1.0.5",
+    "b4a": "^1.8.0",
+    "compact-encoding": "^2.18.0",
+    "hyperbee2": "^2.8.0",
+    "hypercore": "^11.28.1",
+    "hypercore-id-encoding": "^1.3.0",
+    "hyperschema": "^1.20.1",
+    "ready-resource": "^1.2.0",
+    "safety-catch": "^1.0.3"
   },
-  "keywords": [],
-  "author": "",
-  "license": "ISC"
+  "devDependencies": {
+    "brittle": "^3.19.1",
+    "corestore": "^7.7.0",
+    "hypercore-crypto": "^3.6.1",
+    "lunte": "^1.6.0",
+    "prettier": "^3.8.1",
+    "prettier-config-holepunch": "^2.0.0"
+  }
 }