hypercore 11.27.16 → 11.28.0

package/README.md

@@ -522,6 +522,54 @@ Info {
 }
 ```
 
+#### `await core.startMarking()`
+
+This enables marking mode for the "mark & sweep" approach to clear hypercore storage. When called the current markings are cleared.
+
+##### Mark & Sweep
+
+This technique allows for marking blocks that should be kept and assuming all other blocks should be cleared. It can be achieved using the following steps:
+
+1. Enable marking mode via `await core.startMarking()`.
+2. Get all blocks that should be kept.  
+   While the marking mode is enabled, all blocks retrieved (via `.get()`, etc) will be "marked". Marked blocks will not be cleared when sweeping.
+3. Sweep to clear unmarked blocks via `await core.sweep()`.  
+   Once complete, all blocks that were not marked will be cleared.
+
+> [!CAUTION]
+> Be careful that caching does not skip a call to `.get()`.  
+> For example, `hyperbee` has caches for looking up the b-tree nodes that
+> needs to be cleared before using mark & sweep.
+
+Example:
+
+```js
+await core.startMarking()
+await core.get(2)
+await core.get(4)
+await core.sweep() // All blocks but blocks 2 & 4 are cleared
+```
+
+#### `await core.markBlock(start, end = start + 1)`
+
+Manually mark a block or range of blocks to be retained when sweeping. Useful to mark blocks without loading them into memory. `end` is non-inclusive and defaults to `start + 1` so `core.markBlock(index)` only marks the block at `index`.
+
+#### `await core.clearMarkings()`
+
+Manually remove all markings. Automatically called when calling `core.startMarking()`.
+
+#### `await core.sweep(opts)`
+
+Clear all unmarked blocks from storage.
+
+`opts` can include:
+
+```
+{
+  batchSize: 1000 // How frequently to flush clears to storage.
+}
+```
+
 #### `await core.close([{ error }])`
 
 Fully close this core. Passing an error via `{ error }` is optional and all pending replicator requests will be rejected with the error.

package/index.js

@@ -10,9 +10,11 @@ const id = require('hypercore-id-encoding')
 const safetyCatch = require('safety-catch')
 const unslab = require('unslab')
 const flat = require('flat-tree')
+const assert = require('nanoassert')
 
 const { SMALL_WANTS } = require('./lib/feature-flags')
 const { UPDATE_COMPAT } = require('./lib/wants')
+const MarkBitfield = require('./lib/mark-bitfield')
 
 const inspect = require('./lib/inspect')
 const Core = require('./lib/core')
@@ -92,6 +94,10 @@ class Hypercore extends EventEmitter {
 
     this.waits = 0
 
+    // Mark & Sweep GC
+    this._marking = false
+    this._marks = null
+
     this._sessionIndex = -1
     this._stateIndex = -1 // maintained by session state
     this._monitorIndex = -1 // maintained by replication state
@@ -218,6 +224,8 @@ class Hypercore extends EventEmitter {
     const onseq = opts.onseq === undefined ? this.onseq : opts.onseq
     const timeout = opts.timeout === undefined ? this.timeout : opts.timeout
     const weak = opts.weak === undefined ? this.weak : opts.weak
+    const marking = this._marking
+    const marks = this._marks
     const Clz = opts.class || Hypercore
     const s = new Clz(null, this.key, {
       ...opts,
@@ -229,6 +237,8 @@ class Hypercore extends EventEmitter {
       weak,
       parent: this
     })
+    s._marking = marking
+    s._marks = marks
 
     return s
   }
@@ -824,6 +834,7 @@ class Hypercore extends EventEmitter {
       (opts && opts.valueEncoding && c.from(opts.valueEncoding)) || this.valueEncoding
 
     if (this.onseq !== null) this.onseq(index, this)
+    if (this._marking) await this.markBlock(index)
 
     const req = this._get(index, opts)
 
@@ -934,6 +945,82 @@ class Hypercore extends EventEmitter {
     return defaultValue
   }
 
+  _setupMarks() {
+    if (this._marks === null) {
+      const storage = this.snapshotted ? this.core.state.storage : this.state.storage
+      this._marks = new MarkBitfield(storage)
+    }
+  }
+
+  async markBlock(start, end = start + 1) {
+    if (this.opened === false) await this.opening
+
+    this._setupMarks()
+
+    // TODO support as single rocks batch
+    const setPromises = []
+    for (let i = start; i < end; i++) {
+      setPromises.push(this._marks.set(i, true))
+    }
+
+    return Promise.all(setPromises)
+  }
+
+  async clearMarkings() {
+    if (this.opened === false) await this.opening
+
+    this._setupMarks()
+
+    await this._marks.clear()
+    this._marks = null
+  }
+
+  async startMarking() {
+    if (this._marking) {
+      throw ASSERTION("Hypercore cannot be gc'ed when already in gc mode", this.discoveryKey)
+    }
+    if (this.state && this.state.name) {
+      throw ASSERTION("Hypercore cannot be gc'ed when a named session", this.discoveryKey)
+    }
+    if (this.state && this.state.storage.atom) {
+      throw ASSERTION("Hypercore cannot be gc'ed when an atomic session", this.discoveryKey)
+    }
+    if (this.opened === false) await this.opening
+    await this.clearMarkings()
+
+    this._marking = true
+  }
+
+  async sweep({ batchSize = 1000 } = {}) {
+    if (this.opened === false) await this.opening
+
+    assert(!this.snapshotted, 'Cannot sweep a snapshot')
+
+    // No marks - load from storage
+    this._setupMarks()
+
+    let clearing = []
+    let prevIndex = this.length
+    for await (const index of this._marks.createMarkStream({ reverse: true })) {
+      if (index + 1 === prevIndex) {
+        prevIndex = index
+        continue
+      }
+      clearing.push(this.clear(index + 1, prevIndex))
+      if (clearing.length >= batchSize) {
+        await Promise.all(clearing)
+        clearing = []
+      }
+      prevIndex = index
+    }
+    // Clear range from the very start if not marked
+    if (prevIndex > 0) clearing.push(this.clear(0, prevIndex))
+    await Promise.all(clearing)
+
+    this._marking = false
+    await this.clearMarkings()
+  }
+
   createReadStream(opts) {
     return new ReadStream(this, opts)
   }
@@ -962,6 +1049,7 @@ class Hypercore extends EventEmitter {
 
   async truncate(newLength = 0, opts = {}) {
     if (this.opened === false) await this.opening
+    if (this.closing) throw SESSION_CLOSED('Cannot append to a closed session', this.discoveryKey)
 
     const {
       fork = this.state.fork + 1,
@@ -983,6 +1071,7 @@ class Hypercore extends EventEmitter {
 
   async append(blocks, opts = {}) {
     if (this.opened === false) await this.opening
+    if (this.closing) throw SESSION_CLOSED('Cannot append to a closed session', this.discoveryKey)
 
     const isDefault = this.state === this.core.state
     const defaultKeyPair = this.state.name === null ? this.keyPair : null

package/lib/mark-bitfield.js

@@ -0,0 +1,108 @@
+const BigSparseArray = require('big-sparse-array')
+const { Transform } = require('streamx')
+const quickbit = require('./compat').quickbit
+const b4a = require('b4a')
+
+const BITS_PER_PAGE = 32768
+const BYTES_PER_PAGE = BITS_PER_PAGE / 8
+
+class MarkPage {
+  constructor() {
+    this.bitfield = null
+    this.loaded = new Promise((resolve) => {
+      this.load = resolve
+    })
+  }
+
+  setBitfield(bitfield) {
+    this.bitfield = bitfield
+    this.load()
+  }
+
+  get(index) {
+    return quickbit.get(this.bitfield, index)
+  }
+
+  set(index, val) {
+    quickbit.set(this.bitfield, index, val)
+  }
+}
+
+module.exports = class MarkBitfield {
+  static BITS_PER_PAGE = BITS_PER_PAGE
+  static BYTES_PER_PAGE = BYTES_PER_PAGE
+
+  constructor(storage) {
+    this.storage = storage
+    this._pages = new BigSparseArray()
+  }
+
+  async loadPage(pageIndex) {
+    const p = this._pages.set(pageIndex, new MarkPage())
+    const rx = this.storage.read()
+    const pageBuf = rx.getMark(pageIndex)
+    rx.tryFlush()
+    const bitfield = (await pageBuf) ?? b4a.alloc(BYTES_PER_PAGE)
+    await p.setBitfield(bitfield)
+    return p
+  }
+
+  async get(index) {
+    const j = index & (BITS_PER_PAGE - 1)
+    const i = (index - j) / BITS_PER_PAGE
+
+    let p = this._pages.get(i)
+    if (!p) p = await this.loadPage(i)
+
+    return p.get(j)
+  }
+
+  async set(index, val) {
+    const j = index & (BITS_PER_PAGE - 1)
+    const i = (index - j) / BITS_PER_PAGE
+
+    let p = this._pages.get(i)
+
+    if (!p && val) p = await this.loadPage(i)
+
+    if (p) {
+      await p.loaded
+      p.set(j, val)
+      const tx = this.storage.write()
+      tx.putMark(i, p.bitfield)
+      await tx.flush()
+    }
+  }
+
+  async clear() {
+    const tx = this.storage.write()
+    tx.deleteMarkRange(0, -1)
+    await tx.flush()
+    this._pages = new BigSparseArray()
+  }
+
+  createMarkStream({ reverse = false } = {}) {
+    return this.storage.createMarkStream({ reverse }).pipe(
+      new Transform({
+        transform({ index, page }, cb) {
+          let bitIndex = reverse
+            ? quickbit.findLast(page, true, BITS_PER_PAGE)
+            : quickbit.findFirst(page, true, 0)
+          while (bitIndex !== -1) {
+            const blockIndex = index * BITS_PER_PAGE + bitIndex
+            this.push(blockIndex)
+
+            // Account for `bitIndex` being either causing infinite loop
+            if (bitIndex === 0 && reverse) break
+
+            bitIndex = reverse
+              ? quickbit.findLast(page, true, bitIndex - 1)
+              : quickbit.findFirst(page, true, bitIndex + 1)
+          }
+
+          cb(null)
+        }
+      })
+    )
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "11.27.16",
+  "version": "11.28.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
@@ -54,7 +54,7 @@
     "hypercore-crypto": "^3.2.1",
     "hypercore-errors": "^1.5.0",
     "hypercore-id-encoding": "^1.2.0",
-    "hypercore-storage": "^2.0.0",
+    "hypercore-storage": "^2.8.0",
     "is-options": "^1.0.1",
     "nanoassert": "^2.0.0",
     "protomux": "^3.5.0",