@web3-storage/pail 0.4.0

package/src/crdt/batch/api.ts

@@ -0,0 +1,31 @@
+import {
+  Batcher,
+  BatcherShardEntry,
+  ShardDiff,
+  ShardBlockView,
+  BlockFetcher,
+  ShardLink,
+  UnknownLink
+} from '../../batch/api.js'
+import { Operation, BatchOperation, EventLink, Result } from '../api.js'
+
+export {
+  Batcher,
+  BatcherShardEntry,
+  ShardBlockView,
+  BlockFetcher,
+  ShardLink,
+  UnknownLink,
+  Operation,
+  BatchOperation,
+  EventLink,
+  Result
+}
+
+export interface CRDTBatcher extends Batcher {
+  /**
+   * Encode all altered shards in the batch and return the new root CID, new
+   * clock head, the new clock event and the difference blocks.
+   */
+  commit (): Promise<Result>
+}

package/src/crdt/batch/index.js

@@ -0,0 +1,156 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js'
+import * as Shard from '../../shard.js'
+import { ShardFetcher, ShardBlock } from '../../shard.js'
+import * as Batch from '../../batch/index.js'
+import { BatchCommittedError } from '../../batch/index.js'
+import * as CRDT from '../index.js'
+import * as Clock from '../../clock/index.js'
+import { EventBlock } from '../../clock/index.js'
+import { MemoryBlockstore, MultiBlockFetcher } from '../../block.js'
+
+export { BatchCommittedError }
+
+/** @implements {API.CRDTBatcher} */
+class Batcher {
+  #committed = false
+
+  /**
+   * @param {object} init
+   * @param {API.BlockFetcher} init.blocks Block storage.
+   * @param {API.EventLink<API.Operation>[]} init.head Merkle clock head.
+   * @param {API.BatcherShardEntry[]} init.entries The entries in this shard.
+   * @param {string} init.prefix Key prefix.
+   * @param {number} init.maxSize
+   * @param {number} init.maxKeyLength
+   * @param {API.ShardBlockView} init.base Original shard this batcher is based on.
+   * @param {API.ShardBlockView[]} init.additions Additions to include in the committed batch.
+   * @param {API.ShardBlockView[]} init.removals Removals to include in the committed batch.
+   */
+  constructor ({ blocks, head, entries, prefix, maxSize, maxKeyLength, base, additions, removals }) {
+    this.blocks = blocks
+    this.head = head
+    this.prefix = prefix
+    this.entries = entries
+    this.base = base
+    this.maxSize = maxSize
+    this.maxKeyLength = maxKeyLength
+    this.additions = additions
+    this.removals = removals
+    /** @type {API.BatchOperation['ops']} */
+    this.ops = []
+  }
+
+  /**
+   * @param {string} key The key of the value to put.
+   * @param {API.UnknownLink} value The value to put.
+   * @returns {Promise<void>}
+   */
+  async put (key, value) {
+    if (this.#committed) throw new BatchCommittedError()
+    await Batch.put(this.blocks, this, key, value)
+    this.ops.push({ type: 'put', key, value })
+  }
+
+  async commit () {
+    if (this.#committed) throw new BatchCommittedError()
+    this.#committed = true
+
+    const res = await Batch.commit(this)
+
+    /** @type {API.Operation} */
+    const data = { type: 'batch', ops: this.ops, root: res.root }
+    const event = await EventBlock.create(data, this.head)
+
+    const mblocks = new MemoryBlockstore()
+    const blocks = new MultiBlockFetcher(mblocks, this.blocks)
+    mblocks.putSync(event.cid, event.bytes)
+
+    const head = await Clock.advance(blocks, this.head, event.cid)
+
+    /** @type {Map<string, API.ShardBlockView>} */
+    const additions = new Map()
+    /** @type {Map<string, API.ShardBlockView>} */
+    const removals = new Map()
+
+    for (const a of this.additions) {
+      additions.set(a.cid.toString(), a)
+    }
+    for (const r of this.removals) {
+      removals.set(r.cid.toString(), r)
+    }
+
+    for (const a of res.additions) {
+      if (removals.has(a.cid.toString())) {
+        removals.delete(a.cid.toString())
+      }
+      additions.set(a.cid.toString(), a)
+    }
+    for (const r of res.removals) {
+      if (additions.has(r.cid.toString())) {
+        additions.delete(r.cid.toString())
+      } else {
+        removals.set(r.cid.toString(), r)
+      }
+    }
+
+    return {
+      head,
+      event,
+      root: res.root,
+      additions: [...additions.values()],
+      removals: [...removals.values()]
+    }
+  }
+
+  /**
+   * @param {object} init
+   * @param {API.BlockFetcher} init.blocks Block storage.
+   * @param {API.EventLink<API.Operation>[]} init.head Merkle clock head.
+   * @param {string} init.prefix
+   */
+  static async create ({ blocks, head, prefix }) {
+    const mblocks = new MemoryBlockstore()
+    blocks = new MultiBlockFetcher(mblocks, blocks)
+
+    if (!head.length) {
+      const base = await ShardBlock.create()
+      mblocks.putSync(base.cid, base.bytes)
+      return new Batcher({
+        blocks,
+        head,
+        entries: [],
+        prefix,
+        base,
+        additions: [base],
+        removals: [],
+        ...Shard.configure(base.value)
+      })
+    }
+
+    const { root, additions, removals } = await CRDT.root(blocks, head)
+    for (const a of additions) {
+      mblocks.putSync(a.cid, a.bytes)
+    }
+
+    const shards = new ShardFetcher(blocks)
+    const base = await shards.get(root)
+    return new Batcher({
+      blocks,
+      head,
+      entries: base.value.entries,
+      prefix,
+      base,
+      additions,
+      removals,
+      ...Shard.configure(base.value)
+    })
+  }
+}
+
+/**
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.EventLink<API.Operation>[]} head Merkle clock head.
+ * @returns {Promise<API.CRDTBatcher>}
+ */
+export const create = (blocks, head) => Batcher.create({ blocks, head, prefix: '' })

package/src/crdt/index.js

@@ -0,0 +1,355 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js'
+import * as Clock from '../clock/index.js'
+import { EventFetcher, EventBlock } from '../clock/index.js'
+import * as Pail from '../index.js'
+import { ShardBlock } from '../shard.js'
+import { MemoryBlockstore, MultiBlockFetcher } from '../block.js'
+import * as Batch from '../batch/index.js'
+
+/**
+ * Put a value (a CID) for the given key. If the key exists it's value is
+ * overwritten.
+ *
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.EventLink<API.Operation>[]} head Merkle clock head.
+ * @param {string} key The key of the value to put.
+ * @param {API.UnknownLink} value The value to put.
+ * @returns {Promise<API.Result>}
+ */
+export const put = async (blocks, head, key, value) => {
+  const mblocks = new MemoryBlockstore()
+  blocks = new MultiBlockFetcher(mblocks, blocks)
+
+  if (!head.length) {
+    const shard = await ShardBlock.create()
+    mblocks.putSync(shard.cid, shard.bytes)
+    const result = await Pail.put(blocks, shard.cid, key, value)
+    /** @type {API.Operation} */
+    const data = { type: 'put', root: result.root, key, value }
+    const event = await EventBlock.create(data, head)
+    head = await Clock.advance(blocks, head, event.cid)
+    return {
+      root: result.root,
+      additions: [shard, ...result.additions],
+      removals: result.removals,
+      head,
+      event
+    }
+  }
+
+  /** @type {EventFetcher<API.Operation>} */
+  const events = new EventFetcher(blocks)
+  const ancestor = await findCommonAncestor(events, head)
+  if (!ancestor) throw new Error('failed to find common ancestor event')
+
+  const aevent = await events.get(ancestor)
+  let { root } = aevent.value.data
+
+  const sorted = await findSortedEvents(events, head, ancestor)
+  /** @type {Map<string, API.ShardBlockView>} */
+  const additions = new Map()
+  /** @type {Map<string, API.ShardBlockView>} */
+  const removals = new Map()
+
+  for (const { value: event } of sorted) {
+    let result
+    if (event.data.type === 'put') {
+      result = await Pail.put(blocks, root, event.data.key, event.data.value)
+    } else if (event.data.type === 'del') {
+      result = await Pail.del(blocks, root, event.data.key)
+    } else if (event.data.type === 'batch') {
+      const batch = await Batch.create(blocks, root)
+      for (const op of event.data.ops) {
+        if (op.type !== 'put') throw new Error(`unsupported batch operation: ${op.type}`)
+        await batch.put(op.key, op.value)
+      }
+      result = await batch.commit()
+    } else {
+      // @ts-expect-error type does not exist on never
+      throw new Error(`unknown operation: ${event.data.type}`)
+    }
+
+    root = result.root
+    for (const a of result.additions) {
+      mblocks.putSync(a.cid, a.bytes)
+      additions.set(a.cid.toString(), a)
+    }
+    for (const r of result.removals) {
+      removals.set(r.cid.toString(), r)
+    }
+  }
+
+  const result = await Pail.put(blocks, root, key, value)
+  // if we didn't change the pail we're done
+  if (result.root.toString() === root.toString()) {
+    return { root, additions: [], removals: [], head }
+  }
+
+  for (const a of result.additions) {
+    mblocks.putSync(a.cid, a.bytes)
+    additions.set(a.cid.toString(), a)
+  }
+  for (const r of result.removals) {
+    removals.set(r.cid.toString(), r)
+  }
+
+  /** @type {API.Operation} */
+  const data = { type: 'put', root: result.root, key, value }
+  const event = await EventBlock.create(data, head)
+  mblocks.putSync(event.cid, event.bytes)
+  head = await Clock.advance(blocks, head, event.cid)
+
+  // filter blocks that were added _and_ removed
+  for (const k of removals.keys()) {
+    if (additions.has(k)) {
+      additions.delete(k)
+      removals.delete(k)
+    }
+  }
+
+  return {
+    root: result.root,
+    additions: [...additions.values()],
+    removals: [...removals.values()],
+    head,
+    event
+  }
+}
+
+/**
+ * Delete the value for the given key from the bucket. If the key is not found
+ * no operation occurs.
+ *
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.EventLink<API.Operation>[]} head Merkle clock head.
+ * @param {string} key The key of the value to delete.
+ * @param {object} [options]
+ * @returns {Promise<API.Result>}
+ */
+export const del = async (blocks, head, key, options) => {
+  throw new Error('not implemented')
+}
+
+/**
+ * Determine the effective pail root given the current merkle clock head.
+ *
+ * Clocks with multiple head events may return blocks that were added or
+ * removed while playing forward events from their common ancestor.
+ *
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.EventLink<API.Operation>[]} head Merkle clock head.
+ * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
+ */
+export const root = async (blocks, head) => {
+  if (!head.length) throw new Error('cannot determine root of headless clock')
+
+  const mblocks = new MemoryBlockstore()
+  blocks = new MultiBlockFetcher(mblocks, blocks)
+
+  /** @type {EventFetcher<API.Operation>} */
+  const events = new EventFetcher(blocks)
+
+  if (head.length === 1) {
+    const event = await events.get(head[0])
+    const { root } = event.value.data
+    return { root, additions: [], removals: [] }
+  }
+
+  const ancestor = await findCommonAncestor(events, head)
+  if (!ancestor) throw new Error('failed to find common ancestor event')
+
+  const aevent = await events.get(ancestor)
+  let { root } = aevent.value.data
+
+  const sorted = await findSortedEvents(events, head, ancestor)
+  /** @type {Map<string, API.ShardBlockView>} */
+  const additions = new Map()
+  /** @type {Map<string, API.ShardBlockView>} */
+  const removals = new Map()
+
+  for (const { value: event } of sorted) {
+    let result
+    if (event.data.type === 'put') {
+      result = await Pail.put(blocks, root, event.data.key, event.data.value)
+    } else if (event.data.type === 'del') {
+      result = await Pail.del(blocks, root, event.data.key)
+    } else if (event.data.type === 'batch') {
+      const batch = await Batch.create(blocks, root)
+      for (const op of event.data.ops) {
+        if (op.type !== 'put') throw new Error(`unsupported batch operation: ${op.type}`)
+        await batch.put(op.key, op.value)
+      }
+      result = await batch.commit()
+    } else {
+      // @ts-expect-error type does not exist on never
+      throw new Error(`unknown operation: ${event.data.type}`)
+    }
+
+    root = result.root
+    for (const a of result.additions) {
+      mblocks.putSync(a.cid, a.bytes)
+      additions.set(a.cid.toString(), a)
+    }
+    for (const r of result.removals) {
+      removals.set(r.cid.toString(), r)
+    }
+  }
+
+  // filter blocks that were added _and_ removed
+  for (const k of removals.keys()) {
+    if (additions.has(k)) {
+      additions.delete(k)
+      removals.delete(k)
+    }
+  }
+
+  return {
+    root,
+    additions: [...additions.values()],
+    removals: [...removals.values()]
+  }
+}
+
+/**
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.EventLink<API.Operation>[]} head Merkle clock head.
+ * @param {string} key The key of the value to retrieve.
+ */
+export const get = async (blocks, head, key) => {
+  if (!head.length) return
+  const result = await root(blocks, head)
+  if (result.additions.length) {
+    blocks = new MultiBlockFetcher(new MemoryBlockstore(result.additions), blocks)
+  }
+  return Pail.get(blocks, result.root, key)
+}
+
+/**
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.EventLink<API.Operation>[]} head Merkle clock head.
+ * @param {object} [options]
+ * @param {string} [options.prefix]
+ */
+export const entries = async function * (blocks, head, options) {
+  if (!head.length) return
+  const result = await root(blocks, head)
+  if (result.additions.length) {
+    blocks = new MultiBlockFetcher(new MemoryBlockstore(result.additions), blocks)
+  }
+  yield * Pail.entries(blocks, result.root, options)
+}
+
+/**
+ * Find the common ancestor event of the passed children. A common ancestor is
+ * the first single event in the DAG that _all_ paths from children lead to.
+ *
+ * @param {EventFetcher<API.Operation>} events
+ * @param  {API.EventLink<API.Operation>[]} children
+ */
+const findCommonAncestor = async (events, children) => {
+  if (!children.length) return
+  const candidates = children.map(c => [c])
+  while (true) {
+    let changed = false
+    for (const c of candidates) {
+      const candidate = await findAncestorCandidate(events, c[c.length - 1])
+      if (!candidate) continue
+      changed = true
+      c.push(candidate)
+      const ancestor = findCommonString(candidates)
+      if (ancestor) return ancestor
+    }
+    if (!changed) return
+  }
+}
+
+/**
+ * @param {EventFetcher<API.Operation>} events
+ * @param {API.EventLink<API.Operation>} root
+ */
+const findAncestorCandidate = async (events, root) => {
+  const { value: event } = await events.get(root)
+  if (!event.parents.length) return root
+  return event.parents.length === 1
+    ? event.parents[0]
+    : findCommonAncestor(events, event.parents)
+}
+
+/**
+ * @template {{ toString: () => string }} T
+ * @param  {Array<T[]>} arrays
+ */
+const findCommonString = (arrays) => {
+  arrays = arrays.map(a => [...a])
+  for (const arr of arrays) {
+    for (const item of arr) {
+      let matched = true
+      for (const other of arrays) {
+        if (arr === other) continue
+        matched = other.some(i => String(i) === String(item))
+        if (!matched) break
+      }
+      if (matched) return item
+    }
+  }
+}
+
+/**
+ * Find and sort events between the head(s) and the tail.
+ * @param {EventFetcher<API.Operation>} events
+ * @param {API.EventLink<API.Operation>[]} head
+ * @param {API.EventLink<API.Operation>} tail
+ */
+const findSortedEvents = async (events, head, tail) => {
+  if (head.length === 1 && head[0].toString() === tail.toString()) {
+    return []
+  }
+
+  // get weighted events - heavier events happened first
+  /** @type {Map<string, { event: API.EventBlockView<API.Operation>, weight: number }>} */
+  const weights = new Map()
+  const all = await Promise.all(head.map(h => findEvents(events, h, tail)))
+  for (const arr of all) {
+    for (const { event, depth } of arr) {
+      const info = weights.get(event.cid.toString())
+      if (info) {
+        info.weight += depth
+      } else {
+        weights.set(event.cid.toString(), { event, weight: depth })
+      }
+    }
+  }
+
+  // group events into buckets by weight
+  /** @type {Map<number, API.EventBlockView<API.Operation>[]>} */
+  const buckets = new Map()
+  for (const { event, weight } of weights.values()) {
+    const bucket = buckets.get(weight)
+    if (bucket) {
+      bucket.push(event)
+    } else {
+      buckets.set(weight, [event])
+    }
+  }
+
+  // sort by weight, and by CID within weight
+  return Array.from(buckets)
+    .sort((a, b) => b[0] - a[0])
+    .flatMap(([, es]) => es.sort((a, b) => String(a.cid) < String(b.cid) ? -1 : 1))
+}
+
+/**
+ * @param {EventFetcher<API.Operation>} events
+ * @param {API.EventLink<API.Operation>} start
+ * @param {API.EventLink<API.Operation>} end
+ * @returns {Promise<Array<{ event: API.EventBlockView<API.Operation>, depth: number }>>}
+ */
+const findEvents = async (events, start, end, depth = 0) => {
+  const event = await events.get(start)
+  const acc = [{ event, depth }]
+  const { parents } = event.value
+  if (parents.length === 1 && String(parents[0]) === String(end)) return acc
+  const rest = await Promise.all(parents.map(p => findEvents(events, p, end, depth + 1)))
+  return acc.concat(...rest)
+}

package/src/diff.js

@@ -0,0 +1,151 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js'
+import { ShardFetcher } from './shard.js'
+
+/**
+ * @typedef {string} K
+ * @typedef {[before: null, after: API.UnknownLink]} AddV
+ * @typedef {[before: API.UnknownLink, after: API.UnknownLink]} UpdateV
+ * @typedef {[before: API.UnknownLink, after: null]} DeleteV
+ * @typedef {[key: K, value: AddV|UpdateV|DeleteV]} KV
+ * @typedef {KV[]} KeysDiff
+ * @typedef {{ keys: KeysDiff, shards: API.ShardDiff }} CombinedDiff
+ */
+
+/**
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.ShardLink} a Base DAG.
+ * @param {API.ShardLink} b Comparison DAG.
+ * @returns {Promise<CombinedDiff>}
+ */
+export const difference = async (blocks, a, b, prefix = '') => {
+  if (isEqual(a, b)) return { keys: [], shards: { additions: [], removals: [] } }
+
+  const shards = new ShardFetcher(blocks)
+  const [ashard, bshard] = await Promise.all([shards.get(a, prefix), shards.get(b, prefix)])
+
+  const aents = new Map(ashard.value.entries)
+  const bents = new Map(bshard.value.entries)
+
+  const keys = /** @type {Map<K, AddV|UpdateV|DeleteV>} */(new Map())
+  const additions = new Map([[bshard.cid.toString(), bshard]])
+  const removals = new Map([[ashard.cid.toString(), ashard]])
+
+  // find shards removed in B
+  for (const [akey, aval] of ashard.value.entries) {
+    const bval = bents.get(akey)
+    if (bval) continue
+    if (!Array.isArray(aval)) {
+      keys.set(`${ashard.prefix}${akey}`, [aval, null])
+      continue
+    }
+    // if shard link _with_ value
+    if (aval[1] != null) {
+      keys.set(`${ashard.prefix}${akey}`, [aval[1], null])
+    }
+    for await (const s of collect(shards, aval[0], `${ashard.prefix}${akey}`)) {
+      for (const [k, v] of s.value.entries) {
+        if (!Array.isArray(v)) {
+          keys.set(`${s.prefix}${k}`, [v, null])
+        } else if (v[1] != null) {
+          keys.set(`${s.prefix}${k}`, [v[1], null])
+        }
+      }
+      removals.set(s.cid.toString(), s)
+    }
+  }
+
+  // find shards added or updated in B
+  for (const [bkey, bval] of bshard.value.entries) {
+    const aval = aents.get(bkey)
+    if (!Array.isArray(bval)) {
+      if (!aval) {
+        keys.set(`${bshard.prefix}${bkey}`, [null, bval])
+      } else if (Array.isArray(aval)) {
+        keys.set(`${bshard.prefix}${bkey}`, [aval[1] ?? null, bval])
+      } else if (!isEqual(aval, bval)) {
+        keys.set(`${bshard.prefix}${bkey}`, [aval, bval])
+      }
+      continue
+    }
+    if (aval && Array.isArray(aval)) { // updated in B
+      if (isEqual(aval[0], bval[0])) {
+        if (bval[1] != null && (aval[1] == null || !isEqual(aval[1], bval[1]))) {
+          keys.set(`${bshard.prefix}${bkey}`, [aval[1] ?? null, bval[1]])
+        }
+        continue // updated value?
+      }
+      const res = await difference(blocks, aval[0], bval[0], `${bshard.prefix}${bkey}`)
+      for (const shard of res.shards.additions) {
+        additions.set(shard.cid.toString(), shard)
+      }
+      for (const shard of res.shards.removals) {
+        removals.set(shard.cid.toString(), shard)
+      }
+      for (const [k, v] of res.keys) {
+        keys.set(k, v)
+      }
+    } else if (aval) { // updated in B value => link+value
+      if (bval[1] == null) {
+        keys.set(`${bshard.prefix}${bkey}`, [aval, null])
+      } else if (!isEqual(aval, bval[1])) {
+        keys.set(`${bshard.prefix}${bkey}`, [aval, bval[1]])
+      }
+      for await (const s of collect(shards, bval[0], `${bshard.prefix}${bkey}`)) {
+        for (const [k, v] of s.value.entries) {
+          if (!Array.isArray(v)) {
+            keys.set(`${s.prefix}${k}`, [null, v])
+          } else if (v[1] != null) {
+            keys.set(`${s.prefix}${k}`, [null, v[1]])
+          }
+        }
+        additions.set(s.cid.toString(), s)
+      }
+    } else { // added in B
+      keys.set(`${bshard.prefix}${bkey}`, [null, bval[0]])
+      for await (const s of collect(shards, bval[0], `${bshard.prefix}${bkey}`)) {
+        for (const [k, v] of s.value.entries) {
+          if (!Array.isArray(v)) {
+            keys.set(`${s.prefix}${k}`, [null, v])
+          } else if (v[1] != null) {
+            keys.set(`${s.prefix}${k}`, [null, v[1]])
+          }
+        }
+        additions.set(s.cid.toString(), s)
+      }
+    }
+  }
+
+  // filter blocks that were added _and_ removed from B
+  for (const k of removals.keys()) {
+    if (additions.has(k)) {
+      additions.delete(k)
+      removals.delete(k)
+    }
+  }
+
+  return {
+    keys: [...keys.entries()].sort((a, b) => a[0] < b[0] ? -1 : 1),
+    shards: { additions: [...additions.values()], removals: [...removals.values()] }
+  }
+}
+
+/**
+ * @param {API.UnknownLink} a
+ * @param {API.UnknownLink} b
+ */
+const isEqual = (a, b) => a.toString() === b.toString()
+
+/**
+ * @param {import('./shard.js').ShardFetcher} shards
+ * @param {API.ShardLink} root
+ * @returns {AsyncIterableIterator<API.ShardBlockView>}
+ */
+async function * collect (shards, root, prefix = '') {
+  const shard = await shards.get(root, prefix)
+  yield shard
+  for (const [k, v] of shard.value.entries) {
+    if (!Array.isArray(v)) continue
+    yield * collect(shards, v[0], `${prefix}${k}`)
+  }
+}