@web3-storage/pail 0.4.0

package/src/api.ts

@@ -0,0 +1,47 @@
+import { Link, UnknownLink, BlockView, Block, Version } from 'multiformats'
+import { sha256 } from 'multiformats/hashes/sha2'
+import * as dagCBOR from '@ipld/dag-cbor'
+
+export { Link, UnknownLink, BlockView, Block, Version }
+
+export type ShardEntryValueValue = UnknownLink
+
+export type ShardEntryLinkValue = [ShardLink]
+
+export type ShardEntryLinkAndValueValue = [ShardLink, UnknownLink]
+
+export type ShardValueEntry = [key: string, value: ShardEntryValueValue]
+
+export type ShardLinkEntry = [key: string, value: ShardEntryLinkValue | ShardEntryLinkAndValueValue]
+
+/** Single key/value entry within a shard. */
+export type ShardEntry = [key: string, value: ShardEntryValueValue | ShardEntryLinkValue | ShardEntryLinkAndValueValue]
+
+export interface Shard extends ShardConfig {
+  entries: ShardEntry[]
+}
+
+export type ShardLink = Link<Shard, typeof dagCBOR.code, typeof sha256.code, 1>
+
+export interface ShardBlockView extends BlockView<Shard, typeof dagCBOR.code, typeof sha256.code, 1> {
+  prefix: string
+}
+
+export interface ShardDiff {
+  additions: ShardBlockView[]
+  removals: ShardBlockView[]
+}
+
+export interface BlockFetcher {
+  get<T = unknown, C extends number = number, A extends number = number, V extends Version = 1> (link: Link<T, C, A, V>):
+    Promise<Block<T, C, A, V> | undefined>
+}
+
+export interface ShardConfig {
+  /** Max encoded shard size in bytes - default 512 KiB. */
+  maxSize: number
+  /** Max key length (in UTF-8 encoded characters) - default 64. */
+  maxKeyLength: number
+}
+
+export type ShardOptions = Partial<ShardConfig>

package/src/batch/api.js

@@ -0,0 +1 @@
+export {}

package/src/batch/api.ts

@@ -0,0 +1,61 @@
+import {
+  UnknownLink,
+  ShardLink,
+  ShardDiff,
+  ShardEntry,
+  ShardEntryValueValue,
+  ShardEntryLinkValue,
+  ShardEntryLinkAndValueValue,
+  ShardConfig,
+  ShardOptions,
+  ShardBlockView,
+  BlockFetcher
+} from '../api.js'
+
+export {
+  UnknownLink,
+  ShardLink,
+  ShardDiff,
+  ShardEntry,
+  ShardEntryValueValue,
+  ShardEntryLinkValue,
+  ShardEntryLinkAndValueValue,
+  ShardConfig,
+  ShardOptions,
+  ShardBlockView,
+  BlockFetcher
+}
+
+export interface BatcherShard extends ShardConfig {
+  base?: ShardBlockView
+  prefix: string
+  entries: BatcherShardEntry[]
+}
+
+export interface BatcherShardInit extends ShardOptions {
+  base?: ShardBlockView
+  prefix?: string
+  entries?: BatcherShardEntry[]
+}
+
+export type BatcherShardEntry = [
+  key: string,
+  value: ShardEntryValueValue | ShardEntryLinkValue | ShardEntryLinkAndValueValue | ShardEntryShardValue | ShardEntryShardAndValueValue
+]
+
+export type ShardEntryShardValue = [BatcherShard]
+
+export type ShardEntryShardAndValueValue = [BatcherShard, UnknownLink]
+
+export interface Batcher {
+  /**
+   * Put a value (a CID) for the given key. If the key exists it's value is
+   * overwritten.
+   */
+  put (key: string, value: UnknownLink): Promise<void>
+  /**
+   * Encode all altered shards in the batch and return the new root CID and
+   * difference blocks.
+   */
+  commit (): Promise<{ root: ShardLink } & ShardDiff>
+}

package/src/batch/index.js

@@ -0,0 +1,245 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js'
+import { ShardFetcher } from '../shard.js'
+import * as Shard from '../shard.js'
+import * as BatcherShard from './shard.js'
+
+/** @implements {API.Batcher} */
+class Batcher {
+  #committed = false
+
+  /**
+   * @param {object} init
+   * @param {API.BlockFetcher} init.blocks Block storage.
+   * @param {API.BatcherShardEntry[]} init.entries The entries in this shard.
+   * @param {string} init.prefix Key prefix.
+   * @param {number} init.maxSize
+   * @param {number} init.maxKeyLength
+   * @param {API.ShardBlockView} init.base Original shard this batcher is based on.
+   */
+  constructor ({ blocks, entries, prefix, maxSize, maxKeyLength, base }) {
+    this.blocks = blocks
+    this.prefix = prefix
+    this.entries = entries
+    this.base = base
+    this.maxSize = maxSize
+    this.maxKeyLength = maxKeyLength
+  }
+
+  /**
+   * @param {string} key The key of the value to put.
+   * @param {API.UnknownLink} value The value to put.
+   * @returns {Promise<void>}
+   */
+  async put (key, value) {
+    if (this.#committed) throw new BatchCommittedError()
+    return put(this.blocks, this, key, value)
+  }
+
+  async commit () {
+    if (this.#committed) throw new BatchCommittedError()
+    this.#committed = true
+    return commit(this)
+  }
+
+  /**
+   * @param {object} init
+   * @param {API.BlockFetcher} init.blocks Block storage.
+   * @param {API.ShardLink} init.link CID of the shard block.
+   * @param {string} init.prefix
+   */
+  static async create ({ blocks, link, prefix }) {
+    const shards = new ShardFetcher(blocks)
+    const base = await shards.get(link)
+    return new Batcher({ blocks, entries: base.value.entries, prefix, base, ...Shard.configure(base.value) })
+  }
+}
+
+/**
+ * @param {API.BlockFetcher} blocks
+ * @param {API.BatcherShard} shard
+ * @param {string} key The key of the value to put.
+ * @param {API.UnknownLink} value The value to put.
+ * @returns {Promise<void>}
+ */
+export const put = async (blocks, shard, key, value) => {
+  const shards = new ShardFetcher(blocks)
+  const dest = await traverse(shards, key, shard)
+  if (dest.shard !== shard) {
+    shard = dest.shard
+    key = dest.key
+  }
+
+  /** @type {API.BatcherShardEntry} */
+  let entry = [key, value]
+  /** @type {API.BatcherShard|undefined} */
+  let batcher
+
+  // if the key in this shard is longer than allowed, then we need to make some
+  // intermediate shards.
+  if (key.length > shard.maxKeyLength) {
+    const pfxskeys = Array.from(Array(Math.ceil(key.length / shard.maxKeyLength)), (_, i) => {
+      const start = i * shard.maxKeyLength
+      return {
+        prefix: shard.prefix + key.slice(0, start),
+        key: key.slice(start, start + shard.maxKeyLength)
+      }
+    })
+
+    entry = [pfxskeys[pfxskeys.length - 1].key, value]
+    batcher = BatcherShard.create({
+      entries: [entry],
+      prefix: pfxskeys[pfxskeys.length - 1].prefix,
+      ...Shard.configure(shard)
+    })
+
+    for (let i = pfxskeys.length - 2; i > 0; i--) {
+      entry = [pfxskeys[i].key, [batcher]]
+      batcher = BatcherShard.create({
+        entries: [entry],
+        prefix: pfxskeys[i].prefix,
+        ...Shard.configure(shard)
+      })
+    }
+
+    entry = [pfxskeys[0].key, [batcher]]
+  }
+
+  shard.entries = Shard.putEntry(asShardEntries(shard.entries), asShardEntry(entry))
+
+  // TODO: adjust size automatically
+  const size = Shard.encodedLength(Shard.withEntries(asShardEntries(shard.entries), shard))
+  if (size > shard.maxSize) {
+    const common = Shard.findCommonPrefix(
+      asShardEntries(shard.entries),
+      entry[0]
+    )
+    if (!common) throw new Error('shard limit reached')
+    const { prefix } = common
+    /** @type {API.BatcherShardEntry[]} */
+    const matches = common.matches
+
+    const entries = matches
+      .filter(m => m[0] !== prefix)
+      .map(m => {
+        m = [...m]
+        m[0] = m[0].slice(prefix.length)
+        return m
+      })
+
+    const batcher = BatcherShard.create({
+      entries,
+      prefix: shard.prefix + prefix,
+      ...Shard.configure(shard)
+    })
+
+    /** @type {API.ShardEntryShardValue | API.ShardEntryShardAndValueValue} */
+    let value
+    const pfxmatch = matches.find(m => m[0] === prefix)
+    if (pfxmatch) {
+      if (Array.isArray(pfxmatch[1])) {
+        // should not happen! all entries with this prefix should have been
+        // placed within this shard already.
+        throw new Error(`expected "${prefix}" to be a shard value but found a shard link`)
+      }
+      value = [batcher, pfxmatch[1]]
+    } else {
+      value = [batcher]
+    }
+
+    shard.entries = Shard.putEntry(
+      asShardEntries(shard.entries.filter(e => matches.every(m => e[0] !== m[0]))),
+      asShardEntry([prefix, value])
+    )
+  }
+}
+
+/**
+ * Traverse from the passed shard through to the correct shard for the passed
+ * key.
+ *
+ * @param {ShardFetcher} shards
+ * @param {string} key
+ * @param {API.BatcherShard} shard
+ * @returns {Promise<{ shard: API.BatcherShard, key: string }>}
+ */
+export const traverse = async (shards, key, shard) => {
+  for (const e of shard.entries) {
+    const [k, v] = e
+    if (key <= k) break
+    if (key.startsWith(k) && Array.isArray(v)) {
+      if (Shard.isShardLink(v[0])) {
+        const blk = await shards.get(v[0], shard.prefix + k)
+        v[0] = BatcherShard.create({ base: blk, prefix: blk.prefix, ...blk.value })
+      }
+      return traverse(shards, key.slice(k.length), v[0])
+    }
+  }
+  return { shard, key }
+}
+
+/**
+ * Encode all altered shards in the batch and return the new root CID and
+ * difference blocks.
+ *
+ * @param {API.BatcherShard} shard
+ */
+export const commit = async shard => {
+  /** @type {API.ShardBlockView[]} */
+  const additions = []
+  /** @type {API.ShardBlockView[]} */
+  const removals = []
+
+  /** @type {API.ShardEntry[]} */
+  const entries = []
+  for (const entry of shard.entries) {
+    if (Array.isArray(entry[1]) && !Shard.isShardLink(entry[1][0])) {
+      const result = await commit(entry[1][0])
+      entries.push([
+        entry[0],
+        entry[1][1] == null ? [result.root] : [result.root, entry[1][1]]
+      ])
+      additions.push(...result.additions)
+      removals.push(...result.removals)
+    } else {
+      entries.push(asShardEntry(entry))
+    }
+  }
+
+  const block = await Shard.encodeBlock(Shard.withEntries(entries, shard), shard.prefix)
+  additions.push(block)
+
+  if (shard.base && shard.base.cid.toString() === block.cid.toString()) {
+    return { root: block.cid, additions: [], removals: [] }
+  }
+
+  if (shard.base) removals.push(shard.base)
+
+  return { root: block.cid, additions, removals }
+}
+
+/** @param {API.BatcherShardEntry[]} entries */
+const asShardEntries = entries => /** @type {API.ShardEntry[]} */ (entries)
+
+/** @param {API.BatcherShardEntry} entry */
+const asShardEntry = entry => /** @type {API.ShardEntry} */ (entry)
+
+/**
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.ShardLink} root CID of the root shard block.
+ * @returns {Promise<API.Batcher>}
+ */
+export const create = (blocks, root) => Batcher.create({ blocks, link: root, prefix: '' })
+
+export class BatchCommittedError extends Error {
+  /**
+   * @param {string} [message]
+   * @param {ErrorOptions} [options]
+   */
+  constructor (message, options) {
+    super(message ?? 'batch already committed', options)
+    this.code = BatchCommittedError.code
+  }
+
+  static code = 'ERR_BATCH_COMMITTED'
+}

package/src/batch/shard.js

@@ -0,0 +1,14 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js'
+import { configure } from '../shard.js'
+
+/**
+ * @param {API.BatcherShardInit} [init]
+ * @returns {API.BatcherShard}
+ */
+export const create = init => ({
+  base: init?.base,
+  prefix: init?.prefix ?? '',
+  entries: init?.entries ?? [],
+  ...configure(init)
+})

package/src/block.js

@@ -0,0 +1,75 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js'
+import { parse } from 'multiformats/link'
+
+/** @implements {API.BlockFetcher} */
+export class MemoryBlockstore {
+  /** @type {Map<string, Uint8Array>} */
+  #blocks = new Map()
+
+  /**
+   * @param {Array<import('multiformats').Block>} [blocks]
+   */
+  constructor (blocks) {
+    if (blocks) {
+      this.#blocks = new Map(blocks.map(b => [b.cid.toString(), b.bytes]))
+    }
+  }
+
+  /** @type {API.BlockFetcher['get']} */
+  async get (cid) {
+    const bytes = this.#blocks.get(cid.toString())
+    if (!bytes) return
+    return { cid, bytes }
+  }
+
+  /**
+   * @param {API.UnknownLink} cid
+   * @param {Uint8Array} bytes
+   */
+  async put (cid, bytes) {
+    this.#blocks.set(cid.toString(), bytes)
+  }
+
+  /**
+   * @param {API.UnknownLink} cid
+   * @param {Uint8Array} bytes
+   */
+  putSync (cid, bytes) {
+    this.#blocks.set(cid.toString(), bytes)
+  }
+
+  /** @param {API.UnknownLink} cid */
+  async delete (cid) {
+    this.#blocks.delete(cid.toString())
+  }
+
+  /** @param {API.UnknownLink} cid */
+  deleteSync (cid) {
+    this.#blocks.delete(cid.toString())
+  }
+
+  * entries () {
+    for (const [str, bytes] of this.#blocks) {
+      yield { cid: parse(str), bytes }
+    }
+  }
+}
+
+export class MultiBlockFetcher {
+  /** @type {API.BlockFetcher[]} */
+  #fetchers
+
+  /** @param {API.BlockFetcher[]} fetchers */
+  constructor (...fetchers) {
+    this.#fetchers = fetchers
+  }
+
+  /** @type {API.BlockFetcher['get']} */
+  async get (link) {
+    for (const f of this.#fetchers) {
+      const v = await f.get(link)
+      if (v) return v
+    }
+  }
+}

package/src/clock/api.js

@@ -0,0 +1 @@
+export {}

package/src/clock/api.ts

@@ -0,0 +1,12 @@
+import { Link, BlockView } from 'multiformats'
+
+export { BlockFetcher } from '../api.js'
+
+export type EventLink<T> = Link<EventView<T>>
+
+export interface EventView<T> {
+  parents: EventLink<T>[]
+  data: T
+}
+
+export interface EventBlockView<T> extends BlockView<EventView<T>> {}

package/src/clock/index.js

@@ -0,0 +1,179 @@
+import { Block, encode, decode } from 'multiformats/block'
+import { sha256 } from 'multiformats/hashes/sha2'
+import * as cbor from '@ipld/dag-cbor'
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js'
+
+/**
+ * Advance the clock by adding an event.
+ *
+ * @template T
+ * @param {API.BlockFetcher} blocks Block storage.
+ * @param {API.EventLink<T>[]} head The head of the clock.
+ * @param {API.EventLink<T>} event The event to add.
+ */
+export const advance = async (blocks, head, event) => {
+  const events = new EventFetcher(blocks)
+  const headmap = new Map(head.map(cid => [cid.toString(), cid]))
+  if (headmap.has(event.toString())) return head
+
+  // does event contain the clock?
+  let changed = false
+  for (const cid of head) {
+    if (await contains(events, event, cid)) {
+      headmap.delete(cid.toString())
+      headmap.set(event.toString(), event)
+      changed = true
+    }
+  }
+  if (changed) {
+    return [...headmap.values()]
+  }
+
+  // does clock contain the event?
+  for (const p of head) {
+    if (await contains(events, p, event)) {
+      return head
+    }
+  }
+
+  return head.concat(event)
+}
+
+/**
+ * @template T
+ * @extends {Block<API.EventView<T>, typeof cbor.code, typeof sha256.code, 1>}
+ * @implements {API.EventBlockView<T>}
+ */
+export class EventBlock extends Block {
+  /**
+   * @param {object} config
+   * @param {API.EventLink<T>} config.cid
+   * @param {Event} config.value
+   * @param {Uint8Array} config.bytes
+   * @param {string} config.prefix
+   */
+  constructor ({ cid, value, bytes, prefix }) {
+    // @ts-expect-error
+    super({ cid, value, bytes })
+    this.prefix = prefix
+  }
+
+  /**
+   * @template T
+   * @param {T} data
+   * @param {API.EventLink<T>[]} [parents]
+   */
+  static create (data, parents) {
+    return encodeEventBlock({ data, parents: parents ?? [] })
+  }
+}
+
+/** @template T */
+export class EventFetcher {
+  /** @param {API.BlockFetcher} blocks */
+  constructor (blocks) {
+    /** @private */
+    this._blocks = blocks
+  }
+
+  /**
+   * @param {API.EventLink<T>} link
+   * @returns {Promise<API.EventBlockView<T>>}
+   */
+  async get (link) {
+    const block = await this._blocks.get(link)
+    if (!block) throw new Error(`missing block: ${link}`)
+    return decodeEventBlock(block.bytes)
+  }
+}
+
+/**
+ * @template T
+ * @param {API.EventView<T>} value
+ * @returns {Promise<API.EventBlockView<T>>}
+ */
+export const encodeEventBlock = async (value) => {
+  // TODO: sort parents
+  const { cid, bytes } = await encode({ value, codec: cbor, hasher: sha256 })
+  // @ts-expect-error
+  return new Block({ cid, value, bytes })
+}
+
+/**
+ * @template T
+ * @param {Uint8Array} bytes
+ * @returns {Promise<API.EventBlockView<T>>}
+ */
+export const decodeEventBlock = async (bytes) => {
+  const { cid, value } = await decode({ bytes, codec: cbor, hasher: sha256 })
+  // @ts-expect-error
+  return new Block({ cid, value, bytes })
+}
+
+/**
+ * Returns true if event "a" contains event "b". Breadth first search.
+ * @template T
+ * @param {EventFetcher<T>} events
+ * @param {API.EventLink<T>} a
+ * @param {API.EventLink<T>} b
+ */
+const contains = async (events, a, b) => {
+  if (a.toString() === b.toString()) return true
+  const [{ value: aevent }, { value: bevent }] = await Promise.all([events.get(a), events.get(b)])
+  const links = [...aevent.parents]
+  while (links.length) {
+    const link = links.shift()
+    if (!link) break
+    if (link.toString() === b.toString()) return true
+    // if any of b's parents are this link, then b cannot exist in any of the
+    // tree below, since that would create a cycle.
+    if (bevent.parents.some(p => link.toString() === p.toString())) continue
+    const { value: event } = await events.get(link)
+    links.push(...event.parents)
+  }
+  return false
+}
+
+/**
+ * @template T
+ * @param {API.BlockFetcher} blocks Block storage.
+ * @param {API.EventLink<T>[]} head
+ * @param {object} [options]
+ * @param {(b: API.EventBlockView<T>) => string} [options.renderNodeLabel]
+ */
+export const vis = async function * (blocks, head, options = {}) {
+  const renderNodeLabel = options.renderNodeLabel ?? (b => shortLink(b.cid))
+  const events = new EventFetcher(blocks)
+  yield 'digraph clock {'
+  yield '  node [shape=point fontname="Courier"]; head;'
+  const hevents = await Promise.all(head.map(link => events.get(link)))
+  /** @type {import('multiformats').Link<API.EventView<any>>[]} */
+  const links = []
+  const nodes = new Set()
+  for (const e of hevents) {
+    nodes.add(e.cid.toString())
+    yield `  node [shape=oval fontname="Courier"]; ${e.cid} [label="${renderNodeLabel(e)}"];`
+    yield `  head -> ${e.cid};`
+    for (const p of e.value.parents) {
+      yield `  ${e.cid} -> ${p};`
+    }
+    links.push(...e.value.parents)
+  }
+  while (links.length) {
+    const link = links.shift()
+    if (!link) break
+    if (nodes.has(link.toString())) continue
+    nodes.add(link.toString())
+    const block = await events.get(link)
+    yield `  node [shape=oval]; ${link} [label="${renderNodeLabel(block)}" fontname="Courier"];`
+    for (const p of block.value.parents) {
+      yield `  ${link} -> ${p};`
+    }
+    links.push(...block.value.parents)
+  }
+  yield '}'
+}
+
+/** @param {import('multiformats').UnknownLink} l */
+const shortLink = l => `${String(l).slice(0, 4)}..${String(l).slice(-4)}`

package/src/crdt/api.js

@@ -0,0 +1 @@
+export {}

package/src/crdt/api.ts

@@ -0,0 +1,33 @@
+import { ShardDiff, ShardLink, UnknownLink } from '../api.js'
+import { EventLink, EventBlockView } from '../clock/api.js'
+
+export { BlockFetcher, UnknownLink, ShardBlockView, ShardDiff, ShardLink } from '../api.js'
+export { EventBlockView, EventLink } from '../clock/api.js'
+
+export interface Result extends ShardDiff {
+  root: ShardLink
+  head: EventLink<Operation>[]
+  event?: EventBlockView<Operation>
+}
+
+export type Operation = (
+  | PutOperation
+  | DeleteOperation
+  | BatchOperation
+) & { root: ShardLink }
+
+export interface PutOperation {
+  type: 'put',
+  key: string
+  value: UnknownLink
+}
+
+export interface DeleteOperation {
+  type: 'del',
+  key: string
+}
+
+export interface BatchOperation {
+  type: 'batch',
+  ops: Array<PutOperation|DeleteOperation>
+}

package/src/crdt/batch/api.js

@@ -0,0 +1 @@
+export {}