@web3-storage/pail 0.6.0-alpha.4 → 0.6.2

package/src/batch/api.ts

@@ -1,59 +0,0 @@
-import {
-  UnknownLink,
-  ShardLink,
-  ShardDiff,
-  ShardEntry,
-  ShardEntryValueValue,
-  ShardEntryLinkValue,
-  ShardEntryLinkAndValueValue,
-  ShardConfig,
-  ShardOptions,
-  ShardBlockView,
-  BlockFetcher
-} from '../api.js'
-
-export {
-  UnknownLink,
-  ShardLink,
-  ShardDiff,
-  ShardEntry,
-  ShardEntryValueValue,
-  ShardEntryLinkValue,
-  ShardEntryLinkAndValueValue,
-  ShardConfig,
-  ShardOptions,
-  ShardBlockView,
-  BlockFetcher
-}
-
-export interface BatcherShard extends ShardConfig {
-  base?: ShardBlockView
-  entries: BatcherShardEntry[]
-}
-
-export interface BatcherShardInit extends ShardOptions {
-  base?: ShardBlockView
-  entries?: BatcherShardEntry[]
-}
-
-export type BatcherShardEntry = [
-  key: string,
-  value: ShardEntryValueValue | ShardEntryLinkValue | ShardEntryLinkAndValueValue | ShardEntryShardValue | ShardEntryShardAndValueValue
-]
-
-export type ShardEntryShardValue = [BatcherShard]
-
-export type ShardEntryShardAndValueValue = [BatcherShard, UnknownLink]
-
-export interface Batcher {
-  /**
-   * Put a value (a CID) for the given key. If the key exists it's value is
-   * overwritten.
-   */
-  put (key: string, value: UnknownLink): Promise<void>
-  /**
-   * Encode all altered shards in the batch and return the new root CID and
-   * difference blocks.
-   */
-  commit (): Promise<{ root: ShardLink } & ShardDiff>
-}

package/src/batch/index.js

@@ -1,248 +0,0 @@
-// eslint-disable-next-line no-unused-vars
-import * as API from './api.js'
-import { ShardFetcher } from '../shard.js'
-import * as Shard from '../shard.js'
-import * as BatcherShard from './shard.js'
-
-/** @implements {API.Batcher} */
-class Batcher {
-  #committed = false
-
-  /**
-   * @param {object} init
-   * @param {API.BlockFetcher} init.blocks Block storage.
-   * @param {API.BatcherShardEntry[]} init.entries The entries in this shard.
-   * @param {string} init.prefix Key prefix.
-   * @param {number} init.version Shard compatibility version.
-   * @param {string} init.keyChars Characters allowed in keys, referring to a known character set.
-   * @param {number} init.maxKeySize Max key size in bytes.
-   * @param {API.ShardBlockView} init.base Original shard this batcher is based on.
-   */
-  constructor ({ blocks, entries, prefix, version, keyChars, maxKeySize, base }) {
-    this.blocks = blocks
-    this.prefix = prefix
-    this.entries = [...entries]
-    this.base = base
-    this.version = version
-    this.keyChars = keyChars
-    this.maxKeySize = maxKeySize
-  }
-
-  /**
-   * @param {string} key The key of the value to put.
-   * @param {API.UnknownLink} value The value to put.
-   * @returns {Promise<void>}
-   */
-  async put (key, value) {
-    if (this.#committed) throw new BatchCommittedError()
-    return put(this.blocks, this, key, value)
-  }
-
-  async commit () {
-    if (this.#committed) throw new BatchCommittedError()
-    this.#committed = true
-    return commit(this)
-  }
-
-  /**
-   * @param {object} init
-   * @param {API.BlockFetcher} init.blocks Block storage.
-   * @param {API.ShardLink} init.link CID of the shard block.
-   */
-  static async create ({ blocks, link }) {
-    const shards = new ShardFetcher(blocks)
-    const base = await shards.get(link)
-    return new Batcher({ blocks, base, ...base.value })
-  }
-}
-
-/**
- * @param {API.BlockFetcher} blocks
- * @param {API.BatcherShard} shard
- * @param {string} key The key of the value to put.
- * @param {API.UnknownLink} value The value to put.
- * @returns {Promise<void>}
- */
-export const put = async (blocks, shard, key, value) => {
-  const shards = new ShardFetcher(blocks)
-  const dest = await traverse(shards, key, shard)
-  if (dest.shard !== shard) {
-    shard = dest.shard
-    key = dest.key
-  }
-
-  /** @type {API.BatcherShardEntry} */
-  let entry = [key, value]
-  /** @type {API.BatcherShard|undefined} */
-  let batcher
-
-  // if the key in this shard is longer than allowed, then we need to make some
-  // intermediate shards.
-  if (key.length > shard.maxKeyLength) {
-    const pfxskeys = Array.from(Array(Math.ceil(key.length / shard.maxKeyLength)), (_, i) => {
-      const start = i * shard.maxKeyLength
-      return {
-        prefix: shard.prefix + key.slice(0, start),
-        key: key.slice(start, start + shard.maxKeyLength)
-      }
-    })
-
-    entry = [pfxskeys[pfxskeys.length - 1].key, value]
-    batcher = BatcherShard.create({
-      entries: [entry],
-      prefix: pfxskeys[pfxskeys.length - 1].prefix,
-      ...Shard.configure(shard)
-    })
-
-    for (let i = pfxskeys.length - 2; i > 0; i--) {
-      entry = [pfxskeys[i].key, [batcher]]
-      batcher = BatcherShard.create({
-        entries: [entry],
-        prefix: pfxskeys[i].prefix,
-        ...Shard.configure(shard)
-      })
-    }
-
-    entry = [pfxskeys[0].key, [batcher]]
-  }
-
-  shard.entries = Shard.putEntry(asShardEntries(shard.entries), asShardEntry(entry))
-
-  // TODO: adjust size automatically
-  const size = BatcherShard.encodedLength(shard)
-  if (size > shard.maxSize) {
-    const common = Shard.findCommonPrefix(
-      asShardEntries(shard.entries),
-      entry[0]
-    )
-    if (!common) throw new Error('shard limit reached')
-    const { prefix } = common
-    /** @type {API.BatcherShardEntry[]} */
-    const matches = common.matches
-
-    const entries = matches
-      .filter(m => m[0] !== prefix)
-      .map(m => {
-        m = [...m]
-        m[0] = m[0].slice(prefix.length)
-        return m
-      })
-
-    const batcher = BatcherShard.create({
-      entries,
-      prefix: shard.prefix + prefix,
-      ...Shard.configure(shard)
-    })
-
-    /** @type {API.ShardEntryShardValue | API.ShardEntryShardAndValueValue} */
-    let value
-    const pfxmatch = matches.find(m => m[0] === prefix)
-    if (pfxmatch) {
-      if (Array.isArray(pfxmatch[1])) {
-        // should not happen! all entries with this prefix should have been
-        // placed within this shard already.
-        throw new Error(`expected "${prefix}" to be a shard value but found a shard link`)
-      }
-      value = [batcher, pfxmatch[1]]
-    } else {
-      value = [batcher]
-    }
-
-    shard.entries = Shard.putEntry(
-      asShardEntries(shard.entries.filter(e => matches.every(m => e[0] !== m[0]))),
-      asShardEntry([prefix, value])
-    )
-  }
-}
-
-/**
- * Traverse from the passed shard through to the correct shard for the passed
- * key.
- *
- * @param {ShardFetcher} shards
- * @param {string} key
- * @param {API.BatcherShard} shard
- * @returns {Promise<{ shard: API.BatcherShard, key: string }>}
- */
-export const traverse = async (shards, key, shard) => {
-  for (let i = 0; i < shard.entries.length; i++) {
-    const [k, v] = shard.entries[i]
-    if (key <= k) break
-    if (key.startsWith(k) && Array.isArray(v)) {
-      if (Shard.isShardLink(v[0])) {
-        const blk = await shards.get(v[0])
-        const batcher = BatcherShard.create({ base: blk, ...blk.value })
-        shard.entries[i] = [k, v[1] == null ? [batcher] : [batcher, v[1]]]
-        return traverse(shards, key.slice(k.length), batcher)
-      }
-      return traverse(shards, key.slice(k.length), v[0])
-    }
-  }
-  return { shard, key }
-}
-
-/**
- * Encode all altered shards in the batch and return the new root CID and
- * difference blocks.
- *
- * @param {API.BatcherShard} shard
- */
-export const commit = async shard => {
-  /** @type {API.ShardBlockView[]} */
-  const additions = []
-  /** @type {API.ShardBlockView[]} */
-  const removals = []
-
-  /** @type {API.ShardEntry[]} */
-  const entries = []
-  for (const entry of shard.entries) {
-    if (Array.isArray(entry[1]) && !Shard.isShardLink(entry[1][0])) {
-      const result = await commit(entry[1][0])
-      entries.push([
-        entry[0],
-        entry[1][1] == null ? [result.root] : [result.root, entry[1][1]]
-      ])
-      additions.push(...result.additions)
-      removals.push(...result.removals)
-    } else {
-      entries.push(asShardEntry(entry))
-    }
-  }
-
-  const block = await Shard.encodeBlock(Shard.withEntries(entries, shard))
-  additions.push(block)
-
-  if (shard.base && shard.base.cid.toString() === block.cid.toString()) {
-    return { root: block.cid, additions: [], removals: [] }
-  }
-
-  if (shard.base) removals.push(shard.base)
-
-  return { root: block.cid, additions, removals }
-}
-
-/** @param {API.BatcherShardEntry[]} entries */
-const asShardEntries = entries => /** @type {API.ShardEntry[]} */ (entries)
-
-/** @param {API.BatcherShardEntry} entry */
-const asShardEntry = entry => /** @type {API.ShardEntry} */ (entry)
-
-/**
- * @param {API.BlockFetcher} blocks Bucket block storage.
- * @param {API.ShardLink} root CID of the root shard block.
- * @returns {Promise<API.Batcher>}
- */
-export const create = (blocks, root) => Batcher.create({ blocks, link: root })
-
-export class BatchCommittedError extends Error {
-  /**
-   * @param {string} [message]
-   * @param {ErrorOptions} [options]
-   */
-  constructor (message, options) {
-    super(message ?? 'batch already committed', options)
-    this.code = BatchCommittedError.code
-  }
-
-  static code = 'ERR_BATCH_COMMITTED'
-}

package/src/batch/shard.js

@@ -1,13 +0,0 @@
-// eslint-disable-next-line no-unused-vars
-import * as API from './api.js'
-import { configure } from '../shard.js'
-
-/**
- * @param {API.BatcherShardInit} [init]
- * @returns {API.BatcherShard}
- */
-export const create = init => ({
-  base: init?.base,
-  entries: [...init?.entries ?? []],
-  ...configure(init)
-})

package/src/block.js

@@ -1,75 +0,0 @@
-// eslint-disable-next-line no-unused-vars
-import * as API from './api.js'
-import { parse } from 'multiformats/link'
-
-/** @implements {API.BlockFetcher} */
-export class MemoryBlockstore {
-  /** @type {Map<string, Uint8Array>} */
-  #blocks = new Map()
-
-  /**
-   * @param {Array<import('multiformats').Block>} [blocks]
-   */
-  constructor (blocks) {
-    if (blocks) {
-      this.#blocks = new Map(blocks.map(b => [b.cid.toString(), b.bytes]))
-    }
-  }
-
-  /** @type {API.BlockFetcher['get']} */
-  async get (cid) {
-    const bytes = this.#blocks.get(cid.toString())
-    if (!bytes) return
-    return { cid, bytes }
-  }
-
-  /**
-   * @param {API.UnknownLink} cid
-   * @param {Uint8Array} bytes
-   */
-  async put (cid, bytes) {
-    this.#blocks.set(cid.toString(), bytes)
-  }
-
-  /**
-   * @param {API.UnknownLink} cid
-   * @param {Uint8Array} bytes
-   */
-  putSync (cid, bytes) {
-    this.#blocks.set(cid.toString(), bytes)
-  }
-
-  /** @param {API.UnknownLink} cid */
-  async delete (cid) {
-    this.#blocks.delete(cid.toString())
-  }
-
-  /** @param {API.UnknownLink} cid */
-  deleteSync (cid) {
-    this.#blocks.delete(cid.toString())
-  }
-
-  * entries () {
-    for (const [str, bytes] of this.#blocks) {
-      yield { cid: parse(str), bytes }
-    }
-  }
-}
-
-export class MultiBlockFetcher {
-  /** @type {API.BlockFetcher[]} */
-  #fetchers
-
-  /** @param {API.BlockFetcher[]} fetchers */
-  constructor (...fetchers) {
-    this.#fetchers = fetchers
-  }
-
-  /** @type {API.BlockFetcher['get']} */
-  async get (link) {
-    for (const f of this.#fetchers) {
-      const v = await f.get(link)
-      if (v) return v
-    }
-  }
-}

package/src/clock/api.js

@@ -1 +0,0 @@
-export {}

package/src/clock/api.ts

@@ -1,12 +0,0 @@
-import { Link, BlockView } from 'multiformats'
-
-export { BlockFetcher } from '../api.js'
-
-export type EventLink<T> = Link<EventView<T>>
-
-export interface EventView<T> {
-  parents: EventLink<T>[]
-  data: T
-}
-
-export interface EventBlockView<T> extends BlockView<EventView<T>> {}

package/src/clock/index.js

@@ -1,182 +0,0 @@
-import { Block, encode, decode } from 'multiformats/block'
-import { sha256 } from 'multiformats/hashes/sha2'
-import * as cbor from '@ipld/dag-cbor'
-// eslint-disable-next-line no-unused-vars
-import * as API from './api.js'
-
-/**
- * Advance the clock by adding an event.
- *
- * @template T
- * @param {API.BlockFetcher} blocks Block storage.
- * @param {API.EventLink<T>[]} head The head of the clock.
- * @param {API.EventLink<T>} event The event to add.
- */
-export const advance = async (blocks, head, event) => {
-  const events = new EventFetcher(blocks)
-  const headmap = new Map(head.map(cid => [cid.toString(), cid]))
-  if (headmap.has(event.toString())) return head
-
-  // does event contain the clock?
-  let changed = false
-  for (const cid of head) {
-    if (await contains(events, event, cid)) {
-      headmap.delete(cid.toString())
-      headmap.set(event.toString(), event)
-      changed = true
-    }
-  }
-  if (changed) {
-    return [...headmap.values()]
-  }
-
-  // does clock contain the event?
-  for (const p of head) {
-    if (await contains(events, p, event)) {
-      return head
-    }
-  }
-
-  return head.concat(event)
-}
-
-/**
- * @template T
- * @extends {Block<API.EventView<T>, typeof cbor.code, typeof sha256.code, 1>}
- * @implements {API.EventBlockView<T>}
- */
-export class EventBlock extends Block {
-  /**
-   * @param {object} config
-   * @param {API.EventLink<T>} config.cid
-   * @param {Event} config.value
-   * @param {Uint8Array} config.bytes
-   * @param {string} config.prefix
-   */
-  constructor ({ cid, value, bytes, prefix }) {
-    // @ts-expect-error
-    super({ cid, value, bytes })
-    this.prefix = prefix
-  }
-
-  /**
-   * @template T
-   * @param {T} data
-   * @param {API.EventLink<T>[]} [parents]
-   */
-  static create (data, parents) {
-    return encodeEventBlock({ data, parents: parents ?? [] })
-  }
-}
-
-/** @template T */
-export class EventFetcher {
-  /** @param {API.BlockFetcher} blocks */
-  constructor (blocks) {
-    /** @private */
-    this._blocks = blocks
-  }
-
-  /**
-   * @param {API.EventLink<T>} link
-   * @returns {Promise<API.EventBlockView<T>>}
-   */
-  async get (link) {
-    const block = await this._blocks.get(link)
-    if (!block) throw new Error(`missing block: ${link}`)
-    return decodeEventBlock(block.bytes)
-  }
-}
-
-/**
- * @template T
- * @param {API.EventView<T>} value
- * @returns {Promise<API.EventBlockView<T>>}
- */
-export const encodeEventBlock = async (value) => {
-  // TODO: sort parents
-  const { cid, bytes } = await encode({ value, codec: cbor, hasher: sha256 })
-  // @ts-expect-error
-  return new Block({ cid, value, bytes })
-}
-
-/**
- * @template T
- * @param {Uint8Array} bytes
- * @returns {Promise<API.EventBlockView<T>>}
- */
-export const decodeEventBlock = async (bytes) => {
-  const { cid, value } = await decode({ bytes, codec: cbor, hasher: sha256 })
-  // @ts-expect-error
-  return new Block({ cid, value, bytes })
-}
-
-/**
- * Returns true if event "a" contains event "b". Breadth first search.
- * @template T
- * @param {EventFetcher<T>} events
- * @param {API.EventLink<T>} a
- * @param {API.EventLink<T>} b
- */
-const contains = async (events, a, b) => {
-  if (a.toString() === b.toString()) return true
-  const [{ value: aevent }, { value: bevent }] = await Promise.all([events.get(a), events.get(b)])
-  const links = [...aevent.parents]
-  const seen = new Set()
-  while (links.length) {
-    const link = links.shift()
-    if (!link) break
-    if (link.toString() === b.toString()) return true
-    // if any of b's parents are this link, then b cannot exist in any of the
-    // tree below, since that would create a cycle.
-    if (bevent.parents.some(p => link.toString() === p.toString())) continue
-    if (seen.has(link.toString())) continue
-    seen.add(link.toString())
-    const { value: event } = await events.get(link)
-    links.push(...event.parents)
-  }
-  return false
-}
-
-/**
- * @template T
- * @param {API.BlockFetcher} blocks Block storage.
- * @param {API.EventLink<T>[]} head
- * @param {object} [options]
- * @param {(b: API.EventBlockView<T>) => string} [options.renderNodeLabel]
- */
-export const vis = async function * (blocks, head, options = {}) {
-  const renderNodeLabel = options.renderNodeLabel ?? (b => shortLink(b.cid))
-  const events = new EventFetcher(blocks)
-  yield 'digraph clock {'
-  yield '  node [shape=point fontname="Courier"]; head;'
-  const hevents = await Promise.all(head.map(link => events.get(link)))
-  /** @type {import('multiformats').Link<API.EventView<any>>[]} */
-  const links = []
-  const nodes = new Set()
-  for (const e of hevents) {
-    nodes.add(e.cid.toString())
-    yield `  node [shape=oval fontname="Courier"]; ${e.cid} [label="${renderNodeLabel(e)}"];`
-    yield `  head -> ${e.cid};`
-    for (const p of e.value.parents) {
-      yield `  ${e.cid} -> ${p};`
-    }
-    links.push(...e.value.parents)
-  }
-  while (links.length) {
-    const link = links.shift()
-    if (!link) break
-    if (nodes.has(link.toString())) continue
-    nodes.add(link.toString())
-    const block = await events.get(link)
-    yield `  node [shape=oval]; ${link} [label="${renderNodeLabel(block)}" fontname="Courier"];`
-    for (const p of block.value.parents) {
-      yield `  ${link} -> ${p};`
-    }
-    links.push(...block.value.parents)
-  }
-  yield '}'
-}
-
-/** @param {import('multiformats').UnknownLink} l */
-const shortLink = l => `${String(l).slice(0, 4)}..${String(l).slice(-4)}`

package/src/crdt/api.js

@@ -1 +0,0 @@
-export {}

package/src/crdt/api.ts

@@ -1,33 +0,0 @@
-import { ShardDiff, ShardLink, UnknownLink } from '../api.js'
-import { EventLink, EventBlockView } from '../clock/api.js'
-
-export { BlockFetcher, UnknownLink, ShardBlockView, ShardDiff, ShardLink } from '../api.js'
-export { EventBlockView, EventLink } from '../clock/api.js'
-
-export interface Result extends ShardDiff {
-  root: ShardLink
-  head: EventLink<Operation>[]
-  event?: EventBlockView<Operation>
-}
-
-export type Operation = (
-  | PutOperation
-  | DeleteOperation
-  | BatchOperation
-) & { root: ShardLink }
-
-export interface PutOperation {
-  type: 'put',
-  key: string
-  value: UnknownLink
-}
-
-export interface DeleteOperation {
-  type: 'del',
-  key: string
-}
-
-export interface BatchOperation {
-  type: 'batch',
-  ops: Array<PutOperation|DeleteOperation>
-}

package/src/crdt/batch/api.js

@@ -1 +0,0 @@
-export {}

package/src/crdt/batch/api.ts

@@ -1,30 +0,0 @@
-import {
-  Batcher,
-  BatcherShardEntry,
-  ShardBlockView,
-  BlockFetcher,
-  ShardLink,
-  UnknownLink
-} from '../../batch/api.js'
-import { Operation, BatchOperation, EventLink, Result } from '../api.js'
-
-export {
-  Batcher,
-  BatcherShardEntry,
-  ShardBlockView,
-  BlockFetcher,
-  ShardLink,
-  UnknownLink,
-  Operation,
-  BatchOperation,
-  EventLink,
-  Result
-}
-
-export interface CRDTBatcher extends Batcher {
-  /**
-   * Encode all altered shards in the batch and return the new root CID, new
-   * clock head, the new clock event and the difference blocks.
-   */
-  commit (): Promise<Result>
-}