@web3-storage/pail 0.4.0

package/src/index.js

@@ -0,0 +1,285 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js'
+import { ShardFetcher } from './shard.js'
+import * as Shard from './shard.js'
+
+/**
+ * Put a value (a CID) for the given key. If the key exists it's value is
+ * overwritten.
+ *
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.ShardLink} root CID of the root node of the bucket.
+ * @param {string} key The key of the value to put.
+ * @param {API.UnknownLink} value The value to put.
+ * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
+ */
+export const put = async (blocks, root, key, value) => {
+  const shards = new ShardFetcher(blocks)
+  const rshard = await shards.get(root)
+  const path = await traverse(shards, rshard, key)
+  const target = path[path.length - 1]
+  const skey = key.slice(target.prefix.length) // key within the shard
+
+  /** @type {API.ShardEntry} */
+  let entry = [skey, value]
+
+  /** @type {API.ShardBlockView[]} */
+  const additions = []
+
+  // if the key in this shard is longer than allowed, then we need to make some
+  // intermediate shards.
+  if (skey.length > target.value.maxKeyLength) {
+    const pfxskeys = Array.from(Array(Math.ceil(skey.length / target.value.maxKeyLength)), (_, i) => {
+      const start = i * target.value.maxKeyLength
+      return {
+        prefix: target.prefix + skey.slice(0, start),
+        skey: skey.slice(start, start + target.value.maxKeyLength)
+      }
+    })
+
+    let child = await Shard.encodeBlock(
+      Shard.withEntries([[pfxskeys[pfxskeys.length - 1].skey, value]], target.value),
+      pfxskeys[pfxskeys.length - 1].prefix
+    )
+    additions.push(child)
+
+    for (let i = pfxskeys.length - 2; i > 0; i--) {
+      child = await Shard.encodeBlock(
+        Shard.withEntries([[pfxskeys[i].skey, [child.cid]]], target.value),
+        pfxskeys[i].prefix
+      )
+      additions.push(child)
+    }
+
+    entry = [pfxskeys[0].skey, [child.cid]]
+  }
+
+  let shard = Shard.withEntries(Shard.putEntry(target.value.entries, entry), target.value)
+  let child = await Shard.encodeBlock(shard, target.prefix)
+
+  if (child.bytes.length > shard.maxSize) {
+    const common = Shard.findCommonPrefix(shard.entries, entry[0])
+    if (!common) throw new Error('shard limit reached')
+    const { prefix, matches } = common
+    const block = await Shard.encodeBlock(
+      Shard.withEntries(
+        matches
+          .filter(([k]) => k !== prefix)
+          .map(([k, v]) => [k.slice(prefix.length), v]),
+        shard
+      ),
+      target.prefix + prefix
+    )
+    additions.push(block)
+
+    /** @type {API.ShardEntryLinkValue | API.ShardEntryLinkAndValueValue} */
+    let value
+    const pfxmatch = matches.find(([k]) => k === prefix)
+    if (pfxmatch) {
+      if (Array.isArray(pfxmatch[1])) {
+        // should not happen! all entries with this prefix should have been
+        // placed within this shard already.
+        throw new Error(`expected "${prefix}" to be a shard value but found a shard link`)
+      }
+      value = [block.cid, pfxmatch[1]]
+    } else {
+      value = [block.cid]
+    }
+
+    shard.entries = shard.entries.filter(e => matches.every(m => e[0] !== m[0]))
+    shard = Shard.withEntries(Shard.putEntry(shard.entries, [prefix, value]), shard)
+    child = await Shard.encodeBlock(shard, target.prefix)
+  }
+
+  // if no change in the target then we're done
+  if (child.cid.toString() === target.cid.toString()) {
+    return { root, additions: [], removals: [] }
+  }
+
+  additions.push(child)
+
+  // path is root -> shard, so work backwards, propagating the new shard CID
+  for (let i = path.length - 2; i >= 0; i--) {
+    const parent = path[i]
+    const key = child.prefix.slice(parent.prefix.length)
+    const value = Shard.withEntries(
+      parent.value.entries.map((entry) => {
+        const [k, v] = entry
+        if (k !== key) return entry
+        if (!Array.isArray(v)) throw new Error(`"${key}" is not a shard link in: ${parent.cid}`)
+        return /** @type {API.ShardEntry} */(v[1] == null ? [k, [child.cid]] : [k, [child.cid, v[1]]])
+      }),
+      parent.value
+    )
+
+    child = await Shard.encodeBlock(value, parent.prefix)
+    additions.push(child)
+  }
+
+  return { root: additions[additions.length - 1].cid, additions, removals: path }
+}
+
+/**
+ * Get the stored value for the given key from the bucket. If the key is not
+ * found, `undefined` is returned.
+ *
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.ShardLink} root CID of the root node of the bucket.
+ * @param {string} key The key of the value to get.
+ * @returns {Promise<API.UnknownLink | undefined>}
+ */
+export const get = async (blocks, root, key) => {
+  const shards = new ShardFetcher(blocks)
+  const rshard = await shards.get(root)
+  const path = await traverse(shards, rshard, key)
+  const target = path[path.length - 1]
+  const skey = key.slice(target.prefix.length) // key within the shard
+  const entry = target.value.entries.find(([k]) => k === skey)
+  if (!entry) return
+  return Array.isArray(entry[1]) ? entry[1][1] : entry[1]
+}
+
+/**
+ * Delete the value for the given key from the bucket. If the key is not found
+ * no operation occurs.
+ *
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.ShardLink} root CID of the root node of the bucket.
+ * @param {string} key The key of the value to delete.
+ * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
+ */
+export const del = async (blocks, root, key) => {
+  const shards = new ShardFetcher(blocks)
+  const rshard = await shards.get(root)
+  const path = await traverse(shards, rshard, key)
+  const target = path[path.length - 1]
+  const skey = key.slice(target.prefix.length) // key within the shard
+
+  const entryidx = target.value.entries.findIndex(([k]) => k === skey)
+  if (entryidx === -1) return { root, additions: [], removals: [] }
+
+  const entry = target.value.entries[entryidx]
+  // cannot delete a shard (without data)
+  if (Array.isArray(entry[1]) && entry[1][1] == null) {
+    return { root, additions: [], removals: [] }
+  }
+
+  /** @type {API.ShardBlockView[]} */
+  const additions = []
+  /** @type {API.ShardBlockView[]} */
+  const removals = [...path]
+
+  let shard = Shard.withEntries([...target.value.entries], target.value)
+
+  if (Array.isArray(entry[1])) {
+    // remove the value from this link+value
+    shard.entries[entryidx] = [entry[0], [entry[1][0]]]
+  } else {
+    shard.entries.splice(entryidx, 1)
+    // if now empty, remove from parent
+    while (!shard.entries.length) {
+      const child = path[path.length - 1]
+      const parent = path[path.length - 2]
+      if (!parent) break
+      path.pop()
+      shard = Shard.withEntries(
+        parent.value.entries.filter(e => {
+          if (!Array.isArray(e[1])) return true
+          return e[1][0].toString() !== child.cid.toString()
+        }),
+        parent.value
+      )
+    }
+  }
+
+  let child = await Shard.encodeBlock(shard, path[path.length - 1].prefix)
+  additions.push(child)
+
+  // path is root -> shard, so work backwards, propagating the new shard CID
+  for (let i = path.length - 2; i >= 0; i--) {
+    const parent = path[i]
+    const key = child.prefix.slice(parent.prefix.length)
+    const value = Shard.withEntries(
+      parent.value.entries.map((entry) => {
+        const [k, v] = entry
+        if (k !== key) return entry
+        if (!Array.isArray(v)) throw new Error(`"${key}" is not a shard link in: ${parent.cid}`)
+        return /** @type {API.ShardEntry} */(v[1] == null ? [k, [child.cid]] : [k, [child.cid, v[1]]])
+      }),
+      parent.value
+    )
+
+    child = await Shard.encodeBlock(value, parent.prefix)
+    additions.push(child)
+  }
+
+  return { root: additions[additions.length - 1].cid, additions, removals }
+}
+
+/**
+ * List entries in the bucket.
+ *
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.ShardLink} root CID of the root node of the bucket.
+ * @param {object} [options]
+ * @param {string} [options.prefix]
+ * @returns {AsyncIterableIterator<API.ShardValueEntry>}
+ */
+export const entries = async function * (blocks, root, options = {}) {
+  const { prefix } = options
+  const shards = new ShardFetcher(blocks)
+  const rshard = await shards.get(root)
+
+  yield * (
+    /** @returns {AsyncIterableIterator<API.ShardValueEntry>} */
+    async function * ents (shard) {
+      for (const entry of shard.value.entries) {
+        const key = shard.prefix + entry[0]
+
+        if (Array.isArray(entry[1])) {
+          if (entry[1][1]) {
+            if (!prefix || (prefix && key.startsWith(prefix))) {
+              yield [key, entry[1][1]]
+            }
+          }
+
+          if (prefix) {
+            if (prefix.length <= key.length && !key.startsWith(prefix)) {
+              continue
+            }
+            if (prefix.length > key.length && !prefix.startsWith(key)) {
+              continue
+            }
+          }
+          yield * ents(await shards.get(entry[1][0], key))
+        } else {
+          if (prefix && !key.startsWith(prefix)) {
+            continue
+          }
+          yield [key, entry[1]]
+        }
+      }
+    }
+  )(rshard)
+}
+
+/**
+ * Traverse from the passed shard block to the target shard block using the
+ * passed key. All traversed shards are returned, starting with the passed
+ * shard and ending with the target.
+ *
+ * @param {ShardFetcher} shards
+ * @param {API.ShardBlockView} shard
+ * @param {string} key
+ * @returns {Promise<[API.ShardBlockView, ...Array<API.ShardBlockView>]>}
+ */
+const traverse = async (shards, shard, key) => {
+  for (const [k, v] of shard.value.entries) {
+    if (key === k) return [shard]
+    if (key.startsWith(k) && Array.isArray(v)) {
+      const path = await traverse(shards, await shards.get(v[0], shard.prefix + k), key.slice(k.length))
+      return [shard, ...path]
+    }
+  }
+  return [shard]
+}

package/src/merge.js

@@ -0,0 +1,43 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js'
+import { difference } from './diff.js'
+import { put, del } from './index.js'
+
+/**
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.ShardLink} base Merge base. Common parent of target DAGs.
+ * @param {API.ShardLink[]} targets Target DAGs to merge.
+ * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
+ */
+export const merge = async (blocks, base, targets) => {
+  const diffs = await Promise.all(targets.map(t => difference(blocks, base, t)))
+  const additions = new Map()
+  const removals = new Map()
+  /** @type {API.BlockFetcher} */
+  const fetcher = { get: cid => additions.get(cid.toString()) ?? blocks.get(cid) }
+
+  let root = base
+  for (const { keys } of diffs) {
+    for (const [k, v] of keys) {
+      let res
+      if (v[1] == null) {
+        res = await del(fetcher, root, k)
+      } else {
+        res = await put(fetcher, root, k, v[1])
+      }
+      for (const blk of res.removals) {
+        if (additions.has(blk.cid.toString())) {
+          additions.delete(blk.cid.toString())
+        } else {
+          removals.set(blk.cid.toString(), blk)
+        }
+      }
+      for (const blk of res.additions) {
+        additions.set(blk.cid.toString(), blk)
+      }
+      root = res.root
+    }
+  }
+
+  return { root, additions: [...additions.values()], removals: [...removals.values()] }
+}

package/src/shard.js

@@ -0,0 +1,248 @@
+import * as Link from 'multiformats/link'
+import { Block, encode, decode } from 'multiformats/block'
+import { sha256 } from 'multiformats/hashes/sha2'
+import * as dagCBOR from '@ipld/dag-cbor'
+import { tokensToLength } from 'cborg/length'
+import { Token, Type } from 'cborg'
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js'
+
+export const MaxKeyLength = 64
+export const MaxShardSize = 512 * 1024
+
+const CID_TAG = new Token(Type.tag, 42)
+
+/**
+ * @extends {Block<API.Shard, typeof dagCBOR.code, typeof sha256.code, 1>}
+ * @implements {API.ShardBlockView}
+ */
+export class ShardBlock extends Block {
+  /**
+   * @param {object} config
+   * @param {API.ShardLink} config.cid
+   * @param {API.Shard} config.value
+   * @param {Uint8Array} config.bytes
+   * @param {string} config.prefix
+   */
+  constructor ({ cid, value, bytes, prefix }) {
+    // @ts-expect-error
+    super({ cid, value, bytes })
+    this.prefix = prefix
+  }
+
+  /** @param {API.ShardOptions} [options] */
+  static create (options) {
+    return encodeBlock(create(options))
+  }
+}
+
+/**
+ * @param {API.ShardOptions} [options]
+ * @returns {API.Shard}
+ */
+export const create = (options) => ({ entries: [], ...configure(options) })
+
+/**
+ * @param {API.ShardOptions} [options]
+ * @returns {API.ShardConfig}
+ */
+export const configure = (options) => ({
+  maxSize: options?.maxSize ?? MaxShardSize,
+  maxKeyLength: options?.maxKeyLength ?? MaxKeyLength
+})
+
+/**
+ * @param {API.ShardEntry[]} entries
+ * @param {API.ShardOptions} [options]
+ * @returns {API.Shard}
+ */
+export const withEntries = (entries, options) => ({ ...create(options), entries })
+
+/** @type {WeakMap<Uint8Array, API.ShardBlockView>} */
+const decodeCache = new WeakMap()
+
+/**
+ * @param {API.Shard} value
+ * @param {string} [prefix]
+ * @returns {Promise<API.ShardBlockView>}
+ */
+export const encodeBlock = async (value, prefix) => {
+  const { cid, bytes } = await encode({ value, codec: dagCBOR, hasher: sha256 })
+  const block = new ShardBlock({ cid, value, bytes, prefix: prefix ?? '' })
+  decodeCache.set(block.bytes, block)
+  return block
+}
+
+/**
+ * @param {Uint8Array} bytes
+ * @param {string} [prefix]
+ * @returns {Promise<API.ShardBlockView>}
+ */
+export const decodeBlock = async (bytes, prefix) => {
+  const block = decodeCache.get(bytes)
+  if (block) return block
+  const { cid, value } = await decode({ bytes, codec: dagCBOR, hasher: sha256 })
+  if (!isShard(value)) throw new Error(`invalid shard: ${cid}`)
+  return new ShardBlock({ cid, value, bytes, prefix: prefix ?? '' })
+}
+
+/**
+ * @param {any} value
+ * @returns {value is API.Shard}
+ */
+export const isShard = (value) =>
+  value != null &&
+  typeof value === 'object' &&
+  Array.isArray(value.entries) &&
+  typeof value.maxSize === 'number' &&
+  typeof value.maxKeyLength === 'number'
+
+/**
+ * @param {any} value
+ * @returns {value is API.ShardLink}
+ */
+export const isShardLink = (value) =>
+  Link.isLink(value) &&
+  value.code === dagCBOR.code
+
+export class ShardFetcher {
+  /** @param {API.BlockFetcher} blocks */
+  constructor (blocks) {
+    this._blocks = blocks
+  }
+
+  /**
+   * @param {API.ShardLink} link
+   * @param {string} [prefix]
+   * @returns {Promise<API.ShardBlockView>}
+   */
+  async get (link, prefix = '') {
+    const block = await this._blocks.get(link)
+    if (!block) throw new Error(`missing block: ${link}`)
+    return decodeBlock(block.bytes, prefix)
+  }
+}
+
+/**
+ * @param {API.ShardEntry[]} target Entries to insert into.
+ * @param {API.ShardEntry} newEntry
+ * @returns {API.ShardEntry[]}
+ */
+export const putEntry = (target, newEntry) => {
+  /** @type {API.ShardEntry[]} */
+  const entries = []
+
+  for (const [i, entry] of target.entries()) {
+    const [k, v] = entry
+    if (newEntry[0] === k) {
+      // if new value is link to shard...
+      if (Array.isArray(newEntry[1])) {
+        // and old value is link to shard
+        // and old value is _also_ link to data
+        // and new value does not have link to data
+        // then preserve old data
+        if (Array.isArray(v) && v[1] != null && newEntry[1][1] == null) {
+          entries.push([k, [newEntry[1][0], v[1]]])
+        } else {
+          entries.push(newEntry)
+        }
+      } else {
+        // shard as well as value?
+        if (Array.isArray(v)) {
+          entries.push([k, [v[0], newEntry[1]]])
+        } else {
+          entries.push(newEntry)
+        }
+      }
+      for (let j = i + 1; j < target.length; j++) {
+        entries.push(target[j])
+      }
+      return entries
+    }
+    if (i === 0 && newEntry[0] < k) {
+      entries.push(newEntry)
+      for (let j = i; j < target.length; j++) {
+        entries.push(target[j])
+      }
+      return entries
+    }
+    if (i > 0 && newEntry[0] > target[i - 1][0] && newEntry[0] < k) {
+      entries.push(newEntry)
+      for (let j = i; j < target.length; j++) {
+        entries.push(target[j])
+      }
+      return entries
+    }
+    entries.push(entry)
+  }
+
+  entries.push(newEntry)
+  return entries
+}
+
+/**
+ * @param {API.ShardEntry[]} entries
+ * @param {string} skey Shard key to use as a base.
+ */
+export const findCommonPrefix = (entries, skey) => {
+  const startidx = entries.findIndex(([k]) => skey === k)
+  if (startidx === -1) throw new Error(`key not found in shard: ${skey}`)
+  let i = startidx
+  /** @type {string} */
+  let pfx
+  while (true) {
+    pfx = entries[i][0].slice(0, -1)
+    if (pfx.length) {
+      while (true) {
+        const matches = entries.filter(entry => entry[0].startsWith(pfx))
+        if (matches.length > 1) return { prefix: pfx, matches }
+        pfx = pfx.slice(0, -1)
+        if (!pfx.length) break
+      }
+    }
+    i++
+    if (i >= entries.length) {
+      i = 0
+    }
+    if (i === startidx) {
+      return
+    }
+  }
+}
+
+/** @param {API.Shard} shard */
+export const encodedLength = (shard) => {
+  let entriesLength = 0
+  for (const entry of shard.entries) {
+    entriesLength += entryEncodedLength(entry)
+  }
+  const tokens = [
+    new Token(Type.map, 3),
+    new Token(Type.string, 'entries'),
+    new Token(Type.array, shard.entries.length),
+    new Token(Type.string, 'maxKeyLength'),
+    new Token(Type.uint, shard.maxKeyLength),
+    new Token(Type.string, 'maxSize'),
+    new Token(Type.uint, shard.maxSize)
+  ]
+  return tokensToLength(tokens) + entriesLength
+}
+
+/** @param {API.ShardEntry} entry */
+const entryEncodedLength = entry => {
+  const tokens = [
+    new Token(Type.array, entry.length),
+    new Token(Type.string, entry[0])
+  ]
+  if (Array.isArray(entry[1])) {
+    tokens.push(new Token(Type.array, entry[1].length))
+    for (const link of entry[1]) {
+      tokens.push(CID_TAG)
+      tokens.push(new Token(Type.bytes, { length: link.byteLength + 1 }))
+    }
+  } else {
+    tokens.push(CID_TAG)
+    tokens.push(new Token(Type.bytes, { length: entry[1].byteLength + 1 }))
+  }
+  return tokensToLength(tokens)
+}