@fireproof/vendor 2.0.2 → 3.0.0

package/src/@web3-storage/pail/src/index.js

@@ -1,406 +0,0 @@
-// eslint-disable-next-line no-unused-vars
-import * as API from './api.js'
-import { ShardFetcher, isPrintableASCII } from './shard.js'
-import * as Shard from './shard.js'
-
-/**
- * Put a value (a CID) for the given key. If the key exists it's value is
- * overwritten.
- *
- * @param {API.BlockFetcher} blocks Bucket block storage.
- * @param {API.ShardLink} root CID of the root node of the bucket.
- * @param {string} key The key of the value to put.
- * @param {API.UnknownLink} value The value to put.
- * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
- */
-export const put = async (blocks, root, key, value) => {
-  const shards = new ShardFetcher(blocks)
-  const rshard = await shards.get(root)
-
-  if (rshard.value.keyChars !== Shard.KeyCharsASCII) {
-    throw new Error(`unsupported key character set: ${rshard.value.keyChars}`)
-  }
-  if (!isPrintableASCII(key)) {
-    throw new Error('key contains non-ASCII characters')
-  }
-  // ensure utf8 encoded key is smaller than max
-  if (new TextEncoder().encode(key).length > rshard.value.maxKeySize) {
-    throw new Error(`UTF-8 encoded key exceeds max size of ${rshard.value.maxKeySize} bytes`)
-  }
-
-  const path = await traverse(shards, rshard, key)
-  const target = path[path.length - 1]
-  const skey = key.slice(target.value.prefix.length) // key within the shard
-
-  /** @type {API.ShardEntry} */
-  let entry = [skey, value]
-  let targetEntries = [...target.value.entries]
-
-  /** @type {API.ShardBlockView[]} */
-  const additions = []
-
-  for (const [i, e] of targetEntries.entries()) {
-    const [k, v] = e
-
-    // is this just a replace?
-    if (k === skey) break
-
-    // do we need to shard this entry?
-    const shortest = k.length < skey.length ? k : skey
-    const other = shortest === k ? skey : k
-    let common = ''
-    for (const char of shortest) {
-      const next = common + char
-      if (!other.startsWith(next)) break
-      common = next
-    }
-    if (common.length) {
-      /** @type {API.ShardEntry[]} */
-      let entries = []
-
-      // if the existing entry key or new key is equal to the common prefix,
-      // then the existing value / new value needs to persist in the parent
-      // shard. Otherwise they persist in this new shard.
-      if (common !== skey) {
-        entries = Shard.putEntry(entries, [skey.slice(common.length), value])
-      }
-      if (common !== k) {
-        entries = Shard.putEntry(entries, [k.slice(common.length), v])
-      }
-
-      let child = await Shard.encodeBlock(
-        Shard.withEntries(entries, { ...target.value, prefix: target.value.prefix + common })
-      )
-      additions.push(child)
-  
-      // need to spread as access by index does not consider utf-16 surrogates
-      const commonChars = [...common]
-
-      // create parent shards for each character of the common prefix
-      for (let i = commonChars.length - 1; i > 0; i--) {
-        const parentConfig = { ...target.value, prefix: target.value.prefix + commonChars.slice(0, i).join('') }
-        /** @type {API.ShardEntryLinkValue | API.ShardEntryValueValue | API.ShardEntryLinkAndValueValue} */
-        let parentValue
-        // if the first iteration and the existing entry key is equal to the
-        // common prefix, then existing value needs to persist in this parent
-        if (i === commonChars.length - 1 && common === k) {
-          if (Array.isArray(v)) throw new Error('found a shard link when expecting a value')
-          parentValue = [child.cid, v]
-        } else if (i === commonChars.length - 1 && common === skey) {
-          parentValue = [child.cid, value]
-        } else {
-          parentValue = [child.cid]
-        }
-        const parent = await Shard.encodeBlock(Shard.withEntries([[commonChars[i], parentValue]], parentConfig))
-        additions.push(parent)
-        child = parent
-      }
-
-      // remove the sharded entry
-      targetEntries.splice(i, 1)
-
-      // create the entry that will be added to target
-      if (commonChars.length === 1 && common === k) {
-        if (Array.isArray(v)) throw new Error('found a shard link when expecting a value')
-        entry = [commonChars[0], [child.cid, v]]
-      } else if (commonChars.length === 1 && common === skey) {
-        entry = [commonChars[0], [child.cid, value]]
-      } else {
-        entry = [commonChars[0], [child.cid]]
-      }
-      break
-    }
-  }
-
-  const shard = Shard.withEntries(Shard.putEntry(targetEntries, entry), target.value)
-  let child = await Shard.encodeBlock(shard)
-
-  // if no change in the target then we're done
-  if (child.cid.toString() === target.cid.toString()) {
-    return { root, additions: [], removals: [] }
-  }
-
-  additions.push(child)
-
-  // path is root -> target, so work backwards, propagating the new shard CID
-  for (let i = path.length - 2; i >= 0; i--) {
-    const parent = path[i]
-    const key = child.value.prefix.slice(parent.value.prefix.length)
-    const value = Shard.withEntries(
-      parent.value.entries.map((entry) => {
-        const [k, v] = entry
-        if (k !== key) return entry
-        if (!Array.isArray(v)) throw new Error(`"${key}" is not a shard link in: ${parent.cid}`)
-        return /** @type {API.ShardEntry} */(v[1] == null ? [k, [child.cid]] : [k, [child.cid, v[1]]])
-      }),
-      parent.value
-    )
-
-    child = await Shard.encodeBlock(value)
-    additions.push(child)
-  }
-
-  return { root: additions[additions.length - 1].cid, additions, removals: path }
-}
-
-/**
- * Get the stored value for the given key from the bucket. If the key is not
- * found, `undefined` is returned.
- *
- * @param {API.BlockFetcher} blocks Bucket block storage.
- * @param {API.ShardLink} root CID of the root node of the bucket.
- * @param {string} key The key of the value to get.
- * @returns {Promise<API.UnknownLink | undefined>}
- */
-export const get = async (blocks, root, key) => {
-  const shards = new ShardFetcher(blocks)
-  const rshard = await shards.get(root)
-  const path = await traverse(shards, rshard, key)
-  const target = path[path.length - 1]
-  const skey = key.slice(target.value.prefix.length) // key within the shard
-  const entry = target.value.entries.find(([k]) => k === skey)
-  if (!entry) return
-  return Array.isArray(entry[1]) ? entry[1][1] : entry[1]
-}
-
-/**
- * Delete the value for the given key from the bucket. If the key is not found
- * no operation occurs.
- *
- * @param {API.BlockFetcher} blocks Bucket block storage.
- * @param {API.ShardLink} root CID of the root node of the bucket.
- * @param {string} key The key of the value to delete.
- * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
- */
-export const del = async (blocks, root, key) => {
-  const shards = new ShardFetcher(blocks)
-  const rshard = await shards.get(root)
-  const path = await traverse(shards, rshard, key)
-  const target = path[path.length - 1]
-  const skey = key.slice(target.value.prefix.length) // key within the shard
-
-  const entryidx = target.value.entries.findIndex(([k]) => k === skey)
-  if (entryidx === -1) return { root, additions: [], removals: [] }
-
-  const entry = target.value.entries[entryidx]
-  // cannot delete a shard (without data)
-  if (Array.isArray(entry[1]) && entry[1][1] == null) {
-    return { root, additions: [], removals: [] }
-  }
-
-  /** @type {API.ShardBlockView[]} */
-  const additions = []
-  /** @type {API.ShardBlockView[]} */
-  const removals = [...path]
-
-  let shard = Shard.withEntries([...target.value.entries], target.value)
-
-  if (Array.isArray(entry[1])) {
-    // remove the value from this link+value
-    shard.entries[entryidx] = [entry[0], [entry[1][0]]]
-  } else {
-    shard.entries.splice(entryidx, 1)
-    // if now empty, remove from parent
-    while (!shard.entries.length) {
-      const child = path[path.length - 1]
-      const parent = path[path.length - 2]
-      if (!parent) break
-      path.pop()
-      shard = Shard.withEntries(
-        parent.value.entries.filter(e => {
-          if (!Array.isArray(e[1])) return true
-          return e[1][0].toString() !== child.cid.toString()
-        }),
-        parent.value
-      )
-    }
-  }
-
-  let child = await Shard.encodeBlock(shard)
-  additions.push(child)
-
-  // path is root -> shard, so work backwards, propagating the new shard CID
-  for (let i = path.length - 2; i >= 0; i--) {
-    const parent = path[i]
-    const key = child.value.prefix.slice(parent.value.prefix.length)
-    const value = Shard.withEntries(
-      parent.value.entries.map((entry) => {
-        const [k, v] = entry
-        if (k !== key) return entry
-        if (!Array.isArray(v)) throw new Error(`"${key}" is not a shard link in: ${parent.cid}`)
-        return /** @type {API.ShardEntry} */(v[1] == null ? [k, [child.cid]] : [k, [child.cid, v[1]]])
-      }),
-      parent.value
-    )
-
-    child = await Shard.encodeBlock(value)
-    additions.push(child)
-  }
-
-  return { root: additions[additions.length - 1].cid, additions, removals }
-}
-
-/**
- * @param {API.EntriesOptions} [options]
- * @returns {options is API.KeyPrefixOption}
- */
-const isKeyPrefixOption = options => {
-  const opts = options ?? {}
-  return 'prefix' in opts && Boolean(opts.prefix)
-}
-
-/**
- * @param {API.EntriesOptions} [options]
- * @returns {options is API.KeyRangeOption}
- */
-const isKeyRangeOption = options => {
-  const opts = options ?? {}
-  return ('gt' in opts && Boolean(opts.gt)) || ('gte' in opts && Boolean(opts.gte)) || ('lt' in opts && Boolean(opts.lt)) || ('lte' in opts && Boolean(opts.lte))
-}
-
-/**
- * @param {API.KeyRangeOption} options
- * @returns {options is API.KeyLowerBoundRangeOption}
- */
-const isKeyLowerBoundRangeOption = options => ('gt' in options && Boolean(options.gt)) || ('gte' in options && Boolean(options.gte))
-
-/**
- * @param {API.KeyLowerBoundRangeOption} options
- * @returns {options is API.KeyLowerBoundRangeInclusiveOption}
- */
-const isKeyLowerBoundRangeInclusiveOption = options => 'gte' in options && Boolean(options.gte)
-
-/**
- * @param {API.KeyLowerBoundRangeOption} options
- * @returns {options is API.KeyLowerBoundRangeExclusiveOption}
- */
-const isKeyLowerBoundRangeExclusiveOption = options => 'gt' in options && Boolean(options.gt)
-
-/**
- * @param {API.KeyRangeOption} options
- * @returns {options is API.KeyUpperBoundRangeOption}
- */
-const isKeyUpperBoundRangeOption = options => ('lt' in options && Boolean(options.lt)) || ('lte' in options && Boolean(options.lte))
-
-/**
- * @param {API.KeyUpperBoundRangeOption} options
- * @returns {options is API.KeyUpperBoundRangeInclusiveOption}
- */
-const isKeyUpperBoundRangeInclusiveOption = options => 'lte' in options && Boolean(options.lte)
-
-/**
- * @param {API.KeyUpperBoundRangeOption} options
- * @returns {options is API.KeyUpperBoundRangeExclusiveOption}
- */
-const isKeyUpperBoundRangeExclusiveOption = options => 'lt' in options && Boolean(options.lt)
-
-/**
- * List entries in the bucket.
- *
- * @param {API.BlockFetcher} blocks Bucket block storage.
- * @param {API.ShardLink} root CID of the root node of the bucket.
- * @param {API.EntriesOptions} [options]
- * @returns {AsyncIterableIterator<API.ShardValueEntry>}
- */
-export const entries = async function * (blocks, root, options) {
-  const hasKeyPrefix = isKeyPrefixOption(options)
-  const hasKeyRange = isKeyRangeOption(options)
-  const hasKeyLowerBoundRange = hasKeyRange && isKeyLowerBoundRangeOption(options)
-  const hasKeyLowerBoundRangeInclusive = hasKeyLowerBoundRange && isKeyLowerBoundRangeInclusiveOption(options)
-  const hasKeyLowerBoundRangeExclusive = hasKeyLowerBoundRange && isKeyLowerBoundRangeExclusiveOption(options)
-  const hasKeyUpperBoundRange = hasKeyRange && isKeyUpperBoundRangeOption(options)
-  const hasKeyUpperBoundRangeInclusive = hasKeyUpperBoundRange && isKeyUpperBoundRangeInclusiveOption(options)
-  const hasKeyUpperBoundRangeExclusive = hasKeyUpperBoundRange && isKeyUpperBoundRangeExclusiveOption(options)
-  const hasKeyUpperAndLowerBoundRange = hasKeyLowerBoundRange && hasKeyUpperBoundRange
-
-  const shards = new ShardFetcher(blocks)
-  const rshard = await shards.get(root)
-
-  yield * (
-    /** @returns {AsyncIterableIterator<API.ShardValueEntry>} */
-    (async function* ents(shard) {
-      for (const entry of shard.value.entries) {
-        const key = shard.value.prefix + entry[0]
-
-        // if array, this is a link to a shard
-        if (Array.isArray(entry[1])) {
-          if (entry[1][1]) {
-            if (
-              (hasKeyPrefix && key.startsWith(options.prefix)) ||
-              (hasKeyUpperAndLowerBoundRange && (
-                ((hasKeyLowerBoundRangeExclusive && key > options.gt) || (hasKeyLowerBoundRangeInclusive && key >= options.gte)) &&
-                ((hasKeyUpperBoundRangeExclusive && key < options.lt) || (hasKeyUpperBoundRangeInclusive && key <= options.lte))
-              )) ||
-              (hasKeyLowerBoundRangeExclusive && key > options.gt) ||
-              (hasKeyLowerBoundRangeInclusive && key >= options.gte) ||
-              (hasKeyUpperBoundRangeExclusive && key < options.lt) ||
-              (hasKeyUpperBoundRangeInclusive && key <= options.lte) ||
-              (!hasKeyPrefix && !hasKeyRange)
-            ) {
-              yield [key, entry[1][1]]
-            }
-          }
-
-          if (hasKeyPrefix) {
-            if (options.prefix.length <= key.length && !key.startsWith(options.prefix)) {
-              continue
-            }
-            if (options.prefix.length > key.length && !options.prefix.startsWith(key)) {
-              continue
-            }
-          } else if (
-            (hasKeyLowerBoundRangeExclusive && (trunc(key, Math.min(key.length, options.gt.length)) < trunc(options.gt, Math.min(key.length, options.gt.length)))) ||
-            (hasKeyLowerBoundRangeInclusive && (trunc(key, Math.min(key.length, options.gte.length)) < trunc(options.gte, Math.min(key.length, options.gte.length)))) ||
-            (hasKeyUpperBoundRangeExclusive && (trunc(key, Math.min(key.length, options.lt.length)) > trunc(options.lt, Math.min(key.length, options.lt.length)))) ||
-            (hasKeyUpperBoundRangeInclusive && (trunc(key, Math.min(key.length, options.lte.length)) > trunc(options.lte, Math.min(key.length, options.lte.length))))
-          ) {
-            continue
-          }
-          yield * ents(await shards.get(entry[1][0]))
-        } else {
-          if (
-            (hasKeyPrefix && key.startsWith(options.prefix)) ||
-            (hasKeyRange && hasKeyUpperAndLowerBoundRange && (
-              ((hasKeyLowerBoundRangeExclusive && key > options.gt) || (hasKeyLowerBoundRangeInclusive && key >= options.gte)) &&
-              ((hasKeyUpperBoundRangeExclusive && key < options.lt) || (hasKeyUpperBoundRangeInclusive && key <= options.lte))
-            )) ||
-            (hasKeyRange && !hasKeyUpperAndLowerBoundRange && (
-              (hasKeyLowerBoundRangeExclusive && key > options.gt) || (hasKeyLowerBoundRangeInclusive && key >= options.gte) ||
-              (hasKeyUpperBoundRangeExclusive && key < options.lt) || (hasKeyUpperBoundRangeInclusive && key <= options.lte)
-            )) ||
-            (!hasKeyPrefix && !hasKeyRange)
-          ) {
-            yield [key, entry[1]]
-          }
-        }
-      }
-    })
-  )(rshard)
-}
-
-/**
- * @param {string} str
- * @param {number} len
- */
-const trunc = (str, len) => str.length <= len ? str : str.slice(0, len)
-
-/**
- * Traverse from the passed shard block to the target shard block using the
- * passed key. All traversed shards are returned, starting with the passed
- * shard and ending with the target.
- *
- * @param {ShardFetcher} shards
- * @param {API.ShardBlockView} shard
- * @param {string} key
- * @returns {Promise<[API.ShardBlockView, ...Array<API.ShardBlockView>]>}
- */
-const traverse = async (shards, shard, key) => {
-  for (const [k, v] of shard.value.entries) {
-    if (key === k) return [shard]
-    if (key.startsWith(k) && Array.isArray(v)) {
-      const path = await traverse(shards, await shards.get(v[0]), key.slice(k.length))
-      return [shard, ...path]
-    }
-  }
-  return [shard]
-}

package/src/@web3-storage/pail/src/merge.js

@@ -1,43 +0,0 @@
-// eslint-disable-next-line no-unused-vars
-import * as API from './api.js'
-import { difference } from './diff.js'
-import { put, del } from './index.js'
-
-/**
- * @param {API.BlockFetcher} blocks Bucket block storage.
- * @param {API.ShardLink} base Merge base. Common parent of target DAGs.
- * @param {API.ShardLink[]} targets Target DAGs to merge.
- * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
- */
-export const merge = async (blocks, base, targets) => {
-  const diffs = await Promise.all(targets.map(t => difference(blocks, base, t)))
-  const additions = new Map()
-  const removals = new Map()
-  /** @type {API.BlockFetcher} */
-  const fetcher = { get: cid => additions.get(cid.toString()) ?? blocks.get(cid) }
-
-  let root = base
-  for (const { keys } of diffs) {
-    for (const [k, v] of keys) {
-      let res
-      if (v[1] == null) {
-        res = await del(fetcher, root, k)
-      } else {
-        res = await put(fetcher, root, k, v[1])
-      }
-      for (const blk of res.removals) {
-        if (additions.has(blk.cid.toString())) {
-          additions.delete(blk.cid.toString())
-        } else {
-          removals.set(blk.cid.toString(), blk)
-        }
-      }
-      for (const blk of res.additions) {
-        additions.set(blk.cid.toString(), blk)
-      }
-      root = res.root
-    }
-  }
-
-  return { root, additions: [...additions.values()], removals: [...removals.values()] }
-}

package/src/@web3-storage/pail/src/shard.js

@@ -1,180 +0,0 @@
-import * as Link from 'multiformats/link'
-import { Block, encode, decode } from 'multiformats/block'
-import { sha256 } from 'multiformats/hashes/sha2'
-import * as dagCBOR from '@ipld/dag-cbor'
-// eslint-disable-next-line no-unused-vars
-import * as API from './api.js'
-
-export const KeyCharsASCII = 'ascii'
-export const MaxKeySize = 4096
-
-/**
- * @extends {Block<API.Shard, typeof dagCBOR.code, typeof sha256.code, 1>}
- * @implements {API.ShardBlockView}
- */
-export class ShardBlock extends Block {
-  /**
-   * @param {object} config
-   * @param {API.ShardLink} config.cid
-   * @param {API.Shard} config.value
-   * @param {Uint8Array} config.bytes
-   */
-  constructor ({ cid, value, bytes }) {
-    // @ts-expect-error
-    super({ cid, value, bytes })
-  }
-
-  /** @param {API.ShardOptions} [options] */
-  static create (options) {
-    return encodeBlock(create(options))
-  }
-}
-
-/**
- * @param {API.ShardOptions} [options]
- * @returns {API.Shard}
- */
-export const create = (options) => ({ entries: [], ...configure(options) })
-
-/**
- * @param {API.ShardOptions} [options]
- * @returns {API.ShardConfig}
- */
-export const configure = (options) => ({
-  version: 1,
-  keyChars: options?.keyChars ?? KeyCharsASCII,
-  maxKeySize: options?.maxKeySize ?? MaxKeySize,
-  prefix: options?.prefix ?? ''
-})
-
-/**
- * @param {API.ShardEntry[]} entries
- * @param {API.ShardOptions} [options]
- * @returns {API.Shard}
- */
-export const withEntries = (entries, options) => ({ ...create(options), entries })
-
-/** @type {WeakMap<Uint8Array, API.ShardBlockView>} */
-const decodeCache = new WeakMap()
-
-/**
- * @param {API.Shard} value
- * @returns {Promise<API.ShardBlockView>}
- */
-export const encodeBlock = async value => {
-  const { cid, bytes } = await encode({ value, codec: dagCBOR, hasher: sha256 })
-  const block = new ShardBlock({ cid, value, bytes })
-  decodeCache.set(block.bytes, block)
-  return block
-}
-
-/**
- * @param {Uint8Array} bytes
- * @returns {Promise<API.ShardBlockView>}
- */
-export const decodeBlock = async bytes => {
-  const block = decodeCache.get(bytes)
-  if (block) return block
-  const { cid, value } = await decode({ bytes, codec: dagCBOR, hasher: sha256 })
-  if (!isShard(value)) throw new Error(`invalid shard: ${cid}`)
-  return new ShardBlock({ cid, value, bytes })
-}
-
-/**
- * @param {any} value
- * @returns {value is API.Shard}
- */
-export const isShard = value =>
-  value != null &&
-  typeof value === 'object' &&
-  Array.isArray(value.entries) &&
-  value.version === 1 &&
-  typeof value.maxKeySize === 'number' &&
-  typeof value.keyChars === 'string' &&
-  typeof value.prefix === 'string'
-
-/**
- * @param {any} value
- * @returns {value is API.ShardLink}
- */
-export const isShardLink = (value) =>
-  Link.isLink(value) &&
-  value.code === dagCBOR.code
-
-export class ShardFetcher {
-  /** @param {API.BlockFetcher} blocks */
-  constructor (blocks) {
-    this._blocks = blocks
-  }
-
-  /**
-   * @param {API.ShardLink} link
-   * @returns {Promise<API.ShardBlockView>}
-   */
-  async get (link) {
-    const block = await this._blocks.get(link)
-    if (!block) throw new Error(`missing block: ${link}`)
-    return decodeBlock(block.bytes)
-  }
-}
-
-/**
- * @param {API.ShardEntry[]} target Entries to insert into.
- * @param {API.ShardEntry} newEntry
- * @returns {API.ShardEntry[]}
- */
-export const putEntry = (target, newEntry) => {
-  /** @type {API.ShardEntry[]} */
-  const entries = []
-
-  for (const [i, entry] of target.entries()) {
-    const [k, v] = entry
-    if (newEntry[0] === k) {
-      // if new value is link to shard...
-      if (Array.isArray(newEntry[1])) {
-        // and old value is link to shard
-        // and old value is _also_ link to data
-        // and new value does not have link to data
-        // then preserve old data
-        if (Array.isArray(v) && v[1] != null && newEntry[1][1] == null) {
-          entries.push([k, [newEntry[1][0], v[1]]])
-        } else {
-          entries.push(newEntry)
-        }
-      } else {
-        // shard as well as value?
-        if (Array.isArray(v)) {
-          entries.push([k, [v[0], newEntry[1]]])
-        } else {
-          entries.push(newEntry)
-        }
-      }
-      for (let j = i + 1; j < target.length; j++) {
-        entries.push(target[j])
-      }
-      return entries
-    }
-    if (i === 0 && newEntry[0] < k) {
-      entries.push(newEntry)
-      for (let j = i; j < target.length; j++) {
-        entries.push(target[j])
-      }
-      return entries
-    }
-    if (i > 0 && newEntry[0] > target[i - 1][0] && newEntry[0] < k) {
-      entries.push(newEntry)
-      for (let j = i; j < target.length; j++) {
-        entries.push(target[j])
-      }
-      return entries
-    }
-    entries.push(entry)
-  }
-
-  entries.push(newEntry)
-
-  return entries
-}
-
-/** @param {string} s */
-export const isPrintableASCII = s => /^[\x20-\x7E]*$/.test(s)