@fireproof/vendor 0.0.0-smoke → 0.0.2-dev-test

package/src/@web3-storage/pail/pail/src/diff.js

@@ -1,151 +0,0 @@
-// eslint-disable-next-line no-unused-vars
-import * as API from './api.js'
-import { ShardFetcher } from './shard.js'
-
-/**
- * @typedef {string} K
- * @typedef {[before: null, after: API.UnknownLink]} AddV
- * @typedef {[before: API.UnknownLink, after: API.UnknownLink]} UpdateV
- * @typedef {[before: API.UnknownLink, after: null]} DeleteV
- * @typedef {[key: K, value: AddV|UpdateV|DeleteV]} KV
- * @typedef {KV[]} KeysDiff
- * @typedef {{ keys: KeysDiff, shards: API.ShardDiff }} CombinedDiff
- */
-
-/**
- * @param {API.BlockFetcher} blocks Bucket block storage.
- * @param {API.ShardLink} a Base DAG.
- * @param {API.ShardLink} b Comparison DAG.
- * @returns {Promise<CombinedDiff>}
- */
-export const difference = async (blocks, a, b) => {
-  if (isEqual(a, b)) return { keys: [], shards: { additions: [], removals: [] } }
-
-  const shards = new ShardFetcher(blocks)
-  const [ashard, bshard] = await Promise.all([shards.get(a), shards.get(b)])
-
-  const aents = new Map(ashard.value.entries)
-  const bents = new Map(bshard.value.entries)
-
-  const keys = /** @type {Map<K, AddV|UpdateV|DeleteV>} */(new Map())
-  const additions = new Map([[bshard.cid.toString(), bshard]])
-  const removals = new Map([[ashard.cid.toString(), ashard]])
-
-  // find shards removed in B
-  for (const [akey, aval] of ashard.value.entries) {
-    const bval = bents.get(akey)
-    if (bval) continue
-    if (!Array.isArray(aval)) {
-      keys.set(`${ashard.value.prefix}${akey}`, [aval, null])
-      continue
-    }
-    // if shard link _with_ value
-    if (aval[1] != null) {
-      keys.set(`${ashard.value.prefix}${akey}`, [aval[1], null])
-    }
-    for await (const s of collect(shards, aval[0])) {
-      for (const [k, v] of s.value.entries) {
-        if (!Array.isArray(v)) {
-          keys.set(`${s.value.prefix}${k}`, [v, null])
-        } else if (v[1] != null) {
-          keys.set(`${s.value.prefix}${k}`, [v[1], null])
-        }
-      }
-      removals.set(s.cid.toString(), s)
-    }
-  }
-
-  // find shards added or updated in B
-  for (const [bkey, bval] of bshard.value.entries) {
-    const aval = aents.get(bkey)
-    if (!Array.isArray(bval)) {
-      if (!aval) {
-        keys.set(`${bshard.value.prefix}${bkey}`, [null, bval])
-      } else if (Array.isArray(aval)) {
-        keys.set(`${bshard.value.prefix}${bkey}`, [aval[1] ?? null, bval])
-      } else if (!isEqual(aval, bval)) {
-        keys.set(`${bshard.value.prefix}${bkey}`, [aval, bval])
-      }
-      continue
-    }
-    if (aval && Array.isArray(aval)) { // updated in B
-      if (isEqual(aval[0], bval[0])) {
-        if (bval[1] != null && (aval[1] == null || !isEqual(aval[1], bval[1]))) {
-          keys.set(`${bshard.value.prefix}${bkey}`, [aval[1] ?? null, bval[1]])
-        }
-        continue // updated value?
-      }
-      const res = await difference(blocks, aval[0], bval[0])
-      for (const shard of res.shards.additions) {
-        additions.set(shard.cid.toString(), shard)
-      }
-      for (const shard of res.shards.removals) {
-        removals.set(shard.cid.toString(), shard)
-      }
-      for (const [k, v] of res.keys) {
-        keys.set(k, v)
-      }
-    } else if (aval) { // updated in B value => link+value
-      if (bval[1] == null) {
-        keys.set(`${bshard.value.prefix}${bkey}`, [aval, null])
-      } else if (!isEqual(aval, bval[1])) {
-        keys.set(`${bshard.value.prefix}${bkey}`, [aval, bval[1]])
-      }
-      for await (const s of collect(shards, bval[0])) {
-        for (const [k, v] of s.value.entries) {
-          if (!Array.isArray(v)) {
-            keys.set(`${s.value.prefix}${k}`, [null, v])
-          } else if (v[1] != null) {
-            keys.set(`${s.value.prefix}${k}`, [null, v[1]])
-          }
-        }
-        additions.set(s.cid.toString(), s)
-      }
-    } else { // added in B
-      keys.set(`${bshard.value.prefix}${bkey}`, [null, bval[0]])
-      for await (const s of collect(shards, bval[0])) {
-        for (const [k, v] of s.value.entries) {
-          if (!Array.isArray(v)) {
-            keys.set(`${s.value.prefix}${k}`, [null, v])
-          } else if (v[1] != null) {
-            keys.set(`${s.value.prefix}${k}`, [null, v[1]])
-          }
-        }
-        additions.set(s.cid.toString(), s)
-      }
-    }
-  }
-
-  // filter blocks that were added _and_ removed from B
-  for (const k of removals.keys()) {
-    if (additions.has(k)) {
-      additions.delete(k)
-      removals.delete(k)
-    }
-  }
-
-  return {
-    keys: [...keys.entries()].sort((a, b) => a[0] < b[0] ? -1 : 1),
-    shards: { additions: [...additions.values()], removals: [...removals.values()] }
-  }
-}
-
-/**
- * @param {API.UnknownLink} a
- * @param {API.UnknownLink} b
- */
-const isEqual = (a, b) => a.toString() === b.toString()
-
-/**
- * @param {import('./shard.js').ShardFetcher} shards
- * @param {API.ShardLink} root
- * @returns {AsyncIterableIterator<API.ShardBlockView>}
- */
-async function * collect (shards, root) {
-  const shard = await shards.get(root)
-  yield shard
-  for (const [, v] of shard.value.entries) {
-    if (!Array.isArray(v)) continue
-    yield * collect(shards, v[0])
-  }
-}

package/src/@web3-storage/pail/pail/src/index.js

@@ -1,406 +0,0 @@
-// eslint-disable-next-line no-unused-vars
-import * as API from './api.js'
-import { ShardFetcher, isPrintableASCII } from './shard.js'
-import * as Shard from './shard.js'
-
-/**
- * Put a value (a CID) for the given key. If the key exists it's value is
- * overwritten.
- *
- * @param {API.BlockFetcher} blocks Bucket block storage.
- * @param {API.ShardLink} root CID of the root node of the bucket.
- * @param {string} key The key of the value to put.
- * @param {API.UnknownLink} value The value to put.
- * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
- */
-export const put = async (blocks, root, key, value) => {
-  const shards = new ShardFetcher(blocks)
-  const rshard = await shards.get(root)
-
-  if (rshard.value.keyChars !== Shard.KeyCharsASCII) {
-    throw new Error(`unsupported key character set: ${rshard.value.keyChars}`)
-  }
-  if (!isPrintableASCII(key)) {
-    throw new Error('key contains non-ASCII characters')
-  }
-  // ensure utf8 encoded key is smaller than max
-  if (new TextEncoder().encode(key).length > rshard.value.maxKeySize) {
-    throw new Error(`UTF-8 encoded key exceeds max size of ${rshard.value.maxKeySize} bytes`)
-  }
-
-  const path = await traverse(shards, rshard, key)
-  const target = path[path.length - 1]
-  const skey = key.slice(target.value.prefix.length) // key within the shard
-
-  /** @type {API.ShardEntry} */
-  let entry = [skey, value]
-  let targetEntries = [...target.value.entries]
-
-  /** @type {API.ShardBlockView[]} */
-  const additions = []
-
-  for (const [i, e] of targetEntries.entries()) {
-    const [k, v] = e
-
-    // is this just a replace?
-    if (k === skey) break
-
-    // do we need to shard this entry?
-    const shortest = k.length < skey.length ? k : skey
-    const other = shortest === k ? skey : k
-    let common = ''
-    for (const char of shortest) {
-      const next = common + char
-      if (!other.startsWith(next)) break
-      common = next
-    }
-    if (common.length) {
-      /** @type {API.ShardEntry[]} */
-      let entries = []
-
-      // if the existing entry key or new key is equal to the common prefix,
-      // then the existing value / new value needs to persist in the parent
-      // shard. Otherwise they persist in this new shard.
-      if (common !== skey) {
-        entries = Shard.putEntry(entries, [skey.slice(common.length), value])
-      }
-      if (common !== k) {
-        entries = Shard.putEntry(entries, [k.slice(common.length), v])
-      }
-
-      let child = await Shard.encodeBlock(
-        Shard.withEntries(entries, { ...target.value, prefix: target.value.prefix + common })
-      )
-      additions.push(child)
-  
-      // need to spread as access by index does not consider utf-16 surrogates
-      const commonChars = [...common]
-
-      // create parent shards for each character of the common prefix
-      for (let i = commonChars.length - 1; i > 0; i--) {
-        const parentConfig = { ...target.value, prefix: target.value.prefix + commonChars.slice(0, i).join('') }
-        /** @type {API.ShardEntryLinkValue | API.ShardEntryValueValue | API.ShardEntryLinkAndValueValue} */
-        let parentValue
-        // if the first iteration and the existing entry key is equal to the
-        // common prefix, then existing value needs to persist in this parent
-        if (i === commonChars.length - 1 && common === k) {
-          if (Array.isArray(v)) throw new Error('found a shard link when expecting a value')
-          parentValue = [child.cid, v]
-        } else if (i === commonChars.length - 1 && common === skey) {
-          parentValue = [child.cid, value]
-        } else {
-          parentValue = [child.cid]
-        }
-        const parent = await Shard.encodeBlock(Shard.withEntries([[commonChars[i], parentValue]], parentConfig))
-        additions.push(parent)
-        child = parent
-      }
-
-      // remove the sharded entry
-      targetEntries.splice(i, 1)
-
-      // create the entry that will be added to target
-      if (commonChars.length === 1 && common === k) {
-        if (Array.isArray(v)) throw new Error('found a shard link when expecting a value')
-        entry = [commonChars[0], [child.cid, v]]
-      } else if (commonChars.length === 1 && common === skey) {
-        entry = [commonChars[0], [child.cid, value]]
-      } else {
-        entry = [commonChars[0], [child.cid]]
-      }
-      break
-    }
-  }
-
-  const shard = Shard.withEntries(Shard.putEntry(targetEntries, entry), target.value)
-  let child = await Shard.encodeBlock(shard)
-
-  // if no change in the target then we're done
-  if (child.cid.toString() === target.cid.toString()) {
-    return { root, additions: [], removals: [] }
-  }
-
-  additions.push(child)
-
-  // path is root -> target, so work backwards, propagating the new shard CID
-  for (let i = path.length - 2; i >= 0; i--) {
-    const parent = path[i]
-    const key = child.value.prefix.slice(parent.value.prefix.length)
-    const value = Shard.withEntries(
-      parent.value.entries.map((entry) => {
-        const [k, v] = entry
-        if (k !== key) return entry
-        if (!Array.isArray(v)) throw new Error(`"${key}" is not a shard link in: ${parent.cid}`)
-        return /** @type {API.ShardEntry} */(v[1] == null ? [k, [child.cid]] : [k, [child.cid, v[1]]])
-      }),
-      parent.value
-    )
-
-    child = await Shard.encodeBlock(value)
-    additions.push(child)
-  }
-
-  return { root: additions[additions.length - 1].cid, additions, removals: path }
-}
-
-/**
- * Get the stored value for the given key from the bucket. If the key is not
- * found, `undefined` is returned.
- *
- * @param {API.BlockFetcher} blocks Bucket block storage.
- * @param {API.ShardLink} root CID of the root node of the bucket.
- * @param {string} key The key of the value to get.
- * @returns {Promise<API.UnknownLink | undefined>}
- */
-export const get = async (blocks, root, key) => {
-  const shards = new ShardFetcher(blocks)
-  const rshard = await shards.get(root)
-  const path = await traverse(shards, rshard, key)
-  const target = path[path.length - 1]
-  const skey = key.slice(target.value.prefix.length) // key within the shard
-  const entry = target.value.entries.find(([k]) => k === skey)
-  if (!entry) return
-  return Array.isArray(entry[1]) ? entry[1][1] : entry[1]
-}
-
-/**
- * Delete the value for the given key from the bucket. If the key is not found
- * no operation occurs.
- *
- * @param {API.BlockFetcher} blocks Bucket block storage.
- * @param {API.ShardLink} root CID of the root node of the bucket.
- * @param {string} key The key of the value to delete.
- * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
- */
-export const del = async (blocks, root, key) => {
-  const shards = new ShardFetcher(blocks)
-  const rshard = await shards.get(root)
-  const path = await traverse(shards, rshard, key)
-  const target = path[path.length - 1]
-  const skey = key.slice(target.value.prefix.length) // key within the shard
-
-  const entryidx = target.value.entries.findIndex(([k]) => k === skey)
-  if (entryidx === -1) return { root, additions: [], removals: [] }
-
-  const entry = target.value.entries[entryidx]
-  // cannot delete a shard (without data)
-  if (Array.isArray(entry[1]) && entry[1][1] == null) {
-    return { root, additions: [], removals: [] }
-  }
-
-  /** @type {API.ShardBlockView[]} */
-  const additions = []
-  /** @type {API.ShardBlockView[]} */
-  const removals = [...path]
-
-  let shard = Shard.withEntries([...target.value.entries], target.value)
-
-  if (Array.isArray(entry[1])) {
-    // remove the value from this link+value
-    shard.entries[entryidx] = [entry[0], [entry[1][0]]]
-  } else {
-    shard.entries.splice(entryidx, 1)
-    // if now empty, remove from parent
-    while (!shard.entries.length) {
-      const child = path[path.length - 1]
-      const parent = path[path.length - 2]
-      if (!parent) break
-      path.pop()
-      shard = Shard.withEntries(
-        parent.value.entries.filter(e => {
-          if (!Array.isArray(e[1])) return true
-          return e[1][0].toString() !== child.cid.toString()
-        }),
-        parent.value
-      )
-    }
-  }
-
-  let child = await Shard.encodeBlock(shard)
-  additions.push(child)
-
-  // path is root -> shard, so work backwards, propagating the new shard CID
-  for (let i = path.length - 2; i >= 0; i--) {
-    const parent = path[i]
-    const key = child.value.prefix.slice(parent.value.prefix.length)
-    const value = Shard.withEntries(
-      parent.value.entries.map((entry) => {
-        const [k, v] = entry
-        if (k !== key) return entry
-        if (!Array.isArray(v)) throw new Error(`"${key}" is not a shard link in: ${parent.cid}`)
-        return /** @type {API.ShardEntry} */(v[1] == null ? [k, [child.cid]] : [k, [child.cid, v[1]]])
-      }),
-      parent.value
-    )
-
-    child = await Shard.encodeBlock(value)
-    additions.push(child)
-  }
-
-  return { root: additions[additions.length - 1].cid, additions, removals }
-}
-
-/**
- * @param {API.EntriesOptions} [options]
- * @returns {options is API.KeyPrefixOption}
- */
-const isKeyPrefixOption = options => {
-  const opts = options ?? {}
-  return 'prefix' in opts && Boolean(opts.prefix)
-}
-
-/**
- * @param {API.EntriesOptions} [options]
- * @returns {options is API.KeyRangeOption}
- */
-const isKeyRangeOption = options => {
-  const opts = options ?? {}
-  return ('gt' in opts && Boolean(opts.gt)) || ('gte' in opts && Boolean(opts.gte)) || ('lt' in opts && Boolean(opts.lt)) || ('lte' in opts && Boolean(opts.lte))
-}
-
-/**
- * @param {API.KeyRangeOption} options
- * @returns {options is API.KeyLowerBoundRangeOption}
- */
-const isKeyLowerBoundRangeOption = options => ('gt' in options && Boolean(options.gt)) || ('gte' in options && Boolean(options.gte))
-
-/**
- * @param {API.KeyLowerBoundRangeOption} options
- * @returns {options is API.KeyLowerBoundRangeInclusiveOption}
- */
-const isKeyLowerBoundRangeInclusiveOption = options => 'gte' in options && Boolean(options.gte)
-
-/**
- * @param {API.KeyLowerBoundRangeOption} options
- * @returns {options is API.KeyLowerBoundRangeExclusiveOption}
- */
-const isKeyLowerBoundRangeExclusiveOption = options => 'gt' in options && Boolean(options.gt)
-
-/**
- * @param {API.KeyRangeOption} options
- * @returns {options is API.KeyUpperBoundRangeOption}
- */
-const isKeyUpperBoundRangeOption = options => ('lt' in options && Boolean(options.lt)) || ('lte' in options && Boolean(options.lte))
-
-/**
- * @param {API.KeyUpperBoundRangeOption} options
- * @returns {options is API.KeyUpperBoundRangeInclusiveOption}
- */
-const isKeyUpperBoundRangeInclusiveOption = options => 'lte' in options && Boolean(options.lte)
-
-/**
- * @param {API.KeyUpperBoundRangeOption} options
- * @returns {options is API.KeyUpperBoundRangeExclusiveOption}
- */
-const isKeyUpperBoundRangeExclusiveOption = options => 'lt' in options && Boolean(options.lt)
-
-/**
- * List entries in the bucket.
- *
- * @param {API.BlockFetcher} blocks Bucket block storage.
- * @param {API.ShardLink} root CID of the root node of the bucket.
- * @param {API.EntriesOptions} [options]
- * @returns {AsyncIterableIterator<API.ShardValueEntry>}
- */
-export const entries = async function * (blocks, root, options) {
-  const hasKeyPrefix = isKeyPrefixOption(options)
-  const hasKeyRange = isKeyRangeOption(options)
-  const hasKeyLowerBoundRange = hasKeyRange && isKeyLowerBoundRangeOption(options)
-  const hasKeyLowerBoundRangeInclusive = hasKeyLowerBoundRange && isKeyLowerBoundRangeInclusiveOption(options)
-  const hasKeyLowerBoundRangeExclusive = hasKeyLowerBoundRange && isKeyLowerBoundRangeExclusiveOption(options)
-  const hasKeyUpperBoundRange = hasKeyRange && isKeyUpperBoundRangeOption(options)
-  const hasKeyUpperBoundRangeInclusive = hasKeyUpperBoundRange && isKeyUpperBoundRangeInclusiveOption(options)
-  const hasKeyUpperBoundRangeExclusive = hasKeyUpperBoundRange && isKeyUpperBoundRangeExclusiveOption(options)
-  const hasKeyUpperAndLowerBoundRange = hasKeyLowerBoundRange && hasKeyUpperBoundRange
-
-  const shards = new ShardFetcher(blocks)
-  const rshard = await shards.get(root)
-
-  yield * (
-    /** @returns {AsyncIterableIterator<API.ShardValueEntry>} */
-    (async function* ents(shard) {
-      for (const entry of shard.value.entries) {
-        const key = shard.value.prefix + entry[0]
-
-        // if array, this is a link to a shard
-        if (Array.isArray(entry[1])) {
-          if (entry[1][1]) {
-            if (
-              (hasKeyPrefix && key.startsWith(options.prefix)) ||
-              (hasKeyUpperAndLowerBoundRange && (
-                ((hasKeyLowerBoundRangeExclusive && key > options.gt) || (hasKeyLowerBoundRangeInclusive && key >= options.gte)) &&
-                ((hasKeyUpperBoundRangeExclusive && key < options.lt) || (hasKeyUpperBoundRangeInclusive && key <= options.lte))
-              )) ||
-              (hasKeyLowerBoundRangeExclusive && key > options.gt) ||
-              (hasKeyLowerBoundRangeInclusive && key >= options.gte) ||
-              (hasKeyUpperBoundRangeExclusive && key < options.lt) ||
-              (hasKeyUpperBoundRangeInclusive && key <= options.lte) ||
-              (!hasKeyPrefix && !hasKeyRange)
-            ) {
-              yield [key, entry[1][1]]
-            }
-          }
-
-          if (hasKeyPrefix) {
-            if (options.prefix.length <= key.length && !key.startsWith(options.prefix)) {
-              continue
-            }
-            if (options.prefix.length > key.length && !options.prefix.startsWith(key)) {
-              continue
-            }
-          } else if (
-            (hasKeyLowerBoundRangeExclusive && (trunc(key, Math.min(key.length, options.gt.length)) < trunc(options.gt, Math.min(key.length, options.gt.length)))) ||
-            (hasKeyLowerBoundRangeInclusive && (trunc(key, Math.min(key.length, options.gte.length)) < trunc(options.gte, Math.min(key.length, options.gte.length)))) ||
-            (hasKeyUpperBoundRangeExclusive && (trunc(key, Math.min(key.length, options.lt.length)) > trunc(options.lt, Math.min(key.length, options.lt.length)))) ||
-            (hasKeyUpperBoundRangeInclusive && (trunc(key, Math.min(key.length, options.lte.length)) > trunc(options.lte, Math.min(key.length, options.lte.length))))
-          ) {
-            continue
-          }
-          yield * ents(await shards.get(entry[1][0]))
-        } else {
-          if (
-            (hasKeyPrefix && key.startsWith(options.prefix)) ||
-            (hasKeyRange && hasKeyUpperAndLowerBoundRange && (
-              ((hasKeyLowerBoundRangeExclusive && key > options.gt) || (hasKeyLowerBoundRangeInclusive && key >= options.gte)) &&
-              ((hasKeyUpperBoundRangeExclusive && key < options.lt) || (hasKeyUpperBoundRangeInclusive && key <= options.lte))
-            )) ||
-            (hasKeyRange && !hasKeyUpperAndLowerBoundRange && (
-              (hasKeyLowerBoundRangeExclusive && key > options.gt) || (hasKeyLowerBoundRangeInclusive && key >= options.gte) ||
-              (hasKeyUpperBoundRangeExclusive && key < options.lt) || (hasKeyUpperBoundRangeInclusive && key <= options.lte)
-            )) ||
-            (!hasKeyPrefix && !hasKeyRange)
-          ) {
-            yield [key, entry[1]]
-          }
-        }
-      }
-    })
-  )(rshard)
-}
-
-/**
- * @param {string} str
- * @param {number} len
- */
-const trunc = (str, len) => str.length <= len ? str : str.slice(0, len)
-
-/**
- * Traverse from the passed shard block to the target shard block using the
- * passed key. All traversed shards are returned, starting with the passed
- * shard and ending with the target.
- *
- * @param {ShardFetcher} shards
- * @param {API.ShardBlockView} shard
- * @param {string} key
- * @returns {Promise<[API.ShardBlockView, ...Array<API.ShardBlockView>]>}
- */
-const traverse = async (shards, shard, key) => {
-  for (const [k, v] of shard.value.entries) {
-    if (key === k) return [shard]
-    if (key.startsWith(k) && Array.isArray(v)) {
-      const path = await traverse(shards, await shards.get(v[0]), key.slice(k.length))
-      return [shard, ...path]
-    }
-  }
-  return [shard]
-}

package/src/@web3-storage/pail/pail/src/merge.js

@@ -1,43 +0,0 @@
-// eslint-disable-next-line no-unused-vars
-import * as API from './api.js'
-import { difference } from './diff.js'
-import { put, del } from './index.js'
-
-/**
- * @param {API.BlockFetcher} blocks Bucket block storage.
- * @param {API.ShardLink} base Merge base. Common parent of target DAGs.
- * @param {API.ShardLink[]} targets Target DAGs to merge.
- * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
- */
-export const merge = async (blocks, base, targets) => {
-  const diffs = await Promise.all(targets.map(t => difference(blocks, base, t)))
-  const additions = new Map()
-  const removals = new Map()
-  /** @type {API.BlockFetcher} */
-  const fetcher = { get: cid => additions.get(cid.toString()) ?? blocks.get(cid) }
-
-  let root = base
-  for (const { keys } of diffs) {
-    for (const [k, v] of keys) {
-      let res
-      if (v[1] == null) {
-        res = await del(fetcher, root, k)
-      } else {
-        res = await put(fetcher, root, k, v[1])
-      }
-      for (const blk of res.removals) {
-        if (additions.has(blk.cid.toString())) {
-          additions.delete(blk.cid.toString())
-        } else {
-          removals.set(blk.cid.toString(), blk)
-        }
-      }
-      for (const blk of res.additions) {
-        additions.set(blk.cid.toString(), blk)
-      }
-      root = res.root
-    }
-  }
-
-  return { root, additions: [...additions.values()], removals: [...removals.values()] }
-}