@fireproof/vendor 2.0.0 → 2.0.2

package/src/@web3-storage/pail/dist/src/index.js

@@ -0,0 +1,356 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js';
+import { ShardFetcher, isPrintableASCII } from './shard.js';
+import * as Shard from './shard.js';
+/**
+ * Put a value (a CID) for the given key. If the key exists it's value is
+ * overwritten.
+ *
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.ShardLink} root CID of the root node of the bucket.
+ * @param {string} key The key of the value to put.
+ * @param {API.UnknownLink} value The value to put.
+ * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
+ */
+export const put = async (blocks, root, key, value) => {
+    const shards = new ShardFetcher(blocks);
+    const rshard = await shards.get(root);
+    if (rshard.value.keyChars !== Shard.KeyCharsASCII) {
+        throw new Error(`unsupported key character set: ${rshard.value.keyChars}`);
+    }
+    if (!isPrintableASCII(key)) {
+        throw new Error('key contains non-ASCII characters');
+    }
+    // ensure utf8 encoded key is smaller than max
+    if (new TextEncoder().encode(key).length > rshard.value.maxKeySize) {
+        throw new Error(`UTF-8 encoded key exceeds max size of ${rshard.value.maxKeySize} bytes`);
+    }
+    const path = await traverse(shards, rshard, key);
+    const target = path[path.length - 1];
+    const skey = key.slice(target.value.prefix.length); // key within the shard
+    /** @type {API.ShardEntry} */
+    let entry = [skey, value];
+    let targetEntries = [...target.value.entries];
+    /** @type {API.ShardBlockView[]} */
+    const additions = [];
+    for (const [i, e] of targetEntries.entries()) {
+        const [k, v] = e;
+        // is this just a replace?
+        if (k === skey)
+            break;
+        // do we need to shard this entry?
+        const shortest = k.length < skey.length ? k : skey;
+        const other = shortest === k ? skey : k;
+        let common = '';
+        for (const char of shortest) {
+            const next = common + char;
+            if (!other.startsWith(next))
+                break;
+            common = next;
+        }
+        if (common.length) {
+            /** @type {API.ShardEntry[]} */
+            let entries = [];
+            // if the existing entry key or new key is equal to the common prefix,
+            // then the existing value / new value needs to persist in the parent
+            // shard. Otherwise they persist in this new shard.
+            if (common !== skey) {
+                entries = Shard.putEntry(entries, [skey.slice(common.length), value]);
+            }
+            if (common !== k) {
+                entries = Shard.putEntry(entries, [k.slice(common.length), v]);
+            }
+            let child = await Shard.encodeBlock(Shard.withEntries(entries, { ...target.value, prefix: target.value.prefix + common }));
+            additions.push(child);
+            // need to spread as access by index does not consider utf-16 surrogates
+            const commonChars = [...common];
+            // create parent shards for each character of the common prefix
+            for (let i = commonChars.length - 1; i > 0; i--) {
+                const parentConfig = { ...target.value, prefix: target.value.prefix + commonChars.slice(0, i).join('') };
+                /** @type {API.ShardEntryLinkValue | API.ShardEntryValueValue | API.ShardEntryLinkAndValueValue} */
+                let parentValue;
+                // if the first iteration and the existing entry key is equal to the
+                // common prefix, then existing value needs to persist in this parent
+                if (i === commonChars.length - 1 && common === k) {
+                    if (Array.isArray(v))
+                        throw new Error('found a shard link when expecting a value');
+                    parentValue = [child.cid, v];
+                }
+                else if (i === commonChars.length - 1 && common === skey) {
+                    parentValue = [child.cid, value];
+                }
+                else {
+                    parentValue = [child.cid];
+                }
+                const parent = await Shard.encodeBlock(Shard.withEntries([[commonChars[i], parentValue]], parentConfig));
+                additions.push(parent);
+                child = parent;
+            }
+            // remove the sharded entry
+            targetEntries.splice(i, 1);
+            // create the entry that will be added to target
+            if (commonChars.length === 1 && common === k) {
+                if (Array.isArray(v))
+                    throw new Error('found a shard link when expecting a value');
+                entry = [commonChars[0], [child.cid, v]];
+            }
+            else if (commonChars.length === 1 && common === skey) {
+                entry = [commonChars[0], [child.cid, value]];
+            }
+            else {
+                entry = [commonChars[0], [child.cid]];
+            }
+            break;
+        }
+    }
+    const shard = Shard.withEntries(Shard.putEntry(targetEntries, entry), target.value);
+    let child = await Shard.encodeBlock(shard);
+    // if no change in the target then we're done
+    if (child.cid.toString() === target.cid.toString()) {
+        return { root, additions: [], removals: [] };
+    }
+    additions.push(child);
+    // path is root -> target, so work backwards, propagating the new shard CID
+    for (let i = path.length - 2; i >= 0; i--) {
+        const parent = path[i];
+        const key = child.value.prefix.slice(parent.value.prefix.length);
+        const value = Shard.withEntries(parent.value.entries.map((entry) => {
+            const [k, v] = entry;
+            if (k !== key)
+                return entry;
+            if (!Array.isArray(v))
+                throw new Error(`"${key}" is not a shard link in: ${parent.cid}`);
+            return /** @type {API.ShardEntry} */ (v[1] == null ? [k, [child.cid]] : [k, [child.cid, v[1]]]);
+        }), parent.value);
+        child = await Shard.encodeBlock(value);
+        additions.push(child);
+    }
+    return { root: additions[additions.length - 1].cid, additions, removals: path };
+};
+/**
+ * Get the stored value for the given key from the bucket. If the key is not
+ * found, `undefined` is returned.
+ *
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.ShardLink} root CID of the root node of the bucket.
+ * @param {string} key The key of the value to get.
+ * @returns {Promise<API.UnknownLink | undefined>}
+ */
+export const get = async (blocks, root, key) => {
+    const shards = new ShardFetcher(blocks);
+    const rshard = await shards.get(root);
+    const path = await traverse(shards, rshard, key);
+    const target = path[path.length - 1];
+    const skey = key.slice(target.value.prefix.length); // key within the shard
+    const entry = target.value.entries.find(([k]) => k === skey);
+    if (!entry)
+        return;
+    return Array.isArray(entry[1]) ? entry[1][1] : entry[1];
+};
+/**
+ * Delete the value for the given key from the bucket. If the key is not found
+ * no operation occurs.
+ *
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.ShardLink} root CID of the root node of the bucket.
+ * @param {string} key The key of the value to delete.
+ * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
+ */
+export const del = async (blocks, root, key) => {
+    const shards = new ShardFetcher(blocks);
+    const rshard = await shards.get(root);
+    const path = await traverse(shards, rshard, key);
+    const target = path[path.length - 1];
+    const skey = key.slice(target.value.prefix.length); // key within the shard
+    const entryidx = target.value.entries.findIndex(([k]) => k === skey);
+    if (entryidx === -1)
+        return { root, additions: [], removals: [] };
+    const entry = target.value.entries[entryidx];
+    // cannot delete a shard (without data)
+    if (Array.isArray(entry[1]) && entry[1][1] == null) {
+        return { root, additions: [], removals: [] };
+    }
+    /** @type {API.ShardBlockView[]} */
+    const additions = [];
+    /** @type {API.ShardBlockView[]} */
+    const removals = [...path];
+    let shard = Shard.withEntries([...target.value.entries], target.value);
+    if (Array.isArray(entry[1])) {
+        // remove the value from this link+value
+        shard.entries[entryidx] = [entry[0], [entry[1][0]]];
+    }
+    else {
+        shard.entries.splice(entryidx, 1);
+        // if now empty, remove from parent
+        while (!shard.entries.length) {
+            const child = path[path.length - 1];
+            const parent = path[path.length - 2];
+            if (!parent)
+                break;
+            path.pop();
+            shard = Shard.withEntries(parent.value.entries.filter(e => {
+                if (!Array.isArray(e[1]))
+                    return true;
+                return e[1][0].toString() !== child.cid.toString();
+            }), parent.value);
+        }
+    }
+    let child = await Shard.encodeBlock(shard);
+    additions.push(child);
+    // path is root -> shard, so work backwards, propagating the new shard CID
+    for (let i = path.length - 2; i >= 0; i--) {
+        const parent = path[i];
+        const key = child.value.prefix.slice(parent.value.prefix.length);
+        const value = Shard.withEntries(parent.value.entries.map((entry) => {
+            const [k, v] = entry;
+            if (k !== key)
+                return entry;
+            if (!Array.isArray(v))
+                throw new Error(`"${key}" is not a shard link in: ${parent.cid}`);
+            return /** @type {API.ShardEntry} */ (v[1] == null ? [k, [child.cid]] : [k, [child.cid, v[1]]]);
+        }), parent.value);
+        child = await Shard.encodeBlock(value);
+        additions.push(child);
+    }
+    return { root: additions[additions.length - 1].cid, additions, removals };
+};
+/**
+ * @param {API.EntriesOptions} [options]
+ * @returns {options is API.KeyPrefixOption}
+ */
+const isKeyPrefixOption = options => {
+    const opts = options ?? {};
+    return 'prefix' in opts && Boolean(opts.prefix);
+};
+/**
+ * @param {API.EntriesOptions} [options]
+ * @returns {options is API.KeyRangeOption}
+ */
+const isKeyRangeOption = options => {
+    const opts = options ?? {};
+    return ('gt' in opts && Boolean(opts.gt)) || ('gte' in opts && Boolean(opts.gte)) || ('lt' in opts && Boolean(opts.lt)) || ('lte' in opts && Boolean(opts.lte));
+};
+/**
+ * @param {API.KeyRangeOption} options
+ * @returns {options is API.KeyLowerBoundRangeOption}
+ */
+const isKeyLowerBoundRangeOption = options => ('gt' in options && Boolean(options.gt)) || ('gte' in options && Boolean(options.gte));
+/**
+ * @param {API.KeyLowerBoundRangeOption} options
+ * @returns {options is API.KeyLowerBoundRangeInclusiveOption}
+ */
+const isKeyLowerBoundRangeInclusiveOption = options => 'gte' in options && Boolean(options.gte);
+/**
+ * @param {API.KeyLowerBoundRangeOption} options
+ * @returns {options is API.KeyLowerBoundRangeExclusiveOption}
+ */
+const isKeyLowerBoundRangeExclusiveOption = options => 'gt' in options && Boolean(options.gt);
+/**
+ * @param {API.KeyRangeOption} options
+ * @returns {options is API.KeyUpperBoundRangeOption}
+ */
+const isKeyUpperBoundRangeOption = options => ('lt' in options && Boolean(options.lt)) || ('lte' in options && Boolean(options.lte));
+/**
+ * @param {API.KeyUpperBoundRangeOption} options
+ * @returns {options is API.KeyUpperBoundRangeInclusiveOption}
+ */
+const isKeyUpperBoundRangeInclusiveOption = options => 'lte' in options && Boolean(options.lte);
+/**
+ * @param {API.KeyUpperBoundRangeOption} options
+ * @returns {options is API.KeyUpperBoundRangeExclusiveOption}
+ */
+const isKeyUpperBoundRangeExclusiveOption = options => 'lt' in options && Boolean(options.lt);
+/**
+ * List entries in the bucket.
+ *
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.ShardLink} root CID of the root node of the bucket.
+ * @param {API.EntriesOptions} [options]
+ * @returns {AsyncIterableIterator<API.ShardValueEntry>}
+ */
+export const entries = async function* (blocks, root, options) {
+    const hasKeyPrefix = isKeyPrefixOption(options);
+    const hasKeyRange = isKeyRangeOption(options);
+    const hasKeyLowerBoundRange = hasKeyRange && isKeyLowerBoundRangeOption(options);
+    const hasKeyLowerBoundRangeInclusive = hasKeyLowerBoundRange && isKeyLowerBoundRangeInclusiveOption(options);
+    const hasKeyLowerBoundRangeExclusive = hasKeyLowerBoundRange && isKeyLowerBoundRangeExclusiveOption(options);
+    const hasKeyUpperBoundRange = hasKeyRange && isKeyUpperBoundRangeOption(options);
+    const hasKeyUpperBoundRangeInclusive = hasKeyUpperBoundRange && isKeyUpperBoundRangeInclusiveOption(options);
+    const hasKeyUpperBoundRangeExclusive = hasKeyUpperBoundRange && isKeyUpperBoundRangeExclusiveOption(options);
+    const hasKeyUpperAndLowerBoundRange = hasKeyLowerBoundRange && hasKeyUpperBoundRange;
+    const shards = new ShardFetcher(blocks);
+    const rshard = await shards.get(root);
+    yield* (
+    /** @returns {AsyncIterableIterator<API.ShardValueEntry>} */
+    (async function* ents(shard) {
+        for (const entry of shard.value.entries) {
+            const key = shard.value.prefix + entry[0];
+            // if array, this is a link to a shard
+            if (Array.isArray(entry[1])) {
+                if (entry[1][1]) {
+                    if ((hasKeyPrefix && key.startsWith(options.prefix)) ||
+                        (hasKeyUpperAndLowerBoundRange && (((hasKeyLowerBoundRangeExclusive && key > options.gt) || (hasKeyLowerBoundRangeInclusive && key >= options.gte)) &&
+                            ((hasKeyUpperBoundRangeExclusive && key < options.lt) || (hasKeyUpperBoundRangeInclusive && key <= options.lte)))) ||
+                        (hasKeyLowerBoundRangeExclusive && key > options.gt) ||
+                        (hasKeyLowerBoundRangeInclusive && key >= options.gte) ||
+                        (hasKeyUpperBoundRangeExclusive && key < options.lt) ||
+                        (hasKeyUpperBoundRangeInclusive && key <= options.lte) ||
+                        (!hasKeyPrefix && !hasKeyRange)) {
+                        yield [key, entry[1][1]];
+                    }
+                }
+                if (hasKeyPrefix) {
+                    if (options.prefix.length <= key.length && !key.startsWith(options.prefix)) {
+                        continue;
+                    }
+                    if (options.prefix.length > key.length && !options.prefix.startsWith(key)) {
+                        continue;
+                    }
+                }
+                else if ((hasKeyLowerBoundRangeExclusive && (trunc(key, Math.min(key.length, options.gt.length)) < trunc(options.gt, Math.min(key.length, options.gt.length)))) ||
+                    (hasKeyLowerBoundRangeInclusive && (trunc(key, Math.min(key.length, options.gte.length)) < trunc(options.gte, Math.min(key.length, options.gte.length)))) ||
+                    (hasKeyUpperBoundRangeExclusive && (trunc(key, Math.min(key.length, options.lt.length)) > trunc(options.lt, Math.min(key.length, options.lt.length)))) ||
+                    (hasKeyUpperBoundRangeInclusive && (trunc(key, Math.min(key.length, options.lte.length)) > trunc(options.lte, Math.min(key.length, options.lte.length))))) {
+                    continue;
+                }
+                yield* ents(await shards.get(entry[1][0]));
+            }
+            else {
+                if ((hasKeyPrefix && key.startsWith(options.prefix)) ||
+                    (hasKeyRange && hasKeyUpperAndLowerBoundRange && (((hasKeyLowerBoundRangeExclusive && key > options.gt) || (hasKeyLowerBoundRangeInclusive && key >= options.gte)) &&
+                        ((hasKeyUpperBoundRangeExclusive && key < options.lt) || (hasKeyUpperBoundRangeInclusive && key <= options.lte)))) ||
+                    (hasKeyRange && !hasKeyUpperAndLowerBoundRange && ((hasKeyLowerBoundRangeExclusive && key > options.gt) || (hasKeyLowerBoundRangeInclusive && key >= options.gte) ||
+                        (hasKeyUpperBoundRangeExclusive && key < options.lt) || (hasKeyUpperBoundRangeInclusive && key <= options.lte))) ||
+                    (!hasKeyPrefix && !hasKeyRange)) {
+                    yield [key, entry[1]];
+                }
+            }
+        }
+    }))(rshard);
+};
+/**
+ * @param {string} str
+ * @param {number} len
+ */
+const trunc = (str, len) => str.length <= len ? str : str.slice(0, len);
+/**
+ * Traverse from the passed shard block to the target shard block using the
+ * passed key. All traversed shards are returned, starting with the passed
+ * shard and ending with the target.
+ *
+ * @param {ShardFetcher} shards
+ * @param {API.ShardBlockView} shard
+ * @param {string} key
+ * @returns {Promise<[API.ShardBlockView, ...Array<API.ShardBlockView>]>}
+ */
+const traverse = async (shards, shard, key) => {
+    for (const [k, v] of shard.value.entries) {
+        if (key === k)
+            return [shard];
+        if (key.startsWith(k) && Array.isArray(v)) {
+            const path = await traverse(shards, await shards.get(v[0]), key.slice(k.length));
+            return [shard, ...path];
+        }
+    }
+    return [shard];
+};

package/src/@web3-storage/pail/dist/src/merge.js

@@ -0,0 +1,42 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js';
+import { difference } from './diff.js';
+import { put, del } from './index.js';
+/**
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.ShardLink} base Merge base. Common parent of target DAGs.
+ * @param {API.ShardLink[]} targets Target DAGs to merge.
+ * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
+ */
+export const merge = async (blocks, base, targets) => {
+    const diffs = await Promise.all(targets.map(t => difference(blocks, base, t)));
+    const additions = new Map();
+    const removals = new Map();
+    /** @type {API.BlockFetcher} */
+    const fetcher = { get: cid => additions.get(cid.toString()) ?? blocks.get(cid) };
+    let root = base;
+    for (const { keys } of diffs) {
+        for (const [k, v] of keys) {
+            let res;
+            if (v[1] == null) {
+                res = await del(fetcher, root, k);
+            }
+            else {
+                res = await put(fetcher, root, k, v[1]);
+            }
+            for (const blk of res.removals) {
+                if (additions.has(blk.cid.toString())) {
+                    additions.delete(blk.cid.toString());
+                }
+                else {
+                    removals.set(blk.cid.toString(), blk);
+                }
+            }
+            for (const blk of res.additions) {
+                additions.set(blk.cid.toString(), blk);
+            }
+            root = res.root;
+        }
+    }
+    return { root, additions: [...additions.values()], removals: [...removals.values()] };
+};

package/src/@web3-storage/pail/dist/src/shard.js

@@ -0,0 +1,166 @@
+import * as Link from 'multiformats/link';
+import { Block, encode, decode } from 'multiformats/block';
+import { sha256 } from 'multiformats/hashes/sha2';
+import * as dagCBOR from '@ipld/dag-cbor';
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js';
+export const KeyCharsASCII = 'ascii';
+export const MaxKeySize = 4096;
+/**
+ * @extends {Block<API.Shard, typeof dagCBOR.code, typeof sha256.code, 1>}
+ * @implements {API.ShardBlockView}
+ */
+export class ShardBlock extends Block {
+    /**
+     * @param {object} config
+     * @param {API.ShardLink} config.cid
+     * @param {API.Shard} config.value
+     * @param {Uint8Array} config.bytes
+     */
+    constructor({ cid, value, bytes }) {
+        // @ts-expect-error
+        super({ cid, value, bytes });
+    }
+    /** @param {API.ShardOptions} [options] */
+    static create(options) {
+        return encodeBlock(create(options));
+    }
+}
+/**
+ * @param {API.ShardOptions} [options]
+ * @returns {API.Shard}
+ */
+export const create = (options) => ({ entries: [], ...configure(options) });
+/**
+ * @param {API.ShardOptions} [options]
+ * @returns {API.ShardConfig}
+ */
+export const configure = (options) => ({
+    version: 1,
+    keyChars: options?.keyChars ?? KeyCharsASCII,
+    maxKeySize: options?.maxKeySize ?? MaxKeySize,
+    prefix: options?.prefix ?? ''
+});
+/**
+ * @param {API.ShardEntry[]} entries
+ * @param {API.ShardOptions} [options]
+ * @returns {API.Shard}
+ */
+export const withEntries = (entries, options) => ({ ...create(options), entries });
+/** @type {WeakMap<Uint8Array, API.ShardBlockView>} */
+const decodeCache = new WeakMap();
+/**
+ * @param {API.Shard} value
+ * @returns {Promise<API.ShardBlockView>}
+ */
+export const encodeBlock = async (value) => {
+    const { cid, bytes } = await encode({ value, codec: dagCBOR, hasher: sha256 });
+    const block = new ShardBlock({ cid, value, bytes });
+    decodeCache.set(block.bytes, block);
+    return block;
+};
+/**
+ * @param {Uint8Array} bytes
+ * @returns {Promise<API.ShardBlockView>}
+ */
+export const decodeBlock = async (bytes) => {
+    const block = decodeCache.get(bytes);
+    if (block)
+        return block;
+    const { cid, value } = await decode({ bytes, codec: dagCBOR, hasher: sha256 });
+    if (!isShard(value))
+        throw new Error(`invalid shard: ${cid}`);
+    return new ShardBlock({ cid, value, bytes });
+};
+/**
+ * @param {any} value
+ * @returns {value is API.Shard}
+ */
+export const isShard = value => value != null &&
+    typeof value === 'object' &&
+    Array.isArray(value.entries) &&
+    value.version === 1 &&
+    typeof value.maxKeySize === 'number' &&
+    typeof value.keyChars === 'string' &&
+    typeof value.prefix === 'string';
+/**
+ * @param {any} value
+ * @returns {value is API.ShardLink}
+ */
+export const isShardLink = (value) => Link.isLink(value) &&
+    value.code === dagCBOR.code;
+export class ShardFetcher {
+    /** @param {API.BlockFetcher} blocks */
+    constructor(blocks) {
+        this._blocks = blocks;
+    }
+    /**
+     * @param {API.ShardLink} link
+     * @returns {Promise<API.ShardBlockView>}
+     */
+    async get(link) {
+        const block = await this._blocks.get(link);
+        if (!block)
+            throw new Error(`missing block: ${link}`);
+        return decodeBlock(block.bytes);
+    }
+}
+/**
+ * @param {API.ShardEntry[]} target Entries to insert into.
+ * @param {API.ShardEntry} newEntry
+ * @returns {API.ShardEntry[]}
+ */
+export const putEntry = (target, newEntry) => {
+    /** @type {API.ShardEntry[]} */
+    const entries = [];
+    for (const [i, entry] of target.entries()) {
+        const [k, v] = entry;
+        if (newEntry[0] === k) {
+            // if new value is link to shard...
+            if (Array.isArray(newEntry[1])) {
+                // and old value is link to shard
+                // and old value is _also_ link to data
+                // and new value does not have link to data
+                // then preserve old data
+                if (Array.isArray(v) && v[1] != null && newEntry[1][1] == null) {
+                    entries.push([k, [newEntry[1][0], v[1]]]);
+                }
+                else {
+                    entries.push(newEntry);
+                }
+            }
+            else {
+                // shard as well as value?
+                if (Array.isArray(v)) {
+                    entries.push([k, [v[0], newEntry[1]]]);
+                }
+                else {
+                    entries.push(newEntry);
+                }
+            }
+            for (let j = i + 1; j < target.length; j++) {
+                entries.push(target[j]);
+            }
+            return entries;
+        }
+        if (i === 0 && newEntry[0] < k) {
+            entries.push(newEntry);
+            for (let j = i; j < target.length; j++) {
+                entries.push(target[j]);
+            }
+            return entries;
+        }
+        if (i > 0 && newEntry[0] > target[i - 1][0] && newEntry[0] < k) {
+            entries.push(newEntry);
+            for (let j = i; j < target.length; j++) {
+                entries.push(target[j]);
+            }
+            return entries;
+        }
+        entries.push(entry);
+    }
+    entries.push(newEntry);
+    return entries;
+};
+/** @param {string} s */
+export const isPrintableASCII = s => /^[\x20-\x7E]*$/.test(s);