@fireproof/vendor 2.0.1 → 2.0.2

package/src/@web3-storage/pail/dist/src/crdt/batch/index.js

@@ -0,0 +1,140 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js';
+import * as Shard from '../../shard.js';
+import { ShardFetcher, ShardBlock } from '../../shard.js';
+import * as Batch from '../../batch/index.js';
+import { BatchCommittedError } from '../../batch/index.js';
+import * as CRDT from '../index.js';
+import * as Clock from '../../clock/index.js';
+import { EventBlock } from '../../clock/index.js';
+import { MemoryBlockstore, MultiBlockFetcher } from '../../block.js';
+export { BatchCommittedError };
+/** @implements {API.CRDTBatcher} */
+class Batcher {
+    #committed = false;
+    /**
+     * @param {object} init
+     * @param {API.BlockFetcher} init.blocks Block storage.
+     * @param {API.EventLink<API.Operation>[]} init.head Merkle clock head.
+     * @param {API.BatcherShardEntry[]} init.entries The entries in this shard.
+     * @param {string} init.prefix Key prefix.
+     * @param {number} init.version Shard compatibility version.
+     * @param {string} init.keyChars Characters allowed in keys, referring to a known character set.
+     * @param {number} init.maxKeySize Max key size in bytes.
+     * @param {API.ShardBlockView} init.base Original shard this batcher is based on.
+     * @param {API.ShardBlockView[]} init.additions Additions to include in the committed batch.
+     * @param {API.ShardBlockView[]} init.removals Removals to include in the committed batch.
+     */
+    constructor({ blocks, head, entries, prefix, version, keyChars, maxKeySize, base, additions, removals }) {
+        this.blocks = blocks;
+        this.head = head;
+        this.prefix = prefix;
+        this.entries = [...entries];
+        this.base = base;
+        this.version = version;
+        this.keyChars = keyChars;
+        this.maxKeySize = maxKeySize;
+        this.additions = additions;
+        this.removals = removals;
+        /** @type {API.BatchOperation['ops']} */
+        this.ops = [];
+    }
+    /**
+     * @param {string} key The key of the value to put.
+     * @param {API.UnknownLink} value The value to put.
+     * @returns {Promise<void>}
+     */
+    async put(key, value) {
+        if (this.#committed)
+            throw new BatchCommittedError();
+        await Batch.put(this.blocks, this, key, value);
+        this.ops.push({ type: 'put', key, value });
+    }
+    async commit() {
+        if (this.#committed)
+            throw new BatchCommittedError();
+        this.#committed = true;
+        const res = await Batch.commit(this);
+        /** @type {API.Operation} */
+        const data = { type: 'batch', ops: this.ops, root: res.root };
+        const event = await EventBlock.create(data, this.head);
+        const mblocks = new MemoryBlockstore();
+        const blocks = new MultiBlockFetcher(mblocks, this.blocks);
+        mblocks.putSync(event.cid, event.bytes);
+        const head = await Clock.advance(blocks, this.head, event.cid);
+        /** @type {Map<string, API.ShardBlockView>} */
+        const additions = new Map();
+        /** @type {Map<string, API.ShardBlockView>} */
+        const removals = new Map();
+        for (const a of this.additions) {
+            additions.set(a.cid.toString(), a);
+        }
+        for (const r of this.removals) {
+            removals.set(r.cid.toString(), r);
+        }
+        for (const a of res.additions) {
+            if (removals.has(a.cid.toString())) {
+                removals.delete(a.cid.toString());
+            }
+            additions.set(a.cid.toString(), a);
+        }
+        for (const r of res.removals) {
+            if (additions.has(r.cid.toString())) {
+                additions.delete(r.cid.toString());
+            }
+            else {
+                removals.set(r.cid.toString(), r);
+            }
+        }
+        return {
+            head,
+            event,
+            root: res.root,
+            additions: [...additions.values()],
+            removals: [...removals.values()]
+        };
+    }
+    /**
+     * @param {object} init
+     * @param {API.BlockFetcher} init.blocks Block storage.
+     * @param {API.EventLink<API.Operation>[]} init.head Merkle clock head.
+     */
+    static async create({ blocks, head }) {
+        const mblocks = new MemoryBlockstore();
+        blocks = new MultiBlockFetcher(mblocks, blocks);
+        if (!head.length) {
+            const base = await ShardBlock.create();
+            mblocks.putSync(base.cid, base.bytes);
+            return new Batcher({
+                blocks,
+                head,
+                entries: [],
+                base,
+                additions: [base],
+                removals: [],
+                ...Shard.configure(base.value)
+            });
+        }
+        const { root, additions, removals } = await CRDT.root(blocks, head);
+        for (const a of additions) {
+            mblocks.putSync(a.cid, a.bytes);
+        }
+        const shards = new ShardFetcher(blocks);
+        const base = await shards.get(root);
+        return new Batcher({
+            blocks,
+            head,
+            entries: base.value.entries,
+            base,
+            additions,
+            removals,
+            ...Shard.configure(base.value)
+        });
+    }
+}
+/**
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.EventLink<API.Operation>[]} head Merkle clock head.
+ * @returns {Promise<API.CRDTBatcher>}
+ */
+export const create = (blocks, head) => Batcher.create({ blocks, head });

package/src/@web3-storage/pail/dist/src/crdt/index.js

@@ -0,0 +1,344 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js';
+import * as Clock from '../clock/index.js';
+import { EventFetcher, EventBlock } from '../clock/index.js';
+import * as Pail from '../index.js';
+import { ShardBlock } from '../shard.js';
+import { MemoryBlockstore, MultiBlockFetcher } from '../block.js';
+import * as Batch from '../batch/index.js';
+/**
+ * Put a value (a CID) for the given key. If the key exists it's value is
+ * overwritten.
+ *
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.EventLink<API.Operation>[]} head Merkle clock head.
+ * @param {string} key The key of the value to put.
+ * @param {API.UnknownLink} value The value to put.
+ * @returns {Promise<API.Result>}
+ */
+export const put = async (blocks, head, key, value) => {
+    const mblocks = new MemoryBlockstore();
+    blocks = new MultiBlockFetcher(mblocks, blocks);
+    if (!head.length) {
+        const shard = await ShardBlock.create();
+        mblocks.putSync(shard.cid, shard.bytes);
+        const result = await Pail.put(blocks, shard.cid, key, value);
+        /** @type {API.Operation} */
+        const data = { type: 'put', root: result.root, key, value };
+        const event = await EventBlock.create(data, head);
+        head = await Clock.advance(blocks, head, event.cid);
+        return {
+            root: result.root,
+            additions: [shard, ...result.additions],
+            removals: result.removals,
+            head,
+            event
+        };
+    }
+    /** @type {EventFetcher<API.Operation>} */
+    const events = new EventFetcher(blocks);
+    const ancestor = await findCommonAncestor(events, head);
+    if (!ancestor)
+        throw new Error('failed to find common ancestor event');
+    const aevent = await events.get(ancestor);
+    let { root } = aevent.value.data;
+    const sorted = await findSortedEvents(events, head, ancestor);
+    /** @type {Map<string, API.ShardBlockView>} */
+    const additions = new Map();
+    /** @type {Map<string, API.ShardBlockView>} */
+    const removals = new Map();
+    for (const { value: event } of sorted) {
+        let result;
+        if (event.data.type === 'put') {
+            result = await Pail.put(blocks, root, event.data.key, event.data.value);
+        }
+        else if (event.data.type === 'del') {
+            result = await Pail.del(blocks, root, event.data.key);
+        }
+        else if (event.data.type === 'batch') {
+            const batch = await Batch.create(blocks, root);
+            for (const op of event.data.ops) {
+                if (op.type !== 'put')
+                    throw new Error(`unsupported batch operation: ${op.type}`);
+                await batch.put(op.key, op.value);
+            }
+            result = await batch.commit();
+        }
+        else {
+            // @ts-expect-error type does not exist on never
+            throw new Error(`unknown operation: ${event.data.type}`);
+        }
+        root = result.root;
+        for (const a of result.additions) {
+            mblocks.putSync(a.cid, a.bytes);
+            additions.set(a.cid.toString(), a);
+        }
+        for (const r of result.removals) {
+            removals.set(r.cid.toString(), r);
+        }
+    }
+    const result = await Pail.put(blocks, root, key, value);
+    // if we didn't change the pail we're done
+    if (result.root.toString() === root.toString()) {
+        return { root, additions: [], removals: [], head };
+    }
+    for (const a of result.additions) {
+        mblocks.putSync(a.cid, a.bytes);
+        additions.set(a.cid.toString(), a);
+    }
+    for (const r of result.removals) {
+        removals.set(r.cid.toString(), r);
+    }
+    /** @type {API.Operation} */
+    const data = { type: 'put', root: result.root, key, value };
+    const event = await EventBlock.create(data, head);
+    mblocks.putSync(event.cid, event.bytes);
+    head = await Clock.advance(blocks, head, event.cid);
+    // filter blocks that were added _and_ removed
+    for (const k of removals.keys()) {
+        if (additions.has(k)) {
+            additions.delete(k);
+            removals.delete(k);
+        }
+    }
+    return {
+        root: result.root,
+        additions: [...additions.values()],
+        removals: [...removals.values()],
+        head,
+        event
+    };
+};
+/**
+ * Delete the value for the given key from the bucket. If the key is not found
+ * no operation occurs.
+ *
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.EventLink<API.Operation>[]} head Merkle clock head.
+ * @param {string} key The key of the value to delete.
+ * @param {object} [options]
+ * @returns {Promise<API.Result>}
+ */
+export const del = async (blocks, head, key, options) => {
+    throw new Error('not implemented');
+};
+/**
+ * Determine the effective pail root given the current merkle clock head.
+ *
+ * Clocks with multiple head events may return blocks that were added or
+ * removed while playing forward events from their common ancestor.
+ *
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.EventLink<API.Operation>[]} head Merkle clock head.
+ * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
+ */
+export const root = async (blocks, head) => {
+    if (!head.length)
+        throw new Error('cannot determine root of headless clock');
+    const mblocks = new MemoryBlockstore();
+    blocks = new MultiBlockFetcher(mblocks, blocks);
+    /** @type {EventFetcher<API.Operation>} */
+    const events = new EventFetcher(blocks);
+    if (head.length === 1) {
+        const event = await events.get(head[0]);
+        const { root } = event.value.data;
+        return { root, additions: [], removals: [] };
+    }
+    const ancestor = await findCommonAncestor(events, head);
+    if (!ancestor)
+        throw new Error('failed to find common ancestor event');
+    const aevent = await events.get(ancestor);
+    let { root } = aevent.value.data;
+    const sorted = await findSortedEvents(events, head, ancestor);
+    /** @type {Map<string, API.ShardBlockView>} */
+    const additions = new Map();
+    /** @type {Map<string, API.ShardBlockView>} */
+    const removals = new Map();
+    for (const { value: event } of sorted) {
+        let result;
+        if (event.data.type === 'put') {
+            result = await Pail.put(blocks, root, event.data.key, event.data.value);
+        }
+        else if (event.data.type === 'del') {
+            result = await Pail.del(blocks, root, event.data.key);
+        }
+        else if (event.data.type === 'batch') {
+            const batch = await Batch.create(blocks, root);
+            for (const op of event.data.ops) {
+                if (op.type !== 'put')
+                    throw new Error(`unsupported batch operation: ${op.type}`);
+                await batch.put(op.key, op.value);
+            }
+            result = await batch.commit();
+        }
+        else {
+            // @ts-expect-error type does not exist on never
+            throw new Error(`unknown operation: ${event.data.type}`);
+        }
+        root = result.root;
+        for (const a of result.additions) {
+            mblocks.putSync(a.cid, a.bytes);
+            additions.set(a.cid.toString(), a);
+        }
+        for (const r of result.removals) {
+            removals.set(r.cid.toString(), r);
+        }
+    }
+    // filter blocks that were added _and_ removed
+    for (const k of removals.keys()) {
+        if (additions.has(k)) {
+            additions.delete(k);
+            removals.delete(k);
+        }
+    }
+    return {
+        root,
+        additions: [...additions.values()],
+        removals: [...removals.values()]
+    };
+};
+/**
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.EventLink<API.Operation>[]} head Merkle clock head.
+ * @param {string} key The key of the value to retrieve.
+ */
+export const get = async (blocks, head, key) => {
+    if (!head.length)
+        return;
+    const result = await root(blocks, head);
+    if (result.additions.length) {
+        blocks = new MultiBlockFetcher(new MemoryBlockstore(result.additions), blocks);
+    }
+    return Pail.get(blocks, result.root, key);
+};
+/**
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.EventLink<API.Operation>[]} head Merkle clock head.
+ * @param {API.EntriesOptions} [options]
+ */
+export const entries = async function* (blocks, head, options) {
+    if (!head.length)
+        return;
+    const result = await root(blocks, head);
+    if (result.additions.length) {
+        blocks = new MultiBlockFetcher(new MemoryBlockstore(result.additions), blocks);
+    }
+    yield* Pail.entries(blocks, result.root, options);
+};
+/**
+ * Find the common ancestor event of the passed children. A common ancestor is
+ * the first single event in the DAG that _all_ paths from children lead to.
+ *
+ * @param {EventFetcher<API.Operation>} events
+ * @param  {API.EventLink<API.Operation>[]} children
+ */
+const findCommonAncestor = async (events, children) => {
+    if (!children.length)
+        return;
+    const candidates = children.map(c => [c]);
+    while (true) {
+        let changed = false;
+        for (const c of candidates) {
+            const candidate = await findAncestorCandidate(events, c[c.length - 1]);
+            if (!candidate)
+                continue;
+            changed = true;
+            c.push(candidate);
+            const ancestor = findCommonString(candidates);
+            if (ancestor)
+                return ancestor;
+        }
+        if (!changed)
+            return;
+    }
+};
+/**
+ * @param {EventFetcher<API.Operation>} events
+ * @param {API.EventLink<API.Operation>} root
+ */
+const findAncestorCandidate = async (events, root) => {
+    const { value: event } = await events.get(root);
+    if (!event.parents.length)
+        return root;
+    return event.parents.length === 1
+        ? event.parents[0]
+        : findCommonAncestor(events, event.parents);
+};
+/**
+ * @template {{ toString: () => string }} T
+ * @param  {Array<T[]>} arrays
+ */
+const findCommonString = (arrays) => {
+    arrays = arrays.map(a => [...a]);
+    for (const arr of arrays) {
+        for (const item of arr) {
+            let matched = true;
+            for (const other of arrays) {
+                if (arr === other)
+                    continue;
+                matched = other.some(i => String(i) === String(item));
+                if (!matched)
+                    break;
+            }
+            if (matched)
+                return item;
+        }
+    }
+};
+/**
+ * Find and sort events between the head(s) and the tail.
+ * @param {EventFetcher<API.Operation>} events
+ * @param {API.EventLink<API.Operation>[]} head
+ * @param {API.EventLink<API.Operation>} tail
+ */
+const findSortedEvents = async (events, head, tail) => {
+    if (head.length === 1 && head[0].toString() === tail.toString()) {
+        return [];
+    }
+    // get weighted events - heavier events happened first
+    /** @type {Map<string, { event: API.EventBlockView<API.Operation>, weight: number }>} */
+    const weights = new Map();
+    const all = await Promise.all(head.map(h => findEvents(events, h, tail)));
+    for (const arr of all) {
+        for (const { event, depth } of arr) {
+            const info = weights.get(event.cid.toString());
+            if (info) {
+                info.weight += depth;
+            }
+            else {
+                weights.set(event.cid.toString(), { event, weight: depth });
+            }
+        }
+    }
+    // group events into buckets by weight
+    /** @type {Map<number, API.EventBlockView<API.Operation>[]>} */
+    const buckets = new Map();
+    for (const { event, weight } of weights.values()) {
+        const bucket = buckets.get(weight);
+        if (bucket) {
+            bucket.push(event);
+        }
+        else {
+            buckets.set(weight, [event]);
+        }
+    }
+    // sort by weight, and by CID within weight
+    return Array.from(buckets)
+        .sort((a, b) => b[0] - a[0])
+        .flatMap(([, es]) => es.sort((a, b) => String(a.cid) < String(b.cid) ? -1 : 1));
+};
+/**
+ * @param {EventFetcher<API.Operation>} events
+ * @param {API.EventLink<API.Operation>} start
+ * @param {API.EventLink<API.Operation>} end
+ * @returns {Promise<Array<{ event: API.EventBlockView<API.Operation>, depth: number }>>}
+ */
+const findEvents = async (events, start, end, depth = 0) => {
+    const event = await events.get(start);
+    const acc = [{ event, depth }];
+    const { parents } = event.value;
+    if (parents.length === 1 && String(parents[0]) === String(end))
+        return acc;
+    const rest = await Promise.all(parents.map(p => findEvents(events, p, end, depth + 1)));
+    return acc.concat(...rest);
+};

package/src/@web3-storage/pail/dist/src/diff.js

@@ -0,0 +1,151 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js';
+import { ShardFetcher } from './shard.js';
+/**
+ * @typedef {string} K
+ * @typedef {[before: null, after: API.UnknownLink]} AddV
+ * @typedef {[before: API.UnknownLink, after: API.UnknownLink]} UpdateV
+ * @typedef {[before: API.UnknownLink, after: null]} DeleteV
+ * @typedef {[key: K, value: AddV|UpdateV|DeleteV]} KV
+ * @typedef {KV[]} KeysDiff
+ * @typedef {{ keys: KeysDiff, shards: API.ShardDiff }} CombinedDiff
+ */
+/**
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.ShardLink} a Base DAG.
+ * @param {API.ShardLink} b Comparison DAG.
+ * @returns {Promise<CombinedDiff>}
+ */
+export const difference = async (blocks, a, b) => {
+    if (isEqual(a, b))
+        return { keys: [], shards: { additions: [], removals: [] } };
+    const shards = new ShardFetcher(blocks);
+    const [ashard, bshard] = await Promise.all([shards.get(a), shards.get(b)]);
+    const aents = new Map(ashard.value.entries);
+    const bents = new Map(bshard.value.entries);
+    const keys = /** @type {Map<K, AddV|UpdateV|DeleteV>} */ (new Map());
+    const additions = new Map([[bshard.cid.toString(), bshard]]);
+    const removals = new Map([[ashard.cid.toString(), ashard]]);
+    // find shards removed in B
+    for (const [akey, aval] of ashard.value.entries) {
+        const bval = bents.get(akey);
+        if (bval)
+            continue;
+        if (!Array.isArray(aval)) {
+            keys.set(`${ashard.value.prefix}${akey}`, [aval, null]);
+            continue;
+        }
+        // if shard link _with_ value
+        if (aval[1] != null) {
+            keys.set(`${ashard.value.prefix}${akey}`, [aval[1], null]);
+        }
+        for await (const s of collect(shards, aval[0])) {
+            for (const [k, v] of s.value.entries) {
+                if (!Array.isArray(v)) {
+                    keys.set(`${s.value.prefix}${k}`, [v, null]);
+                }
+                else if (v[1] != null) {
+                    keys.set(`${s.value.prefix}${k}`, [v[1], null]);
+                }
+            }
+            removals.set(s.cid.toString(), s);
+        }
+    }
+    // find shards added or updated in B
+    for (const [bkey, bval] of bshard.value.entries) {
+        const aval = aents.get(bkey);
+        if (!Array.isArray(bval)) {
+            if (!aval) {
+                keys.set(`${bshard.value.prefix}${bkey}`, [null, bval]);
+            }
+            else if (Array.isArray(aval)) {
+                keys.set(`${bshard.value.prefix}${bkey}`, [aval[1] ?? null, bval]);
+            }
+            else if (!isEqual(aval, bval)) {
+                keys.set(`${bshard.value.prefix}${bkey}`, [aval, bval]);
+            }
+            continue;
+        }
+        if (aval && Array.isArray(aval)) { // updated in B
+            if (isEqual(aval[0], bval[0])) {
+                if (bval[1] != null && (aval[1] == null || !isEqual(aval[1], bval[1]))) {
+                    keys.set(`${bshard.value.prefix}${bkey}`, [aval[1] ?? null, bval[1]]);
+                }
+                continue; // updated value?
+            }
+            const res = await difference(blocks, aval[0], bval[0]);
+            for (const shard of res.shards.additions) {
+                additions.set(shard.cid.toString(), shard);
+            }
+            for (const shard of res.shards.removals) {
+                removals.set(shard.cid.toString(), shard);
+            }
+            for (const [k, v] of res.keys) {
+                keys.set(k, v);
+            }
+        }
+        else if (aval) { // updated in B value => link+value
+            if (bval[1] == null) {
+                keys.set(`${bshard.value.prefix}${bkey}`, [aval, null]);
+            }
+            else if (!isEqual(aval, bval[1])) {
+                keys.set(`${bshard.value.prefix}${bkey}`, [aval, bval[1]]);
+            }
+            for await (const s of collect(shards, bval[0])) {
+                for (const [k, v] of s.value.entries) {
+                    if (!Array.isArray(v)) {
+                        keys.set(`${s.value.prefix}${k}`, [null, v]);
+                    }
+                    else if (v[1] != null) {
+                        keys.set(`${s.value.prefix}${k}`, [null, v[1]]);
+                    }
+                }
+                additions.set(s.cid.toString(), s);
+            }
+        }
+        else { // added in B
+            keys.set(`${bshard.value.prefix}${bkey}`, [null, bval[0]]);
+            for await (const s of collect(shards, bval[0])) {
+                for (const [k, v] of s.value.entries) {
+                    if (!Array.isArray(v)) {
+                        keys.set(`${s.value.prefix}${k}`, [null, v]);
+                    }
+                    else if (v[1] != null) {
+                        keys.set(`${s.value.prefix}${k}`, [null, v[1]]);
+                    }
+                }
+                additions.set(s.cid.toString(), s);
+            }
+        }
+    }
+    // filter blocks that were added _and_ removed from B
+    for (const k of removals.keys()) {
+        if (additions.has(k)) {
+            additions.delete(k);
+            removals.delete(k);
+        }
+    }
+    return {
+        keys: [...keys.entries()].sort((a, b) => a[0] < b[0] ? -1 : 1),
+        shards: { additions: [...additions.values()], removals: [...removals.values()] }
+    };
+};
+/**
+ * @param {API.UnknownLink} a
+ * @param {API.UnknownLink} b
+ */
+const isEqual = (a, b) => a.toString() === b.toString();
+/**
+ * @param {import('./shard.js').ShardFetcher} shards
+ * @param {API.ShardLink} root
+ * @returns {AsyncIterableIterator<API.ShardBlockView>}
+ */
+async function* collect(shards, root) {
+    const shard = await shards.get(root);
+    yield shard;
+    for (const [, v] of shard.value.entries) {
+        if (!Array.isArray(v))
+            continue;
+        yield* collect(shards, v[0]);
+    }
+}