@fireproof/core 0.3.21 → 0.4.0

package/dist/blockstore.js

@@ -0,0 +1,242 @@
+import { parse } from 'multiformats/link';
+import { CID } from 'multiformats';
+import { Valet } from './valet.js';
+// const sleep = ms => new Promise(r => setTimeout(r, ms))
+const husherMap = new Map();
+const husher = (id, workFn) => {
+    if (!husherMap.has(id)) {
+        husherMap.set(id, workFn().finally(() => setTimeout(() => husherMap.delete(id), 100)));
+    }
+    return husherMap.get(id);
+};
+/**
+ * @typedef {{ get: (link: import('../src/link').AnyLink) => Promise<AnyBlock | undefined> }} BlockFetcher
+ */
+/**
+ * @typedef {Object} AnyBlock
+ * @property {import('./link').AnyLink} cid - The CID of the block
+ * @property {Uint8Array} bytes - The block's data
+ *
+ * @typedef {Object} Blockstore
+ * @property {function(import('./link').AnyLink): Promise<AnyBlock|undefined>} get - A function to retrieve a block by CID
+ * @property {function(import('./link').AnyLink, Uint8Array): Promise<void>} put - A function to store a block's data and CID
+ *
+ * A blockstore that caches writes to a transaction and only persists them when committed.
+*/
+export class TransactionBlockstore {
+    /** @type {Map<string, Uint8Array>} */
+    committedBlocks = new Map();
+    valet = null;
+    instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4);
+    inflightTransactions = new Set();
+    constructor(name, encryptionKey) {
+        if (name) {
+            this.valet = new Valet(name, encryptionKey);
+        }
+        this.remoteBlockFunction = null;
+    }
+    /**
+     * Get a block from the store.
+     *
+     * @param {import('./link').AnyLink} cid
+     * @returns {Promise<AnyBlock | undefined>}
+     */
+    async get(cid) {
+        const key = cid.toString();
+        // it is safe to read from the in-flight transactions becauase they are immutable
+        const bytes = await Promise.any([this.transactionsGet(key), this.committedGet(key)]).catch(e => {
+            // console.log('networkGet', cid.toString(), e)
+            return this.networkGet(key);
+        });
+        if (!bytes)
+            throw new Error('Missing block: ' + key);
+        return { cid, bytes };
+    }
+    // this iterates over the in-flight transactions
+    // and returns the first matching block it finds
+    async transactionsGet(key) {
+        for (const transaction of this.inflightTransactions) {
+            const got = await transaction.get(key);
+            if (got && got.bytes)
+                return got.bytes;
+        }
+        throw new Error('Missing block: ' + key);
+    }
+    async committedGet(key) {
+        const old = this.committedBlocks.get(key);
+        if (old)
+            return old;
+        if (!this.valet)
+            throw new Error('Missing block: ' + key);
+        const got = await this.valet.getBlock(key);
+        // console.log('committedGet: ' + key)
+        this.committedBlocks.set(key, got);
+        return got;
+    }
+    async clearCommittedCache() {
+        this.committedBlocks.clear();
+    }
+    async networkGet(key) {
+        if (this.remoteBlockFunction) {
+            // todo why is this on valet?
+            const value = await husher(key, async () => await this.remoteBlockFunction(key));
+            if (value) {
+                // console.log('networkGot: ' + key, value.length)
+                doTransaction('networkGot: ' + key, this, async (innerBlockstore) => {
+                    await innerBlockstore.put(CID.parse(key), value);
+                });
+                return value;
+            }
+        }
+        else {
+            return false;
+        }
+    }
+    /**
+     * Add a block to the store. Usually bound to a transaction by a closure.
+     * It sets the lastCid property to the CID of the block that was put.
+     * This is used by the transaction as the head of the car when written to the valet.
+     * We don't have to worry about which transaction we are when we are here because
+     * we are the transactionBlockstore.
+     *
+     * @param {import('./link').AnyLink} cid
+     * @param {Uint8Array} bytes
+     */
+    put(cid, bytes) {
+        throw new Error('use a transaction to put');
+    }
+    /**
+     * Iterate over all blocks in the store.
+     *
+     * @yields {AnyBlock}
+     * @returns {AsyncGenerator<AnyBlock>}
+     */
+    // * entries () {
+    //   // needs transaction blocks?
+    //   // for (const [str, bytes] of this.blocks) {
+    //   //   yield { cid: parse(str), bytes }
+    //   // }
+    //   for (const [str, bytes] of this.committedBlocks) {
+    //     yield { cid: parse(str), bytes }
+    //   }
+    // }
+    /**
+     * Begin a transaction. Ensures the uncommited blocks are empty at the begining.
+     * Returns the blocks to read and write during the transaction.
+     * @returns {InnerBlockstore}
+     * @memberof TransactionBlockstore
+     */
+    begin(label = '') {
+        const innerTransactionBlockstore = new InnerBlockstore(label, this);
+        this.inflightTransactions.add(innerTransactionBlockstore);
+        return innerTransactionBlockstore;
+    }
+    /**
+     * Commit the transaction. Writes the blocks to the store.
+     * @returns {Promise<void>}
+     * @memberof TransactionBlockstore
+     */
+    async commit(innerBlockstore) {
+        await this.doCommit(innerBlockstore);
+    }
+    // first get the transaction blockstore from the map of transaction blockstores
+    // then copy it to committedBlocks
+    // then write the transaction blockstore to a car
+    // then write the car to the valet
+    // then remove the transaction blockstore from the map of transaction blockstores
+    doCommit = async (innerBlockstore) => {
+        const cids = new Set();
+        for (const { cid, bytes } of innerBlockstore.entries()) {
+            const stringCid = cid.toString(); // unnecessary string conversion, can we fix upstream?
+            if (this.committedBlocks.has(stringCid)) {
+                // console.log('Duplicate block: ' + stringCid) // todo some of this can be avoided, cost is extra size on car files
+            }
+            else {
+                this.committedBlocks.set(stringCid, bytes);
+                cids.add(stringCid);
+            }
+        }
+        if (cids.size > 0 && this.valet) {
+            // console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
+            await this.valet.writeTransaction(innerBlockstore, cids);
+        }
+    };
+    /**
+     * Retire the transaction. Clears the uncommited blocks.
+     * @returns {void}
+     * @memberof TransactionBlockstore
+     */
+    retire(innerBlockstore) {
+        this.inflightTransactions.delete(innerBlockstore);
+    }
+}
+/**
+ * Runs a function on an inner blockstore, then persists the change to a car writer
+ * or other outer blockstore.
+ * @param {string} label
+ * @param {TransactionBlockstore} blockstore
+ * @param {(innerBlockstore: Blockstore) => Promise<any>} doFun
+ * @returns {Promise<any>}
+ * @memberof TransactionBlockstore
+ */
+export const doTransaction = async (label, blockstore, doFun) => {
+    // @ts-ignore
+    if (!blockstore.commit)
+        return await doFun(blockstore);
+    // @ts-ignore
+    const innerBlockstore = blockstore.begin(label);
+    try {
+        const result = await doFun(innerBlockstore);
+        // @ts-ignore
+        await blockstore.commit(innerBlockstore);
+        return result;
+    }
+    catch (e) {
+        console.error(`Transaction ${label} failed`, e, e.stack);
+        throw e;
+    }
+    finally {
+        // @ts-ignore
+        blockstore.retire(innerBlockstore);
+    }
+};
+export class InnerBlockstore {
+    /** @type {Map<string, Uint8Array>} */
+    blocks = new Map();
+    lastCid = null;
+    label = '';
+    parentBlockstore = null;
+    constructor(label, parentBlockstore) {
+        this.label = label;
+        this.parentBlockstore = parentBlockstore;
+    }
+    /**
+     * @param {import('./link').AnyLink} cid
+     * @returns {Promise<AnyBlock | undefined>}
+     */
+    async get(cid) {
+        const key = cid.toString();
+        let bytes = this.blocks.get(key);
+        if (bytes) {
+            return { cid, bytes };
+        }
+        bytes = await this.parentBlockstore.committedGet(key);
+        if (bytes) {
+            return { cid, bytes };
+        }
+    }
+    /**
+     * @param {import('./link').AnyLink} cid
+     * @param {Uint8Array} bytes
+     */
+    async put(cid, bytes) {
+        // console.log('put', cid)
+        this.blocks.set(cid.toString(), bytes);
+        this.lastCid = cid;
+    }
+    *entries() {
+        for (const [str, bytes] of this.blocks) {
+            yield { cid: parse(str), bytes };
+        }
+    }
+}

package/dist/clock.js

@@ -0,0 +1,355 @@
+import { Block, encode, decode } from 'multiformats/block';
+import { sha256 } from 'multiformats/hashes/sha2';
+import * as cbor from '@ipld/dag-cbor';
+// @ts-ignore
+import { CIDCounter } from 'prolly-trees/utils';
+/**
+ * @template T
+ * @typedef {{ parents: EventLink<T>[], data: T }} EventView
+ */
+/**
+ * @template T
+ * @typedef {import('multiformats').BlockView<EventView<T>>} EventBlockView
+ */
+/**
+ * @template T
+ * @typedef {import('multiformats').Link<EventView<T>>} EventLink
+ */
+/**
+ * @typedef {{
+*   type: 'put'|'del'
+*   key: string
+*   value: import('./link').AnyLink
+*   root: import('./link').AnyLink
+* }} EventData
+* @typedef {{
+*   root: import('./link').AnyLink
+*   head: import('./clock').EventLink<EventData>[]
+*   event: import('./clock').EventBlockView<EventData>
+* }} Result
+*/
+/**
+ * Advance the clock by adding an event.
+ *
+ * @template T
+ * @param {import('./blockstore').TransactionBlockstore} blocks Block storage.
+ * @param {EventLink<T>[]} head The head of the clock.
+ * @param {EventLink<T>} event The event to add.
+ * @returns {Promise<{head:EventLink<T>[], cids:any[]}>} The new head of the clock.
+ */
+export async function advance(blocks, head, event) {
+    /** @type {EventFetcher<T>} */
+    const events = new EventFetcher(blocks);
+    const headmap = new Map(head.map((cid) => [cid.toString(), cid]));
+    // Check if the headmap already includes the event, return head if it does
+    if (headmap.has(event.toString()))
+        return { head, cids: await events.all() };
+    // Does event contain the clock?
+    let changed = false;
+    for (const cid of head) {
+        if (await contains(events, event, cid)) {
+            headmap.delete(cid.toString());
+            headmap.set(event.toString(), event);
+            changed = true;
+        }
+    }
+    // If the headmap has been changed, return the new headmap values
+    if (changed) {
+        return { head: [...headmap.values()], cids: await events.all() };
+    }
+    // Does clock contain the event?
+    for (const p of head) {
+        if (await contains(events, p, event)) {
+            return { head, cids: await events.all() };
+        }
+    }
+    // Return the head concatenated with the new event if it passes both checks
+    return { head: head.concat(event), cids: await events.all() };
+}
+/**
+ * @template T
+ * @implements {EventBlockView<T>}
+ */
+export class EventBlock extends Block {
+    /**
+     * @param {object} config
+     * @param {EventLink<T>} config.cid
+     * @param {Event} config.value
+     * @param {Uint8Array} config.bytes
+     */
+    constructor({ cid, value, bytes }) {
+        // @ts-expect-error
+        super({ cid, value, bytes });
+    }
+    /**
+     * @template T
+     * @param {T} data
+     * @param {EventLink<T>[]} [parents]
+     */
+    static create(data, parents) {
+        return encodeEventBlock({ data, parents: parents ?? [] });
+    }
+}
+/** @template T */
+export class EventFetcher {
+    /** @param {import('./blockstore').TransactionBlockstore} blocks */
+    constructor(blocks) {
+        /** @private */
+        this._blocks = blocks;
+        this._cids = new CIDCounter();
+        this._cache = new Map();
+    }
+    /**
+     * @param {EventLink<T>} link
+     * @returns {Promise<EventBlockView<T>>}
+     */
+    async get(link) {
+        const slink = link.toString();
+        // console.log('get', link.toString())
+        if (this._cache.has(slink))
+            return this._cache.get(slink);
+        const block = await this._blocks.get(link);
+        this._cids.add({ address: link });
+        if (!block)
+            throw new Error(`missing block: ${link}`);
+        const got = decodeEventBlock(block.bytes);
+        this._cache.set(slink, got);
+        return got;
+    }
+    async all() {
+        // await Promise.all([...this._cids])
+        return this._cids.all();
+    }
+}
+/**
+ * @template T
+ * @param {EventView<T>} value
+ * @returns {Promise<EventBlockView<T>>}
+ */
+export async function encodeEventBlock(value) {
+    // TODO: sort parents
+    const { cid, bytes } = await encode({ value, codec: cbor, hasher: sha256 });
+    // @ts-expect-error
+    return new Block({ cid, value, bytes });
+}
+/**
+ * @template T
+ * @param {Uint8Array} bytes
+ * @returns {Promise<EventBlockView<T>>}
+ */
+export async function decodeEventBlock(bytes) {
+    const { cid, value } = await decode({ bytes, codec: cbor, hasher: sha256 });
+    // @ts-expect-error
+    return new Block({ cid, value, bytes });
+}
+/**
+ * Returns true if event "a" contains event "b". Breadth first search.
+ * @template T
+ * @param {EventFetcher} events
+ * @param {EventLink<T>} a
+ * @param {EventLink<T>} b
+ */
+async function contains(events, a, b) {
+    if (a.toString() === b.toString())
+        return true;
+    const [{ value: aevent }, { value: bevent }] = await Promise.all([events.get(a), events.get(b)]);
+    const links = [...aevent.parents];
+    while (links.length) {
+        const link = links.shift();
+        if (!link)
+            break;
+        if (link.toString() === b.toString())
+            return true;
+        // if any of b's parents are this link, then b cannot exist in any of the
+        // tree below, since that would create a cycle.
+        if (bevent.parents.some((p) => link.toString() === p.toString()))
+            continue;
+        const { value: event } = await events.get(link);
+        links.push(...event.parents);
+    }
+    return false;
+}
+/**
+ * @template T
+ * @param {import('./blockstore').TransactionBlockstore} blocks Block storage.
+ * @param {EventLink<T>[]} head
+ * @param {object} [options]
+ * @param {(b: EventBlockView<T>) => string} [options.renderNodeLabel]
+ */
+export async function* vis(blocks, head, options = {}) {
+    // @ts-ignore
+    const renderNodeLabel = options.renderNodeLabel ?? ((b) => b.value.data.value);
+    const events = new EventFetcher(blocks);
+    yield 'digraph clock {';
+    yield '  node [shape=point fontname="Courier"]; head;';
+    const hevents = await Promise.all(head.map((link) => events.get(link)));
+    const links = [];
+    const nodes = new Set();
+    for (const e of hevents) {
+        nodes.add(e.cid.toString());
+        yield `  node [shape=oval fontname="Courier"]; ${e.cid} [label="${renderNodeLabel(e)}"];`;
+        yield `  head -> ${e.cid};`;
+        for (const p of e.value.parents) {
+            yield `  ${e.cid} -> ${p};`;
+        }
+        links.push(...e.value.parents);
+    }
+    while (links.length) {
+        const link = links.shift();
+        if (!link)
+            break;
+        if (nodes.has(link.toString()))
+            continue;
+        nodes.add(link.toString());
+        const block = await events.get(link);
+        yield `  node [shape=oval]; ${link} [label="${renderNodeLabel(block)}" fontname="Courier"];`;
+        for (const p of block.value.parents) {
+            yield `  ${link} -> ${p};`;
+        }
+        links.push(...block.value.parents);
+    }
+    yield '}';
+}
+export async function findEventsToSync(blocks, head) {
+    // const callTag = Math.random().toString(36).substring(7)
+    const events = new EventFetcher(blocks);
+    // console.time(callTag + '.findCommonAncestorWithSortedEvents')
+    const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head);
+    // console.timeEnd(callTag + '.findCommonAncestorWithSortedEvents')
+    // console.log('sorted', sorted.length)
+    // console.time(callTag + '.contains')
+    const toSync = await asyncFilter(sorted, async (uks) => !(await contains(events, ancestor, uks.cid)));
+    // console.timeEnd(callTag + '.contains')
+    return { cids: events.all(), events: toSync };
+}
+const asyncFilter = async (arr, predicate) => Promise.all(arr.map(predicate)).then((results) => arr.filter((_v, index) => results[index]));
+export async function findCommonAncestorWithSortedEvents(events, children) {
+    // const callTag = Math.random().toString(36).substring(7)
+    // console.time(callTag + '.findCommonAncestor')
+    const ancestor = await findCommonAncestor(events, children);
+    // console.timeEnd(callTag + '.findCommonAncestor')
+    if (!ancestor) {
+        throw new Error('failed to find common ancestor event');
+    }
+    // console.time(callTag + '.findSortedEvents')
+    const sorted = await findSortedEvents(events, children, ancestor);
+    // console.timeEnd(callTag + '.findSortedEvents')
+    return { ancestor, sorted };
+}
+/**
+ * Find the common ancestor event of the passed children. A common ancestor is
+ * the first single event in the DAG that _all_ paths from children lead to.
+ *
+ * @param {import('./clock').EventFetcher} events
+ * @param  {import('./clock').EventLink<EventData>[]} children
+ */
+async function findCommonAncestor(events, children) {
+    if (!children.length)
+        return;
+    const candidates = children.map((c) => [c]);
+    while (true) {
+        let changed = false;
+        for (const c of candidates) {
+            const candidate = await findAncestorCandidate(events, c[c.length - 1]);
+            if (!candidate)
+                continue;
+            changed = true;
+            c.push(candidate);
+            const ancestor = findCommonString(candidates);
+            if (ancestor)
+                return ancestor;
+        }
+        if (!changed)
+            return;
+    }
+}
+/**
+ * @param {import('./clock').EventFetcher} events
+ * @param {import('./clock').EventLink<EventData>} root
+ */
+async function findAncestorCandidate(events, root) {
+    const { value: event } = await events.get(root);
+    if (!event.parents.length)
+        return root;
+    return event.parents.length === 1 ? event.parents[0] : findCommonAncestor(events, event.parents);
+}
+/**
+ * @template {{ toString: () => string }} T
+ * @param  {Array<T[]>} arrays
+ */
+function findCommonString(arrays) {
+    arrays = arrays.map((a) => [...a]);
+    for (const arr of arrays) {
+        for (const item of arr) {
+            let matched = true;
+            for (const other of arrays) {
+                if (arr === other)
+                    continue;
+                matched = other.some((i) => String(i) === String(item));
+                if (!matched)
+                    break;
+            }
+            if (matched)
+                return item;
+        }
+    }
+}
+/**
+ * Find and sort events between the head(s) and the tail.
+ * @param {import('./clock').EventFetcher} events
+ * @param {import('./clock').EventLink<EventData>[]} head
+ * @param {import('./clock').EventLink<EventData>} tail
+ */
+async function findSortedEvents(events, head, tail) {
+    // const callTag = Math.random().toString(36).substring(7)
+    // get weighted events - heavier events happened first
+    /** @type {Map<string, { event: import('./clock').EventBlockView<EventData>, weight: number }>} */
+    const weights = new Map();
+    const all = await Promise.all(head.map((h) => findEvents(events, h, tail)));
+    for (const arr of all) {
+        for (const { event, depth } of arr) {
+            // console.log('event value', event.value.data.value)
+            const info = weights.get(event.cid.toString());
+            if (info) {
+                info.weight += depth;
+            }
+            else {
+                weights.set(event.cid.toString(), { event, weight: depth });
+            }
+        }
+    }
+    // group events into buckets by weight
+    /** @type {Map<number, import('./clock').EventBlockView<EventData>[]>} */
+    const buckets = new Map();
+    for (const { event, weight } of weights.values()) {
+        const bucket = buckets.get(weight);
+        if (bucket) {
+            bucket.push(event);
+        }
+        else {
+            buckets.set(weight, [event]);
+        }
+    }
+    // sort by weight, and by CID within weight
+    const sorted = Array.from(buckets)
+        .sort((a, b) => b[0] - a[0])
+        .flatMap(([, es]) => es.sort((a, b) => (String(a.cid) < String(b.cid) ? -1 : 1)));
+    // console.log('sorted', sorted.map(s => s.value.data.value))
+    return sorted;
+}
+/**
+ * @param {EventFetcher} events
+ * @param {EventLink<EventData>} start
+ * @param {EventLink<EventData>} end
+ * @returns {Promise<Array<{ event: EventBlockView<EventData>, depth: number }>>}
+ */
+async function findEvents(events, start, end, depth = 0) {
+    // console.log('findEvents', start)
+    const event = await events.get(start);
+    const acc = [{ event, depth }];
+    const { parents } = event.value;
+    if (parents.length === 1 && String(parents[0]) === String(end))
+        return acc;
+    const rest = await Promise.all(parents.map((p) => findEvents(events, p, end, depth + 1)));
+    return acc.concat(...rest);
+}

package/dist/crypto.js

@@ -0,0 +1,59 @@
+// @ts-nocheck
+import * as codec from 'encrypted-block';
+import { create, load } from 'prolly-trees/cid-set';
+import { CID } from 'multiformats';
+import { encode, decode, create as mfCreate } from 'multiformats/block';
+import * as dagcbor from '@ipld/dag-cbor';
+import { sha256 as hasher } from 'multiformats/hashes/sha2';
+const createBlock = (bytes, cid) => mfCreate({ cid, bytes, hasher, codec });
+const encrypt = async function* ({ get, cids, hasher, key, cache, chunker, root }) {
+    const set = new Set();
+    let eroot;
+    for (const string of cids) {
+        const cid = CID.parse(string);
+        const unencrypted = await get(cid);
+        const block = await encode({ ...await codec.encrypt({ ...unencrypted, key }), codec, hasher });
+        // console.log(`encrypting ${string} as ${block.cid}`)
+        yield block;
+        set.add(block.cid.toString());
+        if (unencrypted.cid.equals(root))
+            eroot = block.cid;
+    }
+    if (!eroot)
+        throw new Error('cids does not include root');
+    const list = [...set].map(s => CID.parse(s));
+    let last;
+    for await (const node of create({ list, get, cache, chunker, hasher, codec: dagcbor })) {
+        const block = await node.block;
+        yield block;
+        last = block;
+    }
+    const head = [eroot, last.cid];
+    const block = await encode({ value: head, codec: dagcbor, hasher });
+    yield block;
+};
+const decrypt = async function* ({ root, get, key, cache, chunker, hasher }) {
+    const o = { ...await get(root), codec: dagcbor, hasher };
+    const decodedRoot = await decode(o);
+    // console.log('decodedRoot', decodedRoot)
+    const { value: [eroot, tree] } = decodedRoot;
+    const rootBlock = await get(eroot); // should I decrypt?
+    const cidset = await load({ cid: tree, get, cache, chunker, codec, hasher });
+    const { result: nodes } = await cidset.getAllEntries();
+    const unwrap = async (eblock) => {
+        const { bytes, cid } = await codec.decrypt({ ...eblock, key }).catch(e => {
+            console.log('ekey', e);
+            throw new Error('bad key: ' + key.toString('hex'));
+        });
+        const block = await createBlock(bytes, cid);
+        return block;
+    };
+    const promises = [];
+    for (const { cid } of nodes) {
+        if (!rootBlock.cid.equals(cid))
+            promises.push(get(cid).then(unwrap));
+    }
+    yield* promises;
+    yield unwrap(rootBlock);
+};
+export { encrypt, decrypt };