helia 6.1.4 → 7.0.0

package/dist/src/storage.js

@@ -0,0 +1,225 @@
+import { BlockNotFoundWhileOfflineError, InvalidConfigurationError, LoadBlockFailedError } from '@helia/utils';
+import { isPromise } from '@helia/utils';
+import { InvalidMultihashError, InvalidParametersError, setMaxListeners } from '@libp2p/interface';
+import { anySignal } from 'any-signal';
+import { IdentityBlockstore } from 'blockstore-core/identity';
+import filter from 'it-filter';
+import forEach from 'it-foreach';
+import { CustomProgressEvent } from 'progress-events';
+import { equals as uint8ArrayEquals } from 'uint8arrays/equals';
+const DEFAULT_MAX_IDENTITY_HASH_DIGEST_LENGTH = 128;
+export class Storage {
+    child;
+    getHasher;
+    log;
+    logger;
+    blockBrokers;
+    /**
+     * Create a new BlockStorage
+     */
+    constructor(components, init = {}) {
+        this.log = components.logger.forComponent('helia:networked-storage');
+        this.logger = components.logger;
+        this.blockBrokers = components.blockBrokers;
+        this.child = new IdentityBlockstore(components.blockstore, {
+            maxDigestLength: init.maxIdentityHashDigestLength ?? DEFAULT_MAX_IDENTITY_HASH_DIGEST_LENGTH
+        });
+        this.getHasher = components.getHasher;
+    }
+    /**
+     * Put a block to the underlying datastore
+     */
+    async put(cid, block, options = {}) {
+        if (await this.child.has(cid, options)) {
+            options.onProgress?.(new CustomProgressEvent('blocks:put:duplicate', cid));
+            return cid;
+        }
+        options.onProgress?.(new CustomProgressEvent('blocks:put:providers:notify', cid));
+        await Promise.all(this.blockBrokers.map(async (broker) => broker.announce?.(cid, options)));
+        options.onProgress?.(new CustomProgressEvent('blocks:put:blockstore:put', cid));
+        return this.child.put(cid, block, options);
+    }
+    /**
+     * Put a multiple blocks to the underlying datastore
+     */
+    async *putMany(blocks, options = {}) {
+        const missingBlocks = filter(blocks, async ({ cid }) => {
+            const has = await this.child.has(cid, options);
+            if (has) {
+                options.onProgress?.(new CustomProgressEvent('blocks:put-many:duplicate', cid));
+            }
+            return !has;
+        });
+        const notifyEach = forEach(missingBlocks, async ({ cid }) => {
+            options.onProgress?.(new CustomProgressEvent('blocks:put-many:providers:notify', cid));
+            await Promise.all(this.blockBrokers.map(async (broker) => broker.announce?.(cid, options)));
+        });
+        options.onProgress?.(new CustomProgressEvent('blocks:put-many:blockstore:put-many'));
+        yield* this.child.putMany(notifyEach, options);
+    }
+    /**
+     * Get a block by cid
+     */
+    async *get(cid, options = {}) {
+        const has = await this.child.has(cid, options);
+        const offline = options.offline === true;
+        if (!has) {
+            if (offline) {
+                throw new BlockNotFoundWhileOfflineError('The block was present in the blockstore and the node is running offline so cannot fetch it');
+            }
+            const hasher = await this.getHasher(cid.multihash.code);
+            options?.signal?.throwIfAborted();
+            // we do not have the block locally, get it from a block provider
+            options.onProgress?.(new CustomProgressEvent('blocks:get:providers:get', cid));
+            const block = await raceBlockRetrievers(cid, this.blockBrokers, hasher, {
+                ...options,
+                log: this.log
+            });
+            options.onProgress?.(new CustomProgressEvent('blocks:get:blockstore:put', cid));
+            await this.child.put(cid, block, options);
+            // notify other block providers of the new block
+            options.onProgress?.(new CustomProgressEvent('blocks:get:providers:notify', cid));
+            await Promise.all(this.blockBrokers.map(async (broker) => broker.announce?.(cid, options)));
+            yield block;
+            return;
+        }
+        options.onProgress?.(new CustomProgressEvent('blocks:get:blockstore:get', cid));
+        yield* this.child.get(cid, options);
+    }
+    /**
+     * Get multiple blocks back from an (async) iterable of cids
+     */
+    async *getMany(cids, options = {}) {
+        options.onProgress?.(new CustomProgressEvent('blocks:get-many:blockstore:get-many'));
+        yield* this.child.getMany(forEach(cids, async (cid) => {
+            const has = await this.child.has(cid, options);
+            const offline = options.offline === true;
+            if (!has) {
+                if (offline) {
+                    throw new BlockNotFoundWhileOfflineError('The block was present in the blockstore and the node is running offline so cannot fetch it');
+                }
+                const hasher = await this.getHasher(cid.multihash.code);
+                options?.signal?.throwIfAborted();
+                // we do not have the block locally, get it from a block provider
+                options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:get', cid));
+                const block = await raceBlockRetrievers(cid, this.blockBrokers, hasher, {
+                    ...options,
+                    log: this.log
+                });
+                options.onProgress?.(new CustomProgressEvent('blocks:get-many:blockstore:put', cid));
+                await this.child.put(cid, block, options);
+                // notify other block providers of the new block
+                options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:notify', cid));
+                await Promise.all(this.blockBrokers.map(async (broker) => broker.announce?.(cid, options)));
+            }
+        }));
+    }
+    /**
+     * Delete a block from the blockstore
+     */
+    async delete(cid, options = {}) {
+        options.onProgress?.(new CustomProgressEvent('blocks:delete:blockstore:delete', cid));
+        await this.child.delete(cid, options);
+    }
+    /**
+     * Delete multiple blocks from the blockstore
+     */
+    async *deleteMany(cids, options = {}) {
+        options.onProgress?.(new CustomProgressEvent('blocks:delete-many:blockstore:delete-many'));
+        yield* this.child.deleteMany((async function* () {
+            for await (const cid of cids) {
+                yield cid;
+            }
+        }()), options);
+    }
+    async has(cid, options = {}) {
+        return this.child.has(cid, options);
+    }
+    async *getAll(options = {}) {
+        options.onProgress?.(new CustomProgressEvent('blocks:get-all:blockstore:get-many'));
+        yield* this.child.getAll(options);
+    }
+}
+/**
+ * Race block providers cancelling any pending requests once the block has been
+ * found.
+ */
+async function raceBlockRetrievers(cid, blockBrokers, hasher, options) {
+    const validateFn = getCidBlockVerifierFunction(cid, hasher);
+    const controller = new AbortController();
+    const signal = anySignal([controller.signal, options.signal]);
+    setMaxListeners(Infinity, controller.signal, signal);
+    const retrievers = [];
+    for (const broker of blockBrokers) {
+        if (isRetrievingBlockBroker(broker)) {
+            retrievers.push(broker);
+        }
+    }
+    if (retrievers.length === 0) {
+        throw new InvalidConfigurationError(`No block brokers capable of retrieving blocks are configured, the CID ${cid} cannot be fetched from the network`);
+    }
+    try {
+        return await Promise.any(retrievers
+            .map(async (retriever) => {
+            try {
+                let blocksWereValidated = false;
+                const block = await retriever.retrieve(cid, {
+                    ...options,
+                    signal,
+                    validateFn: async (block) => {
+                        await validateFn(block);
+                        options.signal?.throwIfAborted();
+                        blocksWereValidated = true;
+                    }
+                });
+                if (!blocksWereValidated) {
+                    // the blockBroker either did not throw an error when attempting to validate the block
+                    // or did not call the validateFn at all. We should validate the block ourselves
+                    await validateFn(block);
+                    options.signal?.throwIfAborted();
+                }
+                return block;
+            }
+            catch (err) {
+                options.log.error('could not retrieve verified block for %c from %s - %e', cid, retriever.name, err);
+                throw err;
+            }
+        }));
+    }
+    catch (err) {
+        throw new LoadBlockFailedError(err.errors, `Failed to load block for ${cid}`);
+    }
+    finally {
+        // we have the block from the fastest block retriever, abort any still
+        // in-flight retrieve attempts
+        controller.abort();
+        signal.clear();
+    }
+}
+function isRetrievingBlockBroker(broker) {
+    return typeof broker.retrieve === 'function';
+}
+export const getCidBlockVerifierFunction = (cid, hasher) => {
+    if (hasher == null) {
+        throw new InvalidParametersError(`No hasher configured for multihash code 0x${cid.multihash.code.toString(16)}, please configure one. You can look up which hash this is at https://github.com/multiformats/multicodec/blob/master/table.csv`);
+    }
+    return async (block) => {
+        // verify block
+        let hash;
+        const res = hasher.digest(block, {
+            // support truncated hashes where they are truncated
+            truncate: cid.multihash.digest.byteLength
+        });
+        if (isPromise(res)) {
+            hash = await res;
+        }
+        else {
+            hash = res;
+        }
+        if (!uint8ArrayEquals(hash.digest, cid.multihash.digest)) {
+            // if a hash mismatch occurs for a TrustlessGatewayBlockBroker, we should try another gateway
+            throw new InvalidMultihashError('Hash of downloaded block did not match multihash from passed CID');
+        }
+    };
+};
+//# sourceMappingURL=storage.js.map

package/dist/src/storage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.js","sourceRoot":"","sources":["../../src/storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,8BAA8B,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAA;AAC9G,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAClG,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACtC,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAC7D,OAAO,MAAM,MAAM,WAAW,CAAA;AAC9B,OAAO,OAAO,MAAM,YAAY,CAAA;AAChC,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,EAAE,MAAM,IAAI,gBAAgB,EAAE,MAAM,oBAAoB,CAAA;AAoB/D,MAAM,uCAAuC,GAAG,GAAG,CAAA;AAEnD,MAAM,OAAO,OAAO;IACC,KAAK,CAAY;IACjB,SAAS,CAAc;IAChC,GAAG,CAAQ;IACF,MAAM,CAAiB;IACvB,YAAY,CAAU;IAEzC;;OAEG;IACH,YAAa,UAAqC,EAAE,OAAoB,EAAE;QACxE,IAAI,CAAC,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,yBAAyB,CAAC,CAAA;QACpE,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAA;QAC/B,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,YAAY,CAAA;QAC3C,IAAI,CAAC,KAAK,GAAG,IAAI,kBAAkB,CAAC,UAAU,CAAC,UAAU,EAAE;YACzD,eAAe,EAAE,IAAI,CAAC,2BAA2B,IAAI,uCAAuC;SAC7F,CAAC,CAAA;QACF,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC,SAAS,CAAA;IACvC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAE,GAAQ,EAAE,KAAiB,EAAE,UAAkE,EAAE;QAC1G,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;YACvC,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,sBAAsB,EAAE,GAAG,CAAC,CAAC,CAAA;YAC/E,OAAO,GAAG,CAAA;QACZ,CAAC;QAED,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,6BAA6B,EAAE,GAAG,CAAC,CAAC,CAAA;QAEtF,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CACvE,CAAA;QAED,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,2BAA2B,EAAE,GAAG,CAAC,CAAC,CAAA;QAEpF,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;IAC5C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAE,OAAO,CAAE,MAAsD,EAAE,UAAuE,EAAE;QAChJ,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,EAAoB,EAAE;YACvE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YAE9C,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,2BAA2B,EAAE,GAAG,CAAC,CAAC,CAAA;YACtF,CAAC;YAED,OAAO,CAAC,GAAG,CAAA;QACb,CAAC,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,EAAiB,EAAE;YACzE,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,kCAAkC,EAAE,GAAG,CAAC,CAAC,CAAA;YAC3F,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CACvE,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAC,qCAAqC,CAAC,CAAC,CAAA;QACpF,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;IACjD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAE,GAAG,CAAE,GAAQ,EAAE,UAAsF,EAAE;QAC7G,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,KAAK,IAAI,CAAA;QAExC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,IAAI,8BAA8B,CAAC,4FAA4F,CAAC,CAAA;YACxI,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;YACvD,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;YAEjC,iEAAiE;YAEjE,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,0BAA0B,EAAE,GAAG,CAAC,CAAC,CAAA;YACnF,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE;gBACtE,GAAG,OAAO;gBACV,GAAG,EAAE,IAAI,CAAC,GAAG;aACd,CAAC,CAAA;YACF,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,2BAA2B,EAAE,GAAG,CAAC,CAAC,CAAA;YACpF,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;YAEzC,gDAAgD;YAChD,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,6BAA6B,EAAE,GAAG,CAAC,CAAC,CAAA;YACtF,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CACvE,CAAA;YAED,MAAM,KAAK,CAAA;YACX,OAAM;QACR,CAAC;QAED,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,2BAA2B,EAAE,GAAG,CAAC,CAAC,CAAA;QAEpF,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAE,OAAO,CAAE,IAAwC,EAAE,UAA2F,EAAE;QACtJ,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAC,qCAAqC,CAAC,CAAC,CAAA;QAEpF,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAiB,EAAE;YACpE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,KAAK,IAAI,CAAA;YAExC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,IAAI,8BAA8B,CAAC,4FAA4F,CAAC,CAAA;gBACxI,CAAC;gBAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;gBACvD,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;gBAEjC,iEAAiE;gBACjE,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,+BAA+B,EAAE,GAAG,CAAC,CAAC,CAAA;gBACxF,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE;oBACtE,GAAG,OAAO;oBACV,GAAG,EAAE,IAAI,CAAC,GAAG;iBACd,CAAC,CAAA;gBACF,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAA;gBACzF,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;gBAEzC,gDAAgD;gBAChD,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,kCAAkC,EAAE,GAAG,CAAC,CAAC,CAAA;gBAC3F,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CACvE,CAAA;YACH,CAAC;QACH,CAAC,CAAC,CAAC,CAAA;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAE,GAAQ,EAAE,UAAqE,EAAE;QAC7F,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,iCAAiC,EAAE,GAAG,CAAC,CAAC,CAAA;QAE1F,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACvC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAE,UAAU,CAAE,IAAwC,EAAE,UAA0E,EAAE;QACxI,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAC,2CAA2C,CAAC,CAAC,CAAA;QAC1F,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,SAAU,CAAC;YAC7C,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBAC7B,MAAM,GAAG,CAAA;YACX,CAAC;QACH,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAA;IAChB,CAAC;IAED,KAAK,CAAC,GAAG,CAAE,GAAQ,EAAE,UAAwB,EAAE;QAC7C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,CAAE,MAAM,CAAE,UAAsE,EAAE;QACtF,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAC,oCAAoC,CAAC,CAAC,CAAA;QACnF,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACpC,CAAC;CACF;AAED;;;GAGG;AACH,KAAK,UAAU,mBAAmB,CAAE,GAAQ,EAAE,YAA2B,EAAE,MAAuB,EAAE,OAAqC;IACvI,MAAM,UAAU,GAAG,2BAA2B,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IAE3D,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAA;IAC7D,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAEpD,MAAM,UAAU,GAA4D,EAAE,CAAA;IAE9E,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;QAClC,IAAI,uBAAuB,CAAC,MAAM,CAAC,EAAE,CAAC;YACpC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACzB,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,yBAAyB,CAAC,yEAAyE,GAAG,qCAAqC,CAAC,CAAA;IACxJ,CAAC;IAED,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,GAAG,CACtB,UAAU;aACP,GAAG,CAAC,KAAK,EAAC,SAAS,EAAC,EAAE;YACrB,IAAI,CAAC;gBACH,IAAI,mBAAmB,GAAG,KAAK,CAAA;gBAC/B,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE;oBAC1C,GAAG,OAAO;oBACV,MAAM;oBACN,UAAU,EAAE,KAAK,EAAE,KAAiB,EAAiB,EAAE;wBACrD,MAAM,UAAU,CAAC,KAAK,CAAC,CAAA;wBACvB,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;wBAChC,mBAAmB,GAAG,IAAI,CAAA;oBAC5B,CAAC;iBACF,CAAC,CAAA;gBAEF,IAAI,CAAC,mBAAmB,EAAE,CAAC;oBACzB,sFAAsF;oBACtF,gFAAgF;oBAChF,MAAM,UAAU,CAAC,KAAK,CAAC,CAAA;oBACvB,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;gBAClC,CAAC;gBAED,OAAO,KAAK,CAAA;YACd,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,uDAAuD,EAAE,GAAG,EAAE,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;gBACpG,MAAM,GAAG,CAAA;YACX,CAAC;QACH,CAAC,CAAC,CACL,CAAA;IACH,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,MAAM,IAAI,oBAAoB,CAAC,GAAG,CAAC,MAAM,EAAE,4BAA4B,GAAG,EAAE,CAAC,CAAA;IAC/E,CAAC;YAAS,CAAC;QACT,sEAAsE;QACtE,8BAA8B;QAC9B,UAAU,CAAC,KAAK,EAAE,CAAA;QAClB,MAAM,CAAC,KAAK,EAAE,CAAA;IAChB,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAE,MAAmB;IACnD,OAAO,OAAO,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAA;AAC9C,CAAC;AAED,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,GAAQ,EAAE,MAAuB,EAAiD,EAAE;IAC9H,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,sBAAsB,CAAC,6CAA6C,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,gIAAgI,CAAC,CAAA;IAChP,CAAC;IAED,OAAO,KAAK,EAAE,KAAiB,EAAiB,EAAE;QAChD,eAAe;QACf,IAAI,IAA6B,CAAA;QACjC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE;YAC/B,oDAAoD;YACpD,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU;SAC1C,CAAC,CAAA;QAEF,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,GAAG,MAAM,GAAG,CAAA;QAClB,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,GAAG,CAAA;QACZ,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YACzD,6FAA6F;YAC7F,MAAM,IAAI,qBAAqB,CAAC,kEAAkE,CAAC,CAAA;QACrG,CAAC;IACH,CAAC,CAAA;AACH,CAAC,CAAA"}

package/dist/src/version.d.ts

@@ -1,3 +1,3 @@
-export declare const version = "6.1.4";
+export declare const version = "7.0.0";
 export declare const name = "helia";
 //# sourceMappingURL=version.d.ts.map

package/dist/src/version.js

@@ -1,3 +1,3 @@
-export const version = '6.1.4';
+export const version = '7.0.0';
 export const name = 'helia';
 //# sourceMappingURL=version.js.map

package/dist/typedoc-urls.json

@@ -1,10 +1,7 @@
 {
-  "DAGWalker": "https://ipfs.github.io/helia/interfaces/helia.DAGWalker.html",
-  ".:DAGWalker": "https://ipfs.github.io/helia/interfaces/helia.DAGWalker.html",
-  "DefaultLibp2pServices": "https://ipfs.github.io/helia/interfaces/helia.DefaultLibp2pServices.html",
-  "Libp2pDefaultsOptions": "https://ipfs.github.io/helia/interfaces/helia.Libp2pDefaultsOptions.html",
+  "HeliaInit": "https://ipfs.github.io/helia/interfaces/helia.HeliaInit.html",
   "createHelia": "https://ipfs.github.io/helia/functions/helia.createHelia.html",
   ".:createHelia": "https://ipfs.github.io/helia/functions/helia.createHelia.html",
-  "heliaDefaults": "https://ipfs.github.io/helia/functions/helia.heliaDefaults.html",
-  "libp2pDefaults": "https://ipfs.github.io/helia/functions/helia.libp2pDefaults.html"
+  "createHeliaLight": "https://ipfs.github.io/helia/functions/helia.createHeliaLight.html",
+  ".:createHeliaLight": "https://ipfs.github.io/helia/functions/helia.createHeliaLight.html"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "helia",
-  "version": "6.1.4",
+  "version": "7.0.0",
   "description": "An implementation of IPFS in JavaScript",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/ipfs/helia/tree/main/packages/helia#readme",
@@ -49,49 +49,46 @@
     "prepublishOnly": "node scripts/update-version.js && npm run build"
   },
   "dependencies": {
-    "@chainsafe/libp2p-noise": "^17.0.0",
-    "@chainsafe/libp2p-yamux": "^8.0.1",
-    "@helia/block-brokers": "^5.2.4",
-    "@helia/delegated-routing-v1-http-api-client": "^6.0.1",
-    "@helia/interface": "^6.2.1",
-    "@helia/routers": "^5.1.1",
-    "@helia/utils": "^2.5.2",
-    "@ipshipyard/libp2p-auto-tls": "^2.0.1",
-    "@libp2p/autonat": "^3.0.15",
-    "@libp2p/bootstrap": "^12.0.16",
-    "@libp2p/circuit-relay-v2": "^4.2.0",
-    "@libp2p/config": "^1.1.27",
-    "@libp2p/dcutr": "^3.0.15",
-    "@libp2p/http": "^2.0.1",
-    "@libp2p/identify": "^4.1.0",
-    "@libp2p/interface": "^3.2.0",
-    "@libp2p/kad-dht": "^16.2.0",
-    "@libp2p/keychain": "^6.0.12",
-    "@libp2p/mdns": "^12.0.16",
-    "@libp2p/mplex": "^12.0.16",
-    "@libp2p/ping": "^3.1.0",
-    "@libp2p/tcp": "^11.0.15",
-    "@libp2p/tls": "^3.0.15",
-    "@libp2p/upnp-nat": "^4.0.15",
-    "@libp2p/webrtc": "^6.0.16",
-    "@libp2p/websockets": "^10.1.8",
-    "@multiformats/dns": "^1.0.13",
-    "blockstore-core": "^6.1.3",
-    "datastore-core": "^11.0.3",
-    "interface-datastore": "^9.0.3",
-    "ipns": "^10.1.3",
-    "libp2p": "^3.2.0",
-    "multiformats": "^13.4.2"
+    "@helia/bitswap": "^4.0.0",
+    "@helia/http": "^4.0.0",
+    "@helia/interface": "^7.0.0",
+    "@helia/libp2p": "^1.0.0",
+    "@helia/utils": "^3.0.0",
+    "@ipld/dag-cbor": "^10.0.1",
+    "@ipld/dag-json": "^11.0.0",
+    "@ipld/dag-pb": "^4.1.7",
+    "@ipshipyard/crypto": "^1.1.0",
+    "@ipshipyard/keychain": "^1.0.2",
+    "@libp2p/interface": "^3.2.4",
+    "@libp2p/utils": "^7.2.3",
+    "@multiformats/dns": "^1.0.15",
+    "@multiformats/multiaddr": "^13.0.3",
+    "abort-error": "^1.0.2",
+    "any-signal": "^4.2.0",
+    "birnam": "^1.0.0",
+    "blockstore-core": "^7.0.1",
+    "cborg": "^5.1.4",
+    "datastore-core": "^12.0.1",
+    "interface-blockstore": "^7.0.1",
+    "interface-datastore": "^10.0.1",
+    "it-drain": "^3.0.12",
+    "it-filter": "^3.1.6",
+    "it-foreach": "^2.1.7",
+    "it-merge": "^3.0.14",
+    "main-event": "^1.0.4",
+    "mortice": "^3.3.1",
+    "multiformats": "^14.0.0",
+    "progress-events": "^1.1.0",
+    "uint8arrays": "^6.1.1"
   },
   "devDependencies": {
-    "@multiformats/mafmt": "^12.1.6",
-    "@multiformats/multiaddr": "^13.0.1",
-    "aegir": "^47.1.5",
+    "aegir": "^48.0.11",
+    "delay": "^7.0.0",
     "it-all": "^3.0.11",
-    "it-drain": "^3.0.12"
-  },
-  "browser": {
-    "./dist/src/utils/libp2p-defaults.js": "./dist/src/utils/libp2p-defaults.browser.js"
+    "it-map": "^3.1.6",
+    "it-to-buffer": "^5.0.0",
+    "sinon": "^22.0.0",
+    "sinon-ts": "^2.0.0"
   },
   "sideEffects": false
 }

package/src/block-storage.ts

@@ -0,0 +1,194 @@
+import { BlockPinnedError } from '@helia/utils'
+import { start, stop } from '@libp2p/interface'
+import createMortice from 'mortice'
+import type { Routing } from '@helia/interface'
+import type { Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, SessionBlockstore } from '@helia/interface'
+import type { Pins } from '@helia/interface'
+import type { AbortOptions, Startable } from '@libp2p/interface'
+import type { Blockstore, InputPair } from 'interface-blockstore'
+import type { Mortice } from 'mortice'
+import type { CID } from 'multiformats/cid'
+import type { ProgressOptions } from 'progress-events'
+
+export interface BlockStorageInit {
+  holdGcLock?: boolean
+}
+
+export interface GetOptions extends AbortOptions {
+  progress?(evt: Event): void
+}
+
+/**
+ * BlockStorage is a hybrid blockstore that puts/gets blocks from a configured
+ * blockstore (that may be on disk, s3, or something else). If the blocks are
+ * not present Bitswap will be used to fetch them from network peers.
+ */
+export class BlockStorage implements Blocks, Startable {
+  public lock: Mortice
+  private readonly child: Blocks
+  private readonly pins: Pins
+  private readonly routing: Routing
+  private started: boolean
+
+  /**
+   * Create a new BlockStorage
+   */
+  constructor (blockstore: Blocks, pins: Pins, routing: Routing, options: BlockStorageInit = {}) {
+    this.child = blockstore
+    this.pins = pins
+    this.routing = routing
+    this.lock = createMortice({
+      singleProcess: options.holdGcLock
+    })
+    this.started = false
+  }
+
+  isStarted (): boolean {
+    return this.started
+  }
+
+  async start (): Promise<void> {
+    await start(this.child)
+    this.started = true
+  }
+
+  async stop (): Promise<void> {
+    await stop(this.child)
+    this.started = false
+  }
+
+  unwrap (): Blockstore {
+    return this.child
+  }
+
+  /**
+   * Put a block to the underlying datastore
+   */
+  async put (cid: CID, block: Uint8Array, options: AbortOptions & ProgressOptions<PutBlockProgressEvents> = {}): Promise<CID> {
+    options?.signal?.throwIfAborted()
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      return await this.child.put(cid, block, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Put a multiple blocks to the underlying datastore
+   */
+  async * putMany (blocks: Iterable<InputPair> | AsyncIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
+    options?.signal?.throwIfAborted()
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      yield * this.child.putMany(blocks, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Get a block by cid
+   */
+  async * get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): AsyncGenerator<Uint8Array> {
+    options?.signal?.throwIfAborted()
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      yield * this.child.get(cid, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Get multiple blocks back from an (async) iterable of cids
+   */
+  async * getMany (cids: Iterable<CID> | AsyncIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
+    options?.signal?.throwIfAborted()
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      yield * this.child.getMany(cids, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Delete a block from the blockstore
+   */
+  async delete (cid: CID, options: AbortOptions & ProgressOptions<DeleteBlockProgressEvents> = {}): Promise<void> {
+    options?.signal?.throwIfAborted()
+    const releaseLock = await this.lock.writeLock()
+
+    try {
+      if (await this.pins.isPinned(cid)) {
+        throw new BlockPinnedError('Block was pinned - please unpin and try again')
+      }
+
+      // stop re-providing this CID if necessary
+      await this.routing.cancelReprovide(cid, options)
+
+      await this.child.delete(cid, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Delete multiple blocks from the blockstore
+   */
+  async * deleteMany (cids: Iterable<CID> | AsyncIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
+    options?.signal?.throwIfAborted()
+    const releaseLock = await this.lock.writeLock()
+
+    try {
+      const storage = this
+
+      yield * this.child.deleteMany((async function * (): AsyncGenerator<CID> {
+        for await (const cid of cids) {
+          if (await storage.pins.isPinned(cid)) {
+            throw new BlockPinnedError('Block was pinned - please unpin and try again')
+          }
+
+          // stop re-providing this CID if necessary
+          await storage.routing.cancelReprovide(cid, options)
+
+          yield cid
+        }
+      }()), options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  async has (cid: CID, options: AbortOptions = {}): Promise<boolean> {
+    options?.signal?.throwIfAborted()
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      return await this.child.has(cid, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
+    options?.signal?.throwIfAborted()
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      yield * this.child.getAll(options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  createSession (root: CID, options?: AbortOptions): SessionBlockstore {
+    options?.signal?.throwIfAborted()
+    return this.child.createSession(root, options)
+  }
+}

package/src/datastore-version.ts

@@ -0,0 +1,25 @@
+import { InvalidDatastoreVersionError } from '@helia/utils'
+import { Key } from 'interface-datastore'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import type { Datastore } from 'interface-datastore'
+
+const DS_VERSION_KEY = new Key('/version')
+const CURRENT_VERSION = 1
+
+export async function assertDatastoreVersionIsCurrent (datastore: Datastore): Promise<void> {
+  if (!(await datastore.has(DS_VERSION_KEY))) {
+    await datastore.put(DS_VERSION_KEY, uint8ArrayFromString(`${CURRENT_VERSION}`))
+
+    return
+  }
+
+  const buf = await datastore.get(DS_VERSION_KEY)
+  const str = uint8ArrayToString(buf)
+  const version = parseInt(str, 10)
+
+  if (version !== CURRENT_VERSION) {
+    // TODO: write migrations when we break compatibility - for an example, see https://github.com/ipfs/js-ipfs-repo/tree/master/packages/ipfs-repo-migrations
+    throw new InvalidDatastoreVersionError('Invalid datastore version, a datastore migration may be required')
+  }
+}

package/src/get-codec.ts

@@ -0,0 +1,39 @@
+import { UnknownCodecError } from '@helia/interface'
+import { isPromise } from '@helia/utils'
+import * as dagPb from '@ipld/dag-pb'
+import * as raw from 'multiformats/codecs/raw'
+import type { CodecLoader } from '@helia/interface'
+import type { BlockCodec } from 'multiformats/codecs/interface'
+
+export function getCodec (initialCodecs: Array<BlockCodec<any, any>> = [], loadCodec?: CodecLoader): CodecLoader {
+  const codecs: Record<number, BlockCodec<any, any>> = {
+    [dagPb.code]: dagPb,
+    [raw.code]: raw
+  }
+
+  initialCodecs.forEach(codec => {
+    codecs[codec.code] = codec
+  })
+
+  return async (code) => {
+    let codec = codecs[code]
+
+    if (codec == null && loadCodec != null) {
+      const res = loadCodec(code)
+
+      if (isPromise(res)) {
+        codec = await res
+      } else {
+        codec = res
+      }
+
+      codecs[codec.code] = codec
+    }
+
+    if (codec != null) {
+      return codec
+    }
+
+    throw new UnknownCodecError(`Could not load codec for ${code}`)
+  }
+}