@helia/utils 2.3.5-7bf85e0c → 2.4.0

package/dist/src/utils/storage.d.ts

@@ -0,0 +1,56 @@
+import type { HasherLoader } from '@helia/interface';
+import type { BlockBroker, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetrievalOptions } from '@helia/interface/blocks';
+import type { AbortOptions, ComponentLogger, Logger } from '@libp2p/interface';
+import type { Blockstore, InputPair } from 'interface-blockstore';
+import type { AwaitIterable } from 'interface-store';
+import type { CID } from 'multiformats/cid';
+import type { MultihashHasher } from 'multiformats/hashes/interface';
+import type { ProgressEvent, ProgressOptions } from 'progress-events';
+export interface StorageComponents<Broker extends BlockBroker<ProgressEvent<any, any>, ProgressEvent<any, any>>> {
+    blockstore: Blockstore;
+    logger: ComponentLogger;
+    blockBrokers: Broker[];
+    getHasher: HasherLoader;
+}
+export interface StorageInit {
+    maxIdentityHashDigestLength?: number;
+}
+export declare class Storage<Broker extends BlockBroker<ProgressEvent<any, any>, ProgressEvent<any, any>>> implements Blockstore {
+    protected readonly child: Blockstore;
+    protected readonly getHasher: HasherLoader;
+    protected log: Logger;
+    protected readonly logger: ComponentLogger;
+    protected readonly blockBrokers: Broker[];
+    /**
+     * Create a new BlockStorage
+     */
+    constructor(components: StorageComponents<Broker>, init?: StorageInit);
+    /**
+     * Put a block to the underlying datastore
+     */
+    put(cid: CID, block: Uint8Array, options?: AbortOptions & ProgressOptions<PutBlockProgressEvents>): Promise<CID>;
+    /**
+     * Put a multiple blocks to the underlying datastore
+     */
+    putMany(blocks: AwaitIterable<InputPair>, options?: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents>): AsyncGenerator<CID>;
+    /**
+     * Get a block by cid
+     */
+    get(cid: CID, options?: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents>): AsyncGenerator<Uint8Array>;
+    /**
+     * Get multiple blocks back from an (async) iterable of cids
+     */
+    getMany(cids: AwaitIterable<CID>, options?: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents>): AsyncGenerator<Pair>;
+    /**
+     * Delete a block from the blockstore
+     */
+    delete(cid: CID, options?: AbortOptions & ProgressOptions<DeleteBlockProgressEvents>): Promise<void>;
+    /**
+     * Delete multiple blocks from the blockstore
+     */
+    deleteMany(cids: AwaitIterable<CID>, options?: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents>): AsyncGenerator<CID>;
+    has(cid: CID, options?: AbortOptions): Promise<boolean>;
+    getAll(options?: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents>): AsyncGenerator<Pair>;
+}
+export declare const getCidBlockVerifierFunction: (cid: CID, hasher: MultihashHasher) => Required<BlockRetrievalOptions>["validateFn"];
+//# sourceMappingURL=storage.d.ts.map

package/dist/src/utils/storage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../../../src/utils/storage.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,8BAA8B,EAAE,yBAAyB,EAAE,sBAAsB,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAA;AAC3S,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,EAAiB,MAAM,mBAAmB,CAAA;AAC7F,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AACjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,KAAK,EAAmB,eAAe,EAAE,MAAM,+BAA+B,CAAA;AACrF,OAAO,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAErE,MAAM,WAAW,iBAAiB,CAAC,MAAM,SAAS,WAAW,CAAC,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC7G,UAAU,EAAE,UAAU,CAAA;IACtB,MAAM,EAAE,eAAe,CAAA;IACvB,YAAY,EAAE,MAAM,EAAE,CAAA;IACtB,SAAS,EAAE,YAAY,CAAA;CACxB;AAED,MAAM,WAAW,WAAW;IAC1B,2BAA2B,CAAC,EAAE,MAAM,CAAA;CACrC;AAID,qBAAa,OAAO,CAAE,MAAM,SAAS,WAAW,CAAC,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAE,YAAW,UAAU;IACvH,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAA;IACpC,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAA;IAC1C,SAAS,CAAC,GAAG,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAA;IAC1C,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,CAAA;IAEzC;;OAEG;gBACU,UAAU,EAAE,iBAAiB,CAAC,MAAM,CAAC,EAAE,IAAI,GAAE,WAAgB;IAU1E;;OAEG;IACG,GAAG,CAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,GAAE,YAAY,GAAG,eAAe,CAAC,sBAAsB,CAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAiB3H;;OAEG;IACK,OAAO,CAAE,MAAM,EAAE,aAAa,CAAC,SAAS,CAAC,EAAE,OAAO,GAAE,YAAY,GAAG,eAAe,CAAC,2BAA2B,CAAM,GAAG,cAAc,CAAC,GAAG,CAAC;IAsBlJ;;OAEG;IACK,GAAG,CAAE,GAAG,EAAE,GAAG,EAAE,OAAO,GAAE,iBAAiB,GAAG,YAAY,GAAG,eAAe,CAAC,sBAAsB,CAAM,GAAG,cAAc,CAAC,UAAU,CAAC;IAqC5I;;OAEG;IACK,OAAO,CAAE,IAAI,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,OAAO,GAAE,iBAAiB,GAAG,YAAY,GAAG,eAAe,CAAC,2BAA2B,CAAM,GAAG,cAAc,CAAC,IAAI,CAAC;IAiC/J;;OAEG;IACG,MAAM,CAAE,GAAG,EAAE,GAAG,EAAE,OAAO,GAAE,YAAY,GAAG,eAAe,CAAC,yBAAyB,CAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAM/G;;OAEG;IACK,UAAU,CAAE,IAAI,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,OAAO,GAAE,YAAY,GAAG,eAAe,CAAC,8BAA8B,CAAM,GAAG,cAAc,CAAC,GAAG,CAAC;IAS1I,GAAG,CAAE,GAAG,EAAE,GAAG,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,OAAO,CAAC;IAI1D,MAAM,CAAE,OAAO,GAAE,YAAY,GAAG,eAAe,CAAC,0BAA0B,CAAM,GAAG,cAAc,CAAC,IAAI,CAAC;CAIhH;AAqED,eAAO,MAAM,2BAA2B,GAAI,KAAK,GAAG,EAAE,QAAQ,eAAe,KAAG,QAAQ,CAAC,qBAAqB,CAAC,CAAC,YAAY,CAwB3H,CAAA"}

package/dist/src/utils/storage.js

@@ -0,0 +1,225 @@
+import { InvalidMultihashError, InvalidParametersError, setMaxListeners } from '@libp2p/interface';
+import { anySignal } from 'any-signal';
+import { IdentityBlockstore } from 'blockstore-core/identity';
+import filter from 'it-filter';
+import forEach from 'it-foreach';
+import { CustomProgressEvent } from 'progress-events';
+import { equals as uint8ArrayEquals } from 'uint8arrays/equals';
+import { BlockNotFoundWhileOfflineError, InvalidConfigurationError, LoadBlockFailedError } from "../errors.js";
+import { isPromise } from './is-promise.js';
+const DEFAULT_MAX_IDENTITY_HASH_DIGEST_LENGTH = 128;
+export class Storage {
+    child;
+    getHasher;
+    log;
+    logger;
+    blockBrokers;
+    /**
+     * Create a new BlockStorage
+     */
+    constructor(components, init = {}) {
+        this.log = components.logger.forComponent('helia:networked-storage');
+        this.logger = components.logger;
+        this.blockBrokers = components.blockBrokers;
+        this.child = new IdentityBlockstore(components.blockstore, {
+            maxDigestLength: init.maxIdentityHashDigestLength ?? DEFAULT_MAX_IDENTITY_HASH_DIGEST_LENGTH
+        });
+        this.getHasher = components.getHasher;
+    }
+    /**
+     * Put a block to the underlying datastore
+     */
+    async put(cid, block, options = {}) {
+        if (await this.child.has(cid, options)) {
+            options.onProgress?.(new CustomProgressEvent('blocks:put:duplicate', cid));
+            return cid;
+        }
+        options.onProgress?.(new CustomProgressEvent('blocks:put:providers:notify', cid));
+        await Promise.all(this.blockBrokers.map(async (broker) => broker.announce?.(cid, options)));
+        options.onProgress?.(new CustomProgressEvent('blocks:put:blockstore:put', cid));
+        return this.child.put(cid, block, options);
+    }
+    /**
+     * Put a multiple blocks to the underlying datastore
+     */
+    async *putMany(blocks, options = {}) {
+        const missingBlocks = filter(blocks, async ({ cid }) => {
+            const has = await this.child.has(cid, options);
+            if (has) {
+                options.onProgress?.(new CustomProgressEvent('blocks:put-many:duplicate', cid));
+            }
+            return !has;
+        });
+        const notifyEach = forEach(missingBlocks, async ({ cid }) => {
+            options.onProgress?.(new CustomProgressEvent('blocks:put-many:providers:notify', cid));
+            await Promise.all(this.blockBrokers.map(async (broker) => broker.announce?.(cid, options)));
+        });
+        options.onProgress?.(new CustomProgressEvent('blocks:put-many:blockstore:put-many'));
+        yield* this.child.putMany(notifyEach, options);
+    }
+    /**
+     * Get a block by cid
+     */
+    async *get(cid, options = {}) {
+        const has = await this.child.has(cid, options);
+        const offline = options.offline === true;
+        if (!has) {
+            if (offline) {
+                throw new BlockNotFoundWhileOfflineError('The block was present in the blockstore and the node is running offline so cannot fetch it');
+            }
+            const hasher = await this.getHasher(cid.multihash.code);
+            options?.signal?.throwIfAborted();
+            // we do not have the block locally, get it from a block provider
+            options.onProgress?.(new CustomProgressEvent('blocks:get:providers:get', cid));
+            const block = await raceBlockRetrievers(cid, this.blockBrokers, hasher, {
+                ...options,
+                log: this.log
+            });
+            options.onProgress?.(new CustomProgressEvent('blocks:get:blockstore:put', cid));
+            await this.child.put(cid, block, options);
+            // notify other block providers of the new block
+            options.onProgress?.(new CustomProgressEvent('blocks:get:providers:notify', cid));
+            await Promise.all(this.blockBrokers.map(async (broker) => broker.announce?.(cid, options)));
+            yield block;
+            return;
+        }
+        options.onProgress?.(new CustomProgressEvent('blocks:get:blockstore:get', cid));
+        yield* this.child.get(cid, options);
+    }
+    /**
+     * Get multiple blocks back from an (async) iterable of cids
+     */
+    async *getMany(cids, options = {}) {
+        options.onProgress?.(new CustomProgressEvent('blocks:get-many:blockstore:get-many'));
+        yield* this.child.getMany(forEach(cids, async (cid) => {
+            const has = await this.child.has(cid, options);
+            const offline = options.offline === true;
+            if (!has) {
+                if (offline) {
+                    throw new BlockNotFoundWhileOfflineError('The block was present in the blockstore and the node is running offline so cannot fetch it');
+                }
+                const hasher = await this.getHasher(cid.multihash.code);
+                options?.signal?.throwIfAborted();
+                // we do not have the block locally, get it from a block provider
+                options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:get', cid));
+                const block = await raceBlockRetrievers(cid, this.blockBrokers, hasher, {
+                    ...options,
+                    log: this.log
+                });
+                options.onProgress?.(new CustomProgressEvent('blocks:get-many:blockstore:put', cid));
+                await this.child.put(cid, block, options);
+                // notify other block providers of the new block
+                options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:notify', cid));
+                await Promise.all(this.blockBrokers.map(async (broker) => broker.announce?.(cid, options)));
+            }
+        }));
+    }
+    /**
+     * Delete a block from the blockstore
+     */
+    async delete(cid, options = {}) {
+        options.onProgress?.(new CustomProgressEvent('blocks:delete:blockstore:delete', cid));
+        await this.child.delete(cid, options);
+    }
+    /**
+     * Delete multiple blocks from the blockstore
+     */
+    async *deleteMany(cids, options = {}) {
+        options.onProgress?.(new CustomProgressEvent('blocks:delete-many:blockstore:delete-many'));
+        yield* this.child.deleteMany((async function* () {
+            for await (const cid of cids) {
+                yield cid;
+            }
+        }()), options);
+    }
+    async has(cid, options = {}) {
+        return this.child.has(cid, options);
+    }
+    async *getAll(options = {}) {
+        options.onProgress?.(new CustomProgressEvent('blocks:get-all:blockstore:get-many'));
+        yield* this.child.getAll(options);
+    }
+}
+/**
+ * Race block providers cancelling any pending requests once the block has been
+ * found.
+ */
+async function raceBlockRetrievers(cid, blockBrokers, hasher, options) {
+    const validateFn = getCidBlockVerifierFunction(cid, hasher);
+    const controller = new AbortController();
+    const signal = anySignal([controller.signal, options.signal]);
+    setMaxListeners(Infinity, controller.signal, signal);
+    const retrievers = [];
+    for (const broker of blockBrokers) {
+        if (isRetrievingBlockBroker(broker)) {
+            retrievers.push(broker);
+        }
+    }
+    if (retrievers.length === 0) {
+        throw new InvalidConfigurationError(`No block brokers capable of retrieving blocks are configured, the CID ${cid} cannot be fetched from the network`);
+    }
+    try {
+        return await Promise.any(retrievers
+            .map(async (retriever) => {
+            try {
+                let blocksWereValidated = false;
+                const block = await retriever.retrieve(cid, {
+                    ...options,
+                    signal,
+                    validateFn: async (block) => {
+                        await validateFn(block);
+                        options.signal?.throwIfAborted();
+                        blocksWereValidated = true;
+                    }
+                });
+                if (!blocksWereValidated) {
+                    // the blockBroker either did not throw an error when attempting to validate the block
+                    // or did not call the validateFn at all. We should validate the block ourselves
+                    await validateFn(block);
+                    options.signal?.throwIfAborted();
+                }
+                return block;
+            }
+            catch (err) {
+                options.log.error('could not retrieve verified block for %c from %s - %e', cid, retriever.name, err);
+                throw err;
+            }
+        }));
+    }
+    catch (err) {
+        throw new LoadBlockFailedError(err.errors, `Failed to load block for ${cid}`);
+    }
+    finally {
+        // we have the block from the fastest block retriever, abort any still
+        // in-flight retrieve attempts
+        controller.abort();
+        signal.clear();
+    }
+}
+function isRetrievingBlockBroker(broker) {
+    return typeof broker.retrieve === 'function';
+}
+export const getCidBlockVerifierFunction = (cid, hasher) => {
+    if (hasher == null) {
+        throw new InvalidParametersError(`No hasher configured for multihash code 0x${cid.multihash.code.toString(16)}, please configure one. You can look up which hash this is at https://github.com/multiformats/multicodec/blob/master/table.csv`);
+    }
+    return async (block) => {
+        // verify block
+        let hash;
+        const res = hasher.digest(block, {
+            // support truncated hashes where they are truncated
+            truncate: cid.multihash.digest.byteLength
+        });
+        if (isPromise(res)) {
+            hash = await res;
+        }
+        else {
+            hash = res;
+        }
+        if (!uint8ArrayEquals(hash.digest, cid.multihash.digest)) {
+            // if a hash mismatch occurs for a TrustlessGatewayBlockBroker, we should try another gateway
+            throw new InvalidMultihashError('Hash of downloaded block did not match multihash from passed CID');
+        }
+    };
+};
+//# sourceMappingURL=storage.js.map

package/dist/src/utils/storage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.js","sourceRoot":"","sources":["../../../src/utils/storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAClG,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACtC,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAC7D,OAAO,MAAM,MAAM,WAAW,CAAA;AAC9B,OAAO,OAAO,MAAM,YAAY,CAAA;AAChC,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,EAAE,MAAM,IAAI,gBAAgB,EAAE,MAAM,oBAAoB,CAAA;AAC/D,OAAO,EAAE,8BAA8B,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAA;AAC9G,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAqB3C,MAAM,uCAAuC,GAAG,GAAG,CAAA;AAEnD,MAAM,OAAO,OAAO;IACC,KAAK,CAAY;IACjB,SAAS,CAAc;IAChC,GAAG,CAAQ;IACF,MAAM,CAAiB;IACvB,YAAY,CAAU;IAEzC;;OAEG;IACH,YAAa,UAAqC,EAAE,OAAoB,EAAE;QACxE,IAAI,CAAC,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,yBAAyB,CAAC,CAAA;QACpE,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAA;QAC/B,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,YAAY,CAAA;QAC3C,IAAI,CAAC,KAAK,GAAG,IAAI,kBAAkB,CAAC,UAAU,CAAC,UAAU,EAAE;YACzD,eAAe,EAAE,IAAI,CAAC,2BAA2B,IAAI,uCAAuC;SAC7F,CAAC,CAAA;QACF,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC,SAAS,CAAA;IACvC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAE,GAAQ,EAAE,KAAiB,EAAE,UAAkE,EAAE;QAC1G,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;YACvC,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,sBAAsB,EAAE,GAAG,CAAC,CAAC,CAAA;YAC/E,OAAO,GAAG,CAAA;QACZ,CAAC;QAED,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,6BAA6B,EAAE,GAAG,CAAC,CAAC,CAAA;QAEtF,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CACvE,CAAA;QAED,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,2BAA2B,EAAE,GAAG,CAAC,CAAC,CAAA;QAEpF,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;IAC5C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAE,OAAO,CAAE,MAAgC,EAAE,UAAuE,EAAE;QAC1H,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,EAAoB,EAAE;YACvE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YAE9C,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,2BAA2B,EAAE,GAAG,CAAC,CAAC,CAAA;YACtF,CAAC;YAED,OAAO,CAAC,GAAG,CAAA;QACb,CAAC,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,EAAiB,EAAE;YACzE,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,kCAAkC,EAAE,GAAG,CAAC,CAAC,CAAA;YAC3F,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CACvE,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAC,qCAAqC,CAAC,CAAC,CAAA;QACpF,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;IACjD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAE,GAAG,CAAE,GAAQ,EAAE,UAAsF,EAAE;QAC7G,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,KAAK,IAAI,CAAA;QAExC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,IAAI,8BAA8B,CAAC,4FAA4F,CAAC,CAAA;YACxI,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;YACvD,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;YAEjC,iEAAiE;YAEjE,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,0BAA0B,EAAE,GAAG,CAAC,CAAC,CAAA;YACnF,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE;gBACtE,GAAG,OAAO;gBACV,GAAG,EAAE,IAAI,CAAC,GAAG;aACd,CAAC,CAAA;YACF,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,2BAA2B,EAAE,GAAG,CAAC,CAAC,CAAA;YACpF,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;YAEzC,gDAAgD;YAChD,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,6BAA6B,EAAE,GAAG,CAAC,CAAC,CAAA;YACtF,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CACvE,CAAA;YAED,MAAM,KAAK,CAAA;YACX,OAAM;QACR,CAAC;QAED,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,2BAA2B,EAAE,GAAG,CAAC,CAAC,CAAA;QAEpF,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAE,OAAO,CAAE,IAAwB,EAAE,UAA2F,EAAE;QACtI,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAC,qCAAqC,CAAC,CAAC,CAAA;QAEpF,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAiB,EAAE;YACpE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,KAAK,IAAI,CAAA;YAExC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,IAAI,8BAA8B,CAAC,4FAA4F,CAAC,CAAA;gBACxI,CAAC;gBAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;gBACvD,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;gBAEjC,iEAAiE;gBACjE,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,+BAA+B,EAAE,GAAG,CAAC,CAAC,CAAA;gBACxF,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE;oBACtE,GAAG,OAAO;oBACV,GAAG,EAAE,IAAI,CAAC,GAAG;iBACd,CAAC,CAAA;gBACF,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAA;gBACzF,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;gBAEzC,gDAAgD;gBAChD,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,kCAAkC,EAAE,GAAG,CAAC,CAAC,CAAA;gBAC3F,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CACvE,CAAA;YACH,CAAC;QACH,CAAC,CAAC,CAAC,CAAA;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAE,GAAQ,EAAE,UAAqE,EAAE;QAC7F,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,iCAAiC,EAAE,GAAG,CAAC,CAAC,CAAA;QAE1F,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACvC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAE,UAAU,CAAE,IAAwB,EAAE,UAA0E,EAAE;QACxH,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAC,2CAA2C,CAAC,CAAC,CAAA;QAC1F,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,SAAU,CAAC;YAC7C,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBAC7B,MAAM,GAAG,CAAA;YACX,CAAC;QACH,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAA;IAChB,CAAC;IAED,KAAK,CAAC,GAAG,CAAE,GAAQ,EAAE,UAAwB,EAAE;QAC7C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,CAAE,MAAM,CAAE,UAAsE,EAAE;QACtF,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAC,oCAAoC,CAAC,CAAC,CAAA;QACnF,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACpC,CAAC;CACF;AAED;;;GAGG;AACH,KAAK,UAAU,mBAAmB,CAAE,GAAQ,EAAE,YAA2B,EAAE,MAAuB,EAAE,OAAqC;IACvI,MAAM,UAAU,GAAG,2BAA2B,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IAE3D,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAA;IAC7D,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAEpD,MAAM,UAAU,GAA4D,EAAE,CAAA;IAE9E,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;QAClC,IAAI,uBAAuB,CAAC,MAAM,CAAC,EAAE,CAAC;YACpC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACzB,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,yBAAyB,CAAC,yEAAyE,GAAG,qCAAqC,CAAC,CAAA;IACxJ,CAAC;IAED,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,GAAG,CACtB,UAAU;aACP,GAAG,CAAC,KAAK,EAAC,SAAS,EAAC,EAAE;YACrB,IAAI,CAAC;gBACH,IAAI,mBAAmB,GAAG,KAAK,CAAA;gBAC/B,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE;oBAC1C,GAAG,OAAO;oBACV,MAAM;oBACN,UAAU,EAAE,KAAK,EAAE,KAAiB,EAAiB,EAAE;wBACrD,MAAM,UAAU,CAAC,KAAK,CAAC,CAAA;wBACvB,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;wBAChC,mBAAmB,GAAG,IAAI,CAAA;oBAC5B,CAAC;iBACF,CAAC,CAAA;gBAEF,IAAI,CAAC,mBAAmB,EAAE,CAAC;oBACzB,sFAAsF;oBACtF,gFAAgF;oBAChF,MAAM,UAAU,CAAC,KAAK,CAAC,CAAA;oBACvB,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;gBAClC,CAAC;gBAED,OAAO,KAAK,CAAA;YACd,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,uDAAuD,EAAE,GAAG,EAAE,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;gBACpG,MAAM,GAAG,CAAA;YACX,CAAC;QACH,CAAC,CAAC,CACL,CAAA;IACH,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,MAAM,IAAI,oBAAoB,CAAC,GAAG,CAAC,MAAM,EAAE,4BAA4B,GAAG,EAAE,CAAC,CAAA;IAC/E,CAAC;YAAS,CAAC;QACT,sEAAsE;QACtE,8BAA8B;QAC9B,UAAU,CAAC,KAAK,EAAE,CAAA;QAClB,MAAM,CAAC,KAAK,EAAE,CAAA;IAChB,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAE,MAAmB;IACnD,OAAO,OAAO,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAA;AAC9C,CAAC;AAED,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,GAAQ,EAAE,MAAuB,EAAiD,EAAE;IAC9H,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,sBAAsB,CAAC,6CAA6C,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,gIAAgI,CAAC,CAAA;IAChP,CAAC;IAED,OAAO,KAAK,EAAE,KAAiB,EAAiB,EAAE;QAChD,eAAe;QACf,IAAI,IAA6B,CAAA;QACjC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE;YAC/B,oDAAoD;YACpD,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU;SAC1C,CAAC,CAAA;QAEF,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,GAAG,MAAM,GAAG,CAAA;QAClB,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,GAAG,CAAA;QACZ,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YACzD,6FAA6F;YAC7F,MAAM,IAAI,qBAAqB,CAAC,kEAAkE,CAAC,CAAA;QACrG,CAAC;IACH,CAAC,CAAA;AACH,CAAC,CAAA"}

package/dist/typedoc-urls.json

@@ -0,0 +1,19 @@
+{
+  "GraphWalker": "https://ipfs.github.io/helia/interfaces/_helia_car.GraphWalker.html",
+  "breadthFirstWalker": "https://ipfs.github.io/helia/functions/_helia_car.breadthFirstWalker.html",
+  "depthFirstWalker": "https://ipfs.github.io/helia/functions/_helia_car.depthFirstWalker.html",
+  "naturalOrderWalker": "https://ipfs.github.io/helia/functions/_helia_car.naturalOrderWalker.html",
+  "AbstractSession": "https://ipfs.github.io/helia/classes/_helia_utils.AbstractSession.html",
+  "Helia": "https://ipfs.github.io/helia/classes/_helia_utils.Helia.html",
+  ".:Helia": "https://ipfs.github.io/helia/classes/_helia_utils.Helia.html",
+  "AbstractCreateSessionOptions": "https://ipfs.github.io/helia/interfaces/_helia_utils.AbstractCreateSessionOptions.html",
+  "AbstractSessionComponents": "https://ipfs.github.io/helia/interfaces/_helia_utils.AbstractSessionComponents.html",
+  "BlockStorage": "https://ipfs.github.io/helia/interfaces/_helia_utils.BlockStorage.html",
+  "BlockStorageInit": "https://ipfs.github.io/helia/interfaces/_helia_utils.BlockStorageInit.html",
+  "BlockstoreSessionEvents": "https://ipfs.github.io/helia/interfaces/_helia_utils.BlockstoreSessionEvents.html",
+  "GraphNode": "https://ipfs.github.io/helia/interfaces/_helia_utils.GraphNode.html",
+  "GraphWalkerComponents": "https://ipfs.github.io/helia/interfaces/_helia_utils.GraphWalkerComponents.html",
+  "GraphWalkerInit": "https://ipfs.github.io/helia/interfaces/_helia_utils.GraphWalkerInit.html",
+  "HeliaInit": "https://ipfs.github.io/helia/interfaces/_helia_utils.HeliaInit.html",
+  ".:HeliaInit": "https://ipfs.github.io/helia/interfaces/helia.HeliaInit.html"
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helia/utils",
-  "version": "2.3.5-7bf85e0c",
+  "version": "2.4.0",
   "description": "Shared code that implements the Helia API",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/ipfs/helia/tree/main/packages/utils#readme",
@@ -47,13 +47,12 @@
     "test:electron-main": "aegir test -t electron-main"
   },
   "dependencies": {
-    "@helia/interface": "6.0.3-7bf85e0c",
+    "@helia/interface": "^6.1.0",
     "@ipld/dag-cbor": "^9.2.5",
     "@ipld/dag-json": "^10.2.5",
     "@ipld/dag-pb": "^4.1.5",
     "@libp2p/interface": "^3.1.0",
     "@libp2p/keychain": "^6.0.5",
-    "@libp2p/logger": "^6.0.5",
     "@libp2p/utils": "^7.0.5",
     "@multiformats/dns": "^1.0.9",
     "@multiformats/multiaddr": "^13.0.1",
@@ -78,6 +77,7 @@
   },
   "devDependencies": {
     "@libp2p/crypto": "^5.1.12",
+    "@libp2p/logger": "^6.0.5",
     "@libp2p/peer-id": "^6.0.3",
     "@types/sinon": "^21.0.0",
     "aegir": "^47.0.22",

package/src/abstract-session.ts

@@ -30,9 +30,10 @@ interface Request {
 }
 
 export abstract class AbstractSession<Provider, RetrieveBlockProgressEvents extends ProgressEvent> extends TypedEventEmitter<BlockstoreSessionEvents<Provider>> implements BlockBroker<RetrieveBlockProgressEvents> {
+  public abstract name: string
   private initialPeerSearchComplete?: Promise<void>
   private readonly requests: Map<string, Request>
-  private readonly name: string
+  private readonly logName: string
   protected log: Logger
   protected logger: ComponentLogger
   private readonly minProviders: number
@@ -45,9 +46,9 @@ export abstract class AbstractSession<Provider, RetrieveBlockProgressEvents exte
     super()
 
     setMaxListeners(Infinity, this)
-    this.name = init.name
+    this.logName = init.name
     this.logger = components.logger
-    this.log = components.logger.forComponent(this.name)
+    this.log = components.logger.forComponent(this.logName)
     this.requests = new Map()
     this.minProviders = init.minProviders ?? DEFAULT_SESSION_MIN_PROVIDERS
     this.maxProviders = init.maxProviders ?? DEFAULT_SESSION_MAX_PROVIDERS
@@ -79,7 +80,7 @@ export abstract class AbstractSession<Provider, RetrieveBlockProgressEvents exte
 
       if (this.initialPeerSearchComplete == null) {
         first = true
-        this.log = this.logger.forComponent(`${this.name}:${cid}`)
+        this.log = this.logger.forComponent(`${this.logName}:${cid}`)
         this.initialPeerSearchComplete = this.findProviders(cid, this.minProviders, options)
       }
 
@@ -249,6 +250,21 @@ export abstract class AbstractSession<Provider, RetrieveBlockProgressEvents exte
     return false
   }
 
+  async addPeer (peer: PeerId | Multiaddr | Multiaddr[], options?: AbortOptions): Promise<void> {
+    const provider = await this.convertToProvider(peer, options)
+
+    if (provider == null || this.hasProvider(provider)) {
+      return
+    }
+
+    this.providers.push(provider)
+
+    // let the new peer join current queries
+    this.safeDispatchEvent('provider', {
+      detail: provider
+    })
+  }
+
   private async findProviders (cid: CID, count: number, options: AbortOptions): Promise<void> {
     const deferred: DeferredPromise<void> = pDefer()
     let found = 0

package/src/index.ts

@@ -6,7 +6,6 @@
  */
 
 import { contentRoutingSymbol, peerRoutingSymbol, start, stop, TypedEventEmitter } from '@libp2p/interface'
-import { defaultLogger } from '@libp2p/logger'
 import { dns } from '@multiformats/dns'
 import drain from 'it-drain'
 import { CustomProgressEvent } from 'progress-events'
@@ -208,7 +207,7 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
   private readonly log: Logger
 
   constructor (init: Omit<HeliaInit, 'start' | 'libp2p'> & { libp2p: T }) {
-    this.logger = init.logger ?? defaultLogger()
+    this.logger = init.logger ?? init.libp2p.logger
     this.log = this.logger.forComponent('helia')
     this.getHasher = getHasher(init.hashers, init.loadHasher)
     this.getCodec = getCodec(init.codecs, init.loadCodec)
@@ -259,16 +258,16 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
       providerLookupConcurrency: init.providerLookupConcurrency
     })
 
+    components.blockBrokers = init.blockBrokers.map((fn) => {
+      return fn(components)
+    })
+
     const networkedStorage = new NetworkedStorage(components, init)
     this.pins = new PinsImpl(init.datastore, networkedStorage, this.getCodec)
     this.blockstore = new BlockStorage(networkedStorage, this.pins, {
       holdGcLock: init.holdGcLock ?? true
     })
     this.datastore = init.datastore
-
-    components.blockBrokers = init.blockBrokers.map((fn) => {
-      return fn(components)
-    })
   }
 
   async start (): Promise<void> {