@helia/utils 2.5.2 → 3.0.0

package/dist/src/utils/storage.d.ts

@@ -1,56 +0,0 @@
-import type { HasherLoader } from '@helia/interface';
-import type { BlockBroker, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetrievalOptions } from '@helia/interface/blocks';
-import type { AbortOptions, ComponentLogger, Logger } from '@libp2p/interface';
-import type { Blockstore, InputPair } from 'interface-blockstore';
-import type { AwaitIterable } from 'interface-store';
-import type { CID } from 'multiformats/cid';
-import type { MultihashHasher } from 'multiformats/hashes/interface';
-import type { ProgressEvent, ProgressOptions } from 'progress-events';
-export interface StorageComponents<Broker extends BlockBroker<ProgressEvent<any, any>, ProgressEvent<any, any>>> {
-    blockstore: Blockstore;
-    logger: ComponentLogger;
-    blockBrokers: Broker[];
-    getHasher: HasherLoader;
-}
-export interface StorageInit {
-    maxIdentityHashDigestLength?: number;
-}
-export declare class Storage<Broker extends BlockBroker<ProgressEvent<any, any>, ProgressEvent<any, any>>> implements Blockstore {
-    protected readonly child: Blockstore;
-    protected readonly getHasher: HasherLoader;
-    protected log: Logger;
-    protected readonly logger: ComponentLogger;
-    protected readonly blockBrokers: Broker[];
-    /**
-     * Create a new BlockStorage
-     */
-    constructor(components: StorageComponents<Broker>, init?: StorageInit);
-    /**
-     * Put a block to the underlying datastore
-     */
-    put(cid: CID, block: Uint8Array, options?: AbortOptions & ProgressOptions<PutBlockProgressEvents>): Promise<CID>;
-    /**
-     * Put a multiple blocks to the underlying datastore
-     */
-    putMany(blocks: AwaitIterable<InputPair>, options?: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents>): AsyncGenerator<CID>;
-    /**
-     * Get a block by cid
-     */
-    get(cid: CID, options?: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents>): AsyncGenerator<Uint8Array>;
-    /**
-     * Get multiple blocks back from an (async) iterable of cids
-     */
-    getMany(cids: AwaitIterable<CID>, options?: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents>): AsyncGenerator<Pair>;
-    /**
-     * Delete a block from the blockstore
-     */
-    delete(cid: CID, options?: AbortOptions & ProgressOptions<DeleteBlockProgressEvents>): Promise<void>;
-    /**
-     * Delete multiple blocks from the blockstore
-     */
-    deleteMany(cids: AwaitIterable<CID>, options?: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents>): AsyncGenerator<CID>;
-    has(cid: CID, options?: AbortOptions): Promise<boolean>;
-    getAll(options?: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents>): AsyncGenerator<Pair>;
-}
-export declare const getCidBlockVerifierFunction: (cid: CID, hasher: MultihashHasher) => Required<BlockRetrievalOptions>["validateFn"];
-//# sourceMappingURL=storage.d.ts.map

package/dist/src/utils/storage.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../../../src/utils/storage.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,8BAA8B,EAAE,yBAAyB,EAAE,sBAAsB,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAA;AAC3S,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,EAAiB,MAAM,mBAAmB,CAAA;AAC7F,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AACjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,KAAK,EAAmB,eAAe,EAAE,MAAM,+BAA+B,CAAA;AACrF,OAAO,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAErE,MAAM,WAAW,iBAAiB,CAAC,MAAM,SAAS,WAAW,CAAC,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC7G,UAAU,EAAE,UAAU,CAAA;IACtB,MAAM,EAAE,eAAe,CAAA;IACvB,YAAY,EAAE,MAAM,EAAE,CAAA;IACtB,SAAS,EAAE,YAAY,CAAA;CACxB;AAED,MAAM,WAAW,WAAW;IAC1B,2BAA2B,CAAC,EAAE,MAAM,CAAA;CACrC;AAID,qBAAa,OAAO,CAAE,MAAM,SAAS,WAAW,CAAC,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAE,YAAW,UAAU;IACvH,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAA;IACpC,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAA;IAC1C,SAAS,CAAC,GAAG,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAA;IAC1C,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,CAAA;IAEzC;;OAEG;gBACU,UAAU,EAAE,iBAAiB,CAAC,MAAM,CAAC,EAAE,IAAI,GAAE,WAAgB;IAU1E;;OAEG;IACG,GAAG,CAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,GAAE,YAAY,GAAG,eAAe,CAAC,sBAAsB,CAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAiB3H;;OAEG;IACK,OAAO,CAAE,MAAM,EAAE,aAAa,CAAC,SAAS,CAAC,EAAE,OAAO,GAAE,YAAY,GAAG,eAAe,CAAC,2BAA2B,CAAM,GAAG,cAAc,CAAC,GAAG,CAAC;IAsBlJ;;OAEG;IACK,GAAG,CAAE,GAAG,EAAE,GAAG,EAAE,OAAO,GAAE,iBAAiB,GAAG,YAAY,GAAG,eAAe,CAAC,sBAAsB,CAAM,GAAG,cAAc,CAAC,UAAU,CAAC;IAqC5I;;OAEG;IACK,OAAO,CAAE,IAAI,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,OAAO,GAAE,iBAAiB,GAAG,YAAY,GAAG,eAAe,CAAC,2BAA2B,CAAM,GAAG,cAAc,CAAC,IAAI,CAAC;IAiC/J;;OAEG;IACG,MAAM,CAAE,GAAG,EAAE,GAAG,EAAE,OAAO,GAAE,YAAY,GAAG,eAAe,CAAC,yBAAyB,CAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAM/G;;OAEG;IACK,UAAU,CAAE,IAAI,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,OAAO,GAAE,YAAY,GAAG,eAAe,CAAC,8BAA8B,CAAM,GAAG,cAAc,CAAC,GAAG,CAAC;IAS1I,GAAG,CAAE,GAAG,EAAE,GAAG,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,OAAO,CAAC;IAI1D,MAAM,CAAE,OAAO,GAAE,YAAY,GAAG,eAAe,CAAC,0BAA0B,CAAM,GAAG,cAAc,CAAC,IAAI,CAAC;CAIhH;AAqED,eAAO,MAAM,2BAA2B,GAAI,KAAK,GAAG,EAAE,QAAQ,eAAe,KAAG,QAAQ,CAAC,qBAAqB,CAAC,CAAC,YAAY,CAwB3H,CAAA"}

package/dist/src/utils/storage.js

@@ -1,225 +0,0 @@
-import { InvalidMultihashError, InvalidParametersError, setMaxListeners } from '@libp2p/interface';
-import { anySignal } from 'any-signal';
-import { IdentityBlockstore } from 'blockstore-core/identity';
-import filter from 'it-filter';
-import forEach from 'it-foreach';
-import { CustomProgressEvent } from 'progress-events';
-import { equals as uint8ArrayEquals } from 'uint8arrays/equals';
-import { BlockNotFoundWhileOfflineError, InvalidConfigurationError, LoadBlockFailedError } from "../errors.js";
-import { isPromise } from "./is-promise.js";
-const DEFAULT_MAX_IDENTITY_HASH_DIGEST_LENGTH = 128;
-export class Storage {
-    child;
-    getHasher;
-    log;
-    logger;
-    blockBrokers;
-    /**
-     * Create a new BlockStorage
-     */
-    constructor(components, init = {}) {
-        this.log = components.logger.forComponent('helia:networked-storage');
-        this.logger = components.logger;
-        this.blockBrokers = components.blockBrokers;
-        this.child = new IdentityBlockstore(components.blockstore, {
-            maxDigestLength: init.maxIdentityHashDigestLength ?? DEFAULT_MAX_IDENTITY_HASH_DIGEST_LENGTH
-        });
-        this.getHasher = components.getHasher;
-    }
-    /**
-     * Put a block to the underlying datastore
-     */
-    async put(cid, block, options = {}) {
-        if (await this.child.has(cid, options)) {
-            options.onProgress?.(new CustomProgressEvent('blocks:put:duplicate', cid));
-            return cid;
-        }
-        options.onProgress?.(new CustomProgressEvent('blocks:put:providers:notify', cid));
-        await Promise.all(this.blockBrokers.map(async (broker) => broker.announce?.(cid, options)));
-        options.onProgress?.(new CustomProgressEvent('blocks:put:blockstore:put', cid));
-        return this.child.put(cid, block, options);
-    }
-    /**
-     * Put a multiple blocks to the underlying datastore
-     */
-    async *putMany(blocks, options = {}) {
-        const missingBlocks = filter(blocks, async ({ cid }) => {
-            const has = await this.child.has(cid, options);
-            if (has) {
-                options.onProgress?.(new CustomProgressEvent('blocks:put-many:duplicate', cid));
-            }
-            return !has;
-        });
-        const notifyEach = forEach(missingBlocks, async ({ cid }) => {
-            options.onProgress?.(new CustomProgressEvent('blocks:put-many:providers:notify', cid));
-            await Promise.all(this.blockBrokers.map(async (broker) => broker.announce?.(cid, options)));
-        });
-        options.onProgress?.(new CustomProgressEvent('blocks:put-many:blockstore:put-many'));
-        yield* this.child.putMany(notifyEach, options);
-    }
-    /**
-     * Get a block by cid
-     */
-    async *get(cid, options = {}) {
-        const has = await this.child.has(cid, options);
-        const offline = options.offline === true;
-        if (!has) {
-            if (offline) {
-                throw new BlockNotFoundWhileOfflineError('The block was present in the blockstore and the node is running offline so cannot fetch it');
-            }
-            const hasher = await this.getHasher(cid.multihash.code);
-            options?.signal?.throwIfAborted();
-            // we do not have the block locally, get it from a block provider
-            options.onProgress?.(new CustomProgressEvent('blocks:get:providers:get', cid));
-            const block = await raceBlockRetrievers(cid, this.blockBrokers, hasher, {
-                ...options,
-                log: this.log
-            });
-            options.onProgress?.(new CustomProgressEvent('blocks:get:blockstore:put', cid));
-            await this.child.put(cid, block, options);
-            // notify other block providers of the new block
-            options.onProgress?.(new CustomProgressEvent('blocks:get:providers:notify', cid));
-            await Promise.all(this.blockBrokers.map(async (broker) => broker.announce?.(cid, options)));
-            yield block;
-            return;
-        }
-        options.onProgress?.(new CustomProgressEvent('blocks:get:blockstore:get', cid));
-        yield* this.child.get(cid, options);
-    }
-    /**
-     * Get multiple blocks back from an (async) iterable of cids
-     */
-    async *getMany(cids, options = {}) {
-        options.onProgress?.(new CustomProgressEvent('blocks:get-many:blockstore:get-many'));
-        yield* this.child.getMany(forEach(cids, async (cid) => {
-            const has = await this.child.has(cid, options);
-            const offline = options.offline === true;
-            if (!has) {
-                if (offline) {
-                    throw new BlockNotFoundWhileOfflineError('The block was present in the blockstore and the node is running offline so cannot fetch it');
-                }
-                const hasher = await this.getHasher(cid.multihash.code);
-                options?.signal?.throwIfAborted();
-                // we do not have the block locally, get it from a block provider
-                options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:get', cid));
-                const block = await raceBlockRetrievers(cid, this.blockBrokers, hasher, {
-                    ...options,
-                    log: this.log
-                });
-                options.onProgress?.(new CustomProgressEvent('blocks:get-many:blockstore:put', cid));
-                await this.child.put(cid, block, options);
-                // notify other block providers of the new block
-                options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:notify', cid));
-                await Promise.all(this.blockBrokers.map(async (broker) => broker.announce?.(cid, options)));
-            }
-        }));
-    }
-    /**
-     * Delete a block from the blockstore
-     */
-    async delete(cid, options = {}) {
-        options.onProgress?.(new CustomProgressEvent('blocks:delete:blockstore:delete', cid));
-        await this.child.delete(cid, options);
-    }
-    /**
-     * Delete multiple blocks from the blockstore
-     */
-    async *deleteMany(cids, options = {}) {
-        options.onProgress?.(new CustomProgressEvent('blocks:delete-many:blockstore:delete-many'));
-        yield* this.child.deleteMany((async function* () {
-            for await (const cid of cids) {
-                yield cid;
-            }
-        }()), options);
-    }
-    async has(cid, options = {}) {
-        return this.child.has(cid, options);
-    }
-    async *getAll(options = {}) {
-        options.onProgress?.(new CustomProgressEvent('blocks:get-all:blockstore:get-many'));
-        yield* this.child.getAll(options);
-    }
-}
-/**
- * Race block providers cancelling any pending requests once the block has been
- * found.
- */
-async function raceBlockRetrievers(cid, blockBrokers, hasher, options) {
-    const validateFn = getCidBlockVerifierFunction(cid, hasher);
-    const controller = new AbortController();
-    const signal = anySignal([controller.signal, options.signal]);
-    setMaxListeners(Infinity, controller.signal, signal);
-    const retrievers = [];
-    for (const broker of blockBrokers) {
-        if (isRetrievingBlockBroker(broker)) {
-            retrievers.push(broker);
-        }
-    }
-    if (retrievers.length === 0) {
-        throw new InvalidConfigurationError(`No block brokers capable of retrieving blocks are configured, the CID ${cid} cannot be fetched from the network`);
-    }
-    try {
-        return await Promise.any(retrievers
-            .map(async (retriever) => {
-            try {
-                let blocksWereValidated = false;
-                const block = await retriever.retrieve(cid, {
-                    ...options,
-                    signal,
-                    validateFn: async (block) => {
-                        await validateFn(block);
-                        options.signal?.throwIfAborted();
-                        blocksWereValidated = true;
-                    }
-                });
-                if (!blocksWereValidated) {
-                    // the blockBroker either did not throw an error when attempting to validate the block
-                    // or did not call the validateFn at all. We should validate the block ourselves
-                    await validateFn(block);
-                    options.signal?.throwIfAborted();
-                }
-                return block;
-            }
-            catch (err) {
-                options.log.error('could not retrieve verified block for %c from %s - %e', cid, retriever.name, err);
-                throw err;
-            }
-        }));
-    }
-    catch (err) {
-        throw new LoadBlockFailedError(err.errors, `Failed to load block for ${cid}`);
-    }
-    finally {
-        // we have the block from the fastest block retriever, abort any still
-        // in-flight retrieve attempts
-        controller.abort();
-        signal.clear();
-    }
-}
-function isRetrievingBlockBroker(broker) {
-    return typeof broker.retrieve === 'function';
-}
-export const getCidBlockVerifierFunction = (cid, hasher) => {
-    if (hasher == null) {
-        throw new InvalidParametersError(`No hasher configured for multihash code 0x${cid.multihash.code.toString(16)}, please configure one. You can look up which hash this is at https://github.com/multiformats/multicodec/blob/master/table.csv`);
-    }
-    return async (block) => {
-        // verify block
-        let hash;
-        const res = hasher.digest(block, {
-            // support truncated hashes where they are truncated
-            truncate: cid.multihash.digest.byteLength
-        });
-        if (isPromise(res)) {
-            hash = await res;
-        }
-        else {
-            hash = res;
-        }
-        if (!uint8ArrayEquals(hash.digest, cid.multihash.digest)) {
-            // if a hash mismatch occurs for a TrustlessGatewayBlockBroker, we should try another gateway
-            throw new InvalidMultihashError('Hash of downloaded block did not match multihash from passed CID');
-        }
-    };
-};
-//# sourceMappingURL=storage.js.map

package/dist/src/utils/storage.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"storage.js","sourceRoot":"","sources":["../../../src/utils/storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAClG,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACtC,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAC7D,OAAO,MAAM,MAAM,WAAW,CAAA;AAC9B,OAAO,OAAO,MAAM,YAAY,CAAA;AAChC,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,EAAE,MAAM,IAAI,gBAAgB,EAAE,MAAM,oBAAoB,CAAA;AAC/D,OAAO,EAAE,8BAA8B,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAA;AAC9G,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAqB3C,MAAM,uCAAuC,GAAG,GAAG,CAAA;AAEnD,MAAM,OAAO,OAAO;IACC,KAAK,CAAY;IACjB,SAAS,CAAc;IAChC,GAAG,CAAQ;IACF,MAAM,CAAiB;IACvB,YAAY,CAAU;IAEzC;;OAEG;IACH,YAAa,UAAqC,EAAE,OAAoB,EAAE;QACxE,IAAI,CAAC,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,yBAAyB,CAAC,CAAA;QACpE,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAA;QAC/B,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,YAAY,CAAA;QAC3C,IAAI,CAAC,KAAK,GAAG,IAAI,kBAAkB,CAAC,UAAU,CAAC,UAAU,EAAE;YACzD,eAAe,EAAE,IAAI,CAAC,2BAA2B,IAAI,uCAAuC;SAC7F,CAAC,CAAA;QACF,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC,SAAS,CAAA;IACvC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAE,GAAQ,EAAE,KAAiB,EAAE,UAAkE,EAAE;QAC1G,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;YACvC,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,sBAAsB,EAAE,GAAG,CAAC,CAAC,CAAA;YAC/E,OAAO,GAAG,CAAA;QACZ,CAAC;QAED,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,6BAA6B,EAAE,GAAG,CAAC,CAAC,CAAA;QAEtF,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CACvE,CAAA;QAED,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,2BAA2B,EAAE,GAAG,CAAC,CAAC,CAAA;QAEpF,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;IAC5C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAE,OAAO,CAAE,MAAgC,EAAE,UAAuE,EAAE;QAC1H,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,EAAoB,EAAE;YACvE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YAE9C,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,2BAA2B,EAAE,GAAG,CAAC,CAAC,CAAA;YACtF,CAAC;YAED,OAAO,CAAC,GAAG,CAAA;QACb,CAAC,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,EAAiB,EAAE;YACzE,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,kCAAkC,EAAE,GAAG,CAAC,CAAC,CAAA;YAC3F,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CACvE,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAC,qCAAqC,CAAC,CAAC,CAAA;QACpF,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;IACjD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAE,GAAG,CAAE,GAAQ,EAAE,UAAsF,EAAE;QAC7G,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,KAAK,IAAI,CAAA;QAExC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,IAAI,8BAA8B,CAAC,4FAA4F,CAAC,CAAA;YACxI,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;YACvD,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;YAEjC,iEAAiE;YAEjE,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,0BAA0B,EAAE,GAAG,CAAC,CAAC,CAAA;YACnF,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE;gBACtE,GAAG,OAAO;gBACV,GAAG,EAAE,IAAI,CAAC,GAAG;aACd,CAAC,CAAA;YACF,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,2BAA2B,EAAE,GAAG,CAAC,CAAC,CAAA;YACpF,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;YAEzC,gDAAgD;YAChD,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,6BAA6B,EAAE,GAAG,CAAC,CAAC,CAAA;YACtF,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CACvE,CAAA;YAED,MAAM,KAAK,CAAA;YACX,OAAM;QACR,CAAC;QAED,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,2BAA2B,EAAE,GAAG,CAAC,CAAC,CAAA;QAEpF,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAE,OAAO,CAAE,IAAwB,EAAE,UAA2F,EAAE;QACtI,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAC,qCAAqC,CAAC,CAAC,CAAA;QAEpF,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAiB,EAAE;YACpE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,KAAK,IAAI,CAAA;YAExC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,IAAI,8BAA8B,CAAC,4FAA4F,CAAC,CAAA;gBACxI,CAAC;gBAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;gBACvD,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;gBAEjC,iEAAiE;gBACjE,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,+BAA+B,EAAE,GAAG,CAAC,CAAC,CAAA;gBACxF,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE;oBACtE,GAAG,OAAO;oBACV,GAAG,EAAE,IAAI,CAAC,GAAG;iBACd,CAAC,CAAA;gBACF,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAA;gBACzF,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;gBAEzC,gDAAgD;gBAChD,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,kCAAkC,EAAE,GAAG,CAAC,CAAC,CAAA;gBAC3F,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CACvE,CAAA;YACH,CAAC;QACH,CAAC,CAAC,CAAC,CAAA;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAE,GAAQ,EAAE,UAAqE,EAAE;QAC7F,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAM,iCAAiC,EAAE,GAAG,CAAC,CAAC,CAAA;QAE1F,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACvC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAE,UAAU,CAAE,IAAwB,EAAE,UAA0E,EAAE;QACxH,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAC,2CAA2C,CAAC,CAAC,CAAA;QAC1F,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,SAAU,CAAC;YAC7C,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBAC7B,MAAM,GAAG,CAAA;YACX,CAAC;QACH,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAA;IAChB,CAAC;IAED,KAAK,CAAC,GAAG,CAAE,GAAQ,EAAE,UAAwB,EAAE;QAC7C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,CAAE,MAAM,CAAE,UAAsE,EAAE;QACtF,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAC,oCAAoC,CAAC,CAAC,CAAA;QACnF,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACpC,CAAC;CACF;AAED;;;GAGG;AACH,KAAK,UAAU,mBAAmB,CAAE,GAAQ,EAAE,YAA2B,EAAE,MAAuB,EAAE,OAAqC;IACvI,MAAM,UAAU,GAAG,2BAA2B,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IAE3D,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAA;IAC7D,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAEpD,MAAM,UAAU,GAA4D,EAAE,CAAA;IAE9E,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;QAClC,IAAI,uBAAuB,CAAC,MAAM,CAAC,EAAE,CAAC;YACpC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACzB,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,yBAAyB,CAAC,yEAAyE,GAAG,qCAAqC,CAAC,CAAA;IACxJ,CAAC;IAED,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,GAAG,CACtB,UAAU;aACP,GAAG,CAAC,KAAK,EAAC,SAAS,EAAC,EAAE;YACrB,IAAI,CAAC;gBACH,IAAI,mBAAmB,GAAG,KAAK,CAAA;gBAC/B,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE;oBAC1C,GAAG,OAAO;oBACV,MAAM;oBACN,UAAU,EAAE,KAAK,EAAE,KAAiB,EAAiB,EAAE;wBACrD,MAAM,UAAU,CAAC,KAAK,CAAC,CAAA;wBACvB,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;wBAChC,mBAAmB,GAAG,IAAI,CAAA;oBAC5B,CAAC;iBACF,CAAC,CAAA;gBAEF,IAAI,CAAC,mBAAmB,EAAE,CAAC;oBACzB,sFAAsF;oBACtF,gFAAgF;oBAChF,MAAM,UAAU,CAAC,KAAK,CAAC,CAAA;oBACvB,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;gBAClC,CAAC;gBAED,OAAO,KAAK,CAAA;YACd,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,uDAAuD,EAAE,GAAG,EAAE,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;gBACpG,MAAM,GAAG,CAAA;YACX,CAAC;QACH,CAAC,CAAC,CACL,CAAA;IACH,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,MAAM,IAAI,oBAAoB,CAAC,GAAG,CAAC,MAAM,EAAE,4BAA4B,GAAG,EAAE,CAAC,CAAA;IAC/E,CAAC;YAAS,CAAC;QACT,sEAAsE;QACtE,8BAA8B;QAC9B,UAAU,CAAC,KAAK,EAAE,CAAA;QAClB,MAAM,CAAC,KAAK,EAAE,CAAA;IAChB,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAE,MAAmB;IACnD,OAAO,OAAO,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAA;AAC9C,CAAC;AAED,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,GAAQ,EAAE,MAAuB,EAAiD,EAAE;IAC9H,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,sBAAsB,CAAC,6CAA6C,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,gIAAgI,CAAC,CAAA;IAChP,CAAC;IAED,OAAO,KAAK,EAAE,KAAiB,EAAiB,EAAE;QAChD,eAAe;QACf,IAAI,IAA6B,CAAA;QACjC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE;YAC/B,oDAAoD;YACpD,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU;SAC1C,CAAC,CAAA;QAEF,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,GAAG,MAAM,GAAG,CAAA;QAClB,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,GAAG,CAAA;QACZ,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YACzD,6FAA6F;YAC7F,MAAM,IAAI,qBAAqB,CAAC,kEAAkE,CAAC,CAAA;QACrG,CAAC;IACH,CAAC,CAAA;AACH,CAAC,CAAA"}

package/src/pins.ts

@@ -1,247 +0,0 @@
-import { InvalidParametersError } from '@libp2p/interface'
-import { Queue } from '@libp2p/utils'
-import * as cborg from 'cborg'
-import { Key } from 'interface-datastore'
-import toBuffer from 'it-to-buffer'
-import { base36 } from 'multiformats/bases/base36'
-import { createUnsafe } from 'multiformats/block'
-import { CID } from 'multiformats/cid'
-import { CustomProgressEvent } from 'progress-events'
-import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
-import { AlreadyPinnedError } from './errors.ts'
-import type { CodecLoader } from '@helia/interface'
-import type { GetBlockProgressEvents } from '@helia/interface/blocks'
-import type { AddOptions, AddPinEvents, IsPinnedOptions, LsOptions, Pin, Pins, RmOptions } from '@helia/interface/pins'
-import type { AbortOptions } from '@libp2p/interface'
-import type { Blockstore } from 'interface-blockstore'
-import type { Datastore } from 'interface-datastore'
-import type { Version } from 'multiformats/cid'
-import type { ProgressOptions } from 'progress-events'
-
-interface DatastorePin {
-  /**
-   * 0 for a direct pin or an arbitrary (+ve, whole) number or Infinity
-   */
-  depth: number
-
-  /**
-   * User-specific metadata for the pin
-   */
-  metadata: Record<string, string | number | boolean>
-}
-
-interface DatastorePinnedBlock {
-  pinCount: number
-  pinnedBy: Uint8Array[]
-}
-
-/**
- * Callback for updating a {@link DatastorePinnedBlock}'s properties when
- * calling `#updatePinnedBlock`
- *
- * The callback should return `false` to prevent any pinning modifications to
- * the block, and true in all other cases.
- */
-interface WithPinnedBlockCallback {
-  (pinnedBlock: DatastorePinnedBlock): boolean
-}
-
-const DATASTORE_PIN_PREFIX = '/pin/'
-const DATASTORE_BLOCK_PREFIX = '/pinned-block/'
-const DATASTORE_ENCODING = base36
-const DAG_WALK_QUEUE_CONCURRENCY = 1
-
-interface WalkDagOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents | AddPinEvents> {
-  depth: number
-}
-
-function toDSKey (cid: CID): Key {
-  if (cid.version === 0) {
-    cid = cid.toV1()
-  }
-
-  return new Key(`${DATASTORE_PIN_PREFIX}${cid.toString(DATASTORE_ENCODING)}`)
-}
-
-export class PinsImpl implements Pins {
-  private readonly datastore: Datastore
-  private readonly blockstore: Blockstore
-  private readonly getCodec: CodecLoader
-
-  constructor (datastore: Datastore, blockstore: Blockstore, getCodec: CodecLoader) {
-    this.datastore = datastore
-    this.blockstore = blockstore
-    this.getCodec = getCodec
-  }
-
-  async * add (cid: CID<unknown, number, number, Version>, options: AddOptions = {}): AsyncGenerator<CID, void, undefined> {
-    const pinKey = toDSKey(cid)
-
-    if (await this.datastore.has(pinKey)) {
-      throw new AlreadyPinnedError('Already pinned')
-    }
-
-    const depth = Math.round(options.depth ?? Infinity)
-
-    if (depth < 0) {
-      throw new InvalidParametersError('Depth must be greater than or equal to 0')
-    }
-
-    // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs
-    const queue = new Queue<AsyncGenerator<CID>>({
-      concurrency: DAG_WALK_QUEUE_CONCURRENCY
-    })
-
-    for await (const childCid of this.#walkDag(cid, queue, {
-      ...options,
-      depth
-    })) {
-      await this.#updatePinnedBlock(childCid, (pinnedBlock: DatastorePinnedBlock) => {
-        // do not update pinned block if this block is already pinned by this CID
-        if (pinnedBlock.pinnedBy.find(c => uint8ArrayEquals(c, cid.bytes)) != null) {
-          return false
-        }
-
-        pinnedBlock.pinCount++
-        pinnedBlock.pinnedBy.push(cid.bytes)
-        return true
-      }, options)
-
-      yield childCid
-    }
-
-    const pin: DatastorePin = {
-      depth,
-      metadata: options.metadata ?? {}
-    }
-
-    await this.datastore.put(pinKey, cborg.encode(pin), options)
-  }
-
-  /**
-   * Walk a DAG in an iterable fashion
-   */
-  async * #walkDag (cid: CID, queue: Queue<AsyncGenerator<CID>>, options: WalkDagOptions): AsyncGenerator<CID> {
-    if (options.depth === -1) {
-      return
-    }
-
-    const codec = await this.getCodec(cid.code)
-    const bytes = await toBuffer(this.blockstore.get(cid, options))
-    const block = createUnsafe({ bytes, cid, codec })
-
-    yield cid
-
-    // walk dag, ensure all blocks are present
-    for (const [,cid] of block.links()) {
-      yield * await queue.add(async () => {
-        return this.#walkDag(cid, queue, {
-          ...options,
-          depth: options.depth - 1
-        })
-      })
-    }
-  }
-
-  /**
-   * Update the pin count for the CID
-   */
-  async #updatePinnedBlock (cid: CID, withPinnedBlock: WithPinnedBlockCallback, options: AddOptions): Promise<void> {
-    const blockKey = new Key(`${DATASTORE_BLOCK_PREFIX}${DATASTORE_ENCODING.encode(cid.multihash.bytes)}`)
-
-    let pinnedBlock: DatastorePinnedBlock = {
-      pinCount: 0,
-      pinnedBy: []
-    }
-
-    try {
-      pinnedBlock = cborg.decode(await this.datastore.get(blockKey, options))
-    } catch (err: any) {
-      if (err.name !== 'NotFoundError') {
-        throw err
-      }
-    }
-
-    const shouldContinue = withPinnedBlock(pinnedBlock)
-
-    if (!shouldContinue) {
-      return
-    }
-
-    if (pinnedBlock.pinCount === 0) {
-      if (await this.datastore.has(blockKey)) {
-        await this.datastore.delete(blockKey)
-        return
-      }
-    }
-
-    await this.datastore.put(blockKey, cborg.encode(pinnedBlock), options)
-    options.onProgress?.(new CustomProgressEvent<CID>('helia:pin:add', cid))
-  }
-
-  async * rm (cid: CID<unknown, number, number, Version>, options: RmOptions = {}): AsyncGenerator<CID, void, undefined> {
-    const pinKey = toDSKey(cid)
-    const buf = await this.datastore.get(pinKey, options)
-    const pin = cborg.decode(buf)
-
-    await this.datastore.delete(pinKey, options)
-
-    // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs
-    const queue = new Queue<AsyncGenerator<CID>>({
-      concurrency: DAG_WALK_QUEUE_CONCURRENCY
-    })
-
-    for await (const childCid of this.#walkDag(cid, queue, {
-      ...options,
-      depth: pin.depth
-    })) {
-      await this.#updatePinnedBlock(childCid, (pinnedBlock): boolean => {
-        pinnedBlock.pinCount--
-        pinnedBlock.pinnedBy = pinnedBlock.pinnedBy.filter(c => uint8ArrayEquals(c, cid.bytes))
-        return true
-      }, {
-        ...options,
-        depth: pin.depth
-      })
-
-      yield childCid
-    }
-  }
-
-  async * ls (options: LsOptions = {}): AsyncGenerator<Pin, void, undefined> {
-    for await (const { key, value } of this.datastore.query({
-      prefix: DATASTORE_PIN_PREFIX + (options.cid != null ? `${options.cid.toString(base36)}` : '')
-    }, options)) {
-      const cid = CID.parse(key.toString().substring(5), base36)
-      const pin = cborg.decode(value)
-
-      yield {
-        cid,
-        ...pin
-      }
-    }
-  }
-
-  async isPinned (cid: CID, options: IsPinnedOptions = {}): Promise<boolean> {
-    const blockKey = new Key(`${DATASTORE_BLOCK_PREFIX}${DATASTORE_ENCODING.encode(cid.multihash.bytes)}`)
-
-    return this.datastore.has(blockKey, options)
-  }
-
-  async get (cid: CID, options?: AbortOptions): Promise<Pin> {
-    const pinKey = toDSKey(cid)
-    const buf = await this.datastore.get(pinKey, options)
-
-    return cborg.decode(buf)
-  }
-
-  async setMetadata (cid: CID, metadata: Record<string, string | number | boolean> | undefined, options?: AbortOptions): Promise<void> {
-    const pinKey = toDSKey(cid)
-    const buf = await this.datastore.get(pinKey, options)
-    const pin: DatastorePin = cborg.decode(buf)
-
-    pin.metadata = metadata ?? {}
-
-    await this.datastore.put(pinKey, cborg.encode(pin), options)
-  }
-}