@crawlee/core 4.0.0-beta.6 → 4.0.0-beta.61

package/http_clients/base-http-client.js

@@ -1,33 +0,0 @@
-import { applySearchParams } from '@crawlee/utils';
-/**
- * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
- */
-export function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }) {
-    const url = new URL(request.url);
-    const headers = { ...request.headers };
-    applySearchParams(url, searchParams);
-    if ([request.body, form, json].filter((value) => value !== undefined).length > 1) {
-        throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');
-    }
-    const body = (() => {
-        if (form !== undefined) {
-            return new URLSearchParams(form).toString();
-        }
-        if (json !== undefined) {
-            return JSON.stringify(json);
-        }
-        return request.body;
-    })();
-    if (form !== undefined) {
-        headers['content-type'] ??= 'application/x-www-form-urlencoded';
-    }
-    if (json !== undefined) {
-        headers['content-type'] ??= 'application/json';
-    }
-    if (username !== undefined || password !== undefined) {
-        const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64');
-        headers.authorization = `Basic ${encodedAuth}`;
-    }
-    return { ...request, body, url, headers };
-}
-//# sourceMappingURL=base-http-client.js.map

package/http_clients/base-http-client.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"base-http-client.js","sourceRoot":"","sources":["../../src/http_clients/base-http-client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAqB,MAAM,gBAAgB,CAAC;AA8LtE;;GAEG;AACH,MAAM,UAAU,yBAAyB,CAAqD,EAC1F,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACsB;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAEvC,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAErC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;IAC1G,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE;QACf,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QAChD,CAAC;QAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,OAAO,CAAC,IAAI,CAAC;IACxB,CAAC,CAAC,EAAE,CAAC;IAEL,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACrB,OAAO,CAAC,cAAc,CAAC,KAAK,mCAAmC,CAAC;IACpE,CAAC;IAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACrB,OAAO,CAAC,cAAc,CAAC,KAAK,kBAAkB,CAAC;IACnD,CAAC;IAED,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,IAAI,EAAE,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC1F,OAAO,CAAC,aAAa,GAAG,SAAS,WAAW,EAAE,CAAC;IACnD,CAAC;IAED,OAAO,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AAC9C,CAAC"}

package/http_clients/form-data-like.d.ts

@@ -1,67 +0,0 @@
-/**
- * This is copied from https://github.com/octet-stream/form-data-encoder
- */
-interface FileLike {
-    /**
-     * Name of the file referenced by the File object.
-     */
-    readonly name: string;
-    /**
-     * Returns the media type ([`MIME`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types)) of the file represented by a `File` object.
-     */
-    readonly type: string;
-    /**
-     * Size of the file parts in bytes
-     */
-    readonly size: number;
-    /**
-     * The last modified date of the file as the number of milliseconds since the Unix epoch (January 1, 1970 at midnight). Files without a known last modified date return the current date.
-     */
-    readonly lastModified: number;
-    /**
-     * Returns a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream) which upon reading returns the data contained within the [`File`](https://developer.mozilla.org/en-US/docs/Web/API/File).
-     */
-    stream(): ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>;
-    readonly [Symbol.toStringTag]?: string;
-}
-/**
- * A `string` or `File` that represents a single value from a set of `FormData` key-value pairs.
- */
-type FormDataEntryValue = string | FileLike;
-/**
- * This interface reflects minimal shape of the FormData
- */
-export interface FormDataLike {
-    /**
-     * Appends a new value onto an existing key inside a FormData object,
-     * or adds the key if it does not already exist.
-     *
-     * The difference between `set()` and `append()` is that if the specified key already exists, `set()` will overwrite all existing values with the new one, whereas `append()` will append the new value onto the end of the existing set of values.
-     *
-     * @param name The name of the field whose data is contained in `value`.
-     * @param value The field's value. This can be [`Blob`](https://developer.mozilla.org/en-US/docs/Web/API/Blob)
-      or [`File`](https://developer.mozilla.org/en-US/docs/Web/API/File). If none of these are specified the value is converted to a string.
-     * @param fileName The filename reported to the server, when a Blob or File is passed as the second parameter. The default filename for Blob objects is "blob". The default filename for File objects is the file's filename.
-     */
-    append(name: string, value: unknown, fileName?: string): void;
-    /**
-     * Returns all the values associated with a given key from within a `FormData` object.
-     *
-     * @param {string} name A name of the value you want to retrieve.
-     *
-     * @returns An array of `FormDataEntryValue` whose key matches the value passed in the `name` parameter. If the key doesn't exist, the method returns an empty list.
-     */
-    getAll(name: string): FormDataEntryValue[];
-    /**
-     * Returns an [`iterator`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols) allowing to go through the `FormData` key/value pairs.
-     * The key of each pair is a string; the value is a [`FormDataValue`](https://developer.mozilla.org/en-US/docs/Web/API/FormDataEntryValue).
-     */
-    entries(): IterableIterator<[string, FormDataEntryValue]>;
-    /**
-     * An alias for FormDataLike#entries()
-     */
-    [Symbol.iterator](): IterableIterator<[string, FormDataEntryValue]>;
-    readonly [Symbol.toStringTag]?: string;
-}
-export {};
-//# sourceMappingURL=form-data-like.d.ts.map

package/http_clients/form-data-like.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"form-data-like.d.ts","sourceRoot":"","sources":["../../src/http_clients/form-data-like.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,UAAU,QAAQ;IACd;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;OAEG;IACH,MAAM,IAAI,cAAc,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;IACjE,QAAQ,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC;CAC1C;AAED;;GAEG;AACH,KAAK,kBAAkB,GAAG,MAAM,GAAG,QAAQ,CAAC;AAC5C;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB;;;;;;;;;;OAUG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9D;;;;;;OAMG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,kBAAkB,EAAE,CAAC;IAC3C;;;OAGG;IACH,OAAO,IAAI,gBAAgB,CAAC,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC,CAAC;IAC1D;;OAEG;IACH,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC,CAAC;IACpE,QAAQ,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC;CAC1C"}

package/http_clients/form-data-like.js

@@ -1,5 +0,0 @@
-/**
- * This is copied from https://github.com/octet-stream/form-data-encoder
- */
-export {};
-//# sourceMappingURL=form-data-like.js.map

package/http_clients/form-data-like.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"form-data-like.js","sourceRoot":"","sources":["../../src/http_clients/form-data-like.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/http_clients/got-scraping-http-client.d.ts

@@ -1,15 +0,0 @@
-import type { BaseHttpClient, HttpRequest, HttpResponse, RedirectHandler, ResponseTypes, StreamingHttpResponse } from './base-http-client.js';
-/**
- * A HTTP client implementation based on the `got-scraping` library.
- */
-export declare class GotScrapingHttpClient implements BaseHttpClient {
-    /**
-     * @inheritDoc
-     */
-    sendRequest<TResponseType extends keyof ResponseTypes>(request: HttpRequest<TResponseType>): Promise<HttpResponse<TResponseType>>;
-    /**
-     * @inheritDoc
-     */
-    stream(request: HttpRequest, handleRedirect?: RedirectHandler): Promise<StreamingHttpResponse>;
-}
-//# sourceMappingURL=got-scraping-http-client.d.ts.map

package/http_clients/got-scraping-http-client.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"got-scraping-http-client.d.ts","sourceRoot":"","sources":["../../src/http_clients/got-scraping-http-client.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACR,cAAc,EACd,WAAW,EACX,YAAY,EACZ,eAAe,EACf,aAAa,EACb,qBAAqB,EACxB,MAAM,uBAAuB,CAAC;AAE/B;;GAEG;AACH,qBAAa,qBAAsB,YAAW,cAAc;IACxD;;OAEG;IACG,WAAW,CAAC,aAAa,SAAS,MAAM,aAAa,EACvD,OAAO,EAAE,WAAW,CAAC,aAAa,CAAC,GACpC,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;IAgBvC;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,WAAW,EAAE,cAAc,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,qBAAqB,CAAC;CAmDvG"}

package/http_clients/got-scraping-http-client.js

@@ -1,69 +0,0 @@
-import { gotScraping } from 'got-scraping';
-/**
- * A HTTP client implementation based on the `got-scraping` library.
- */
-export class GotScrapingHttpClient {
-    /**
-     * @inheritDoc
-     */
-    async sendRequest(request) {
-        const gotResult = await gotScraping({
-            ...request,
-            retry: {
-                limit: 0,
-                ...request.retry,
-            },
-        });
-        return {
-            ...gotResult,
-            body: gotResult.body,
-            request: { url: request.url, ...gotResult.request },
-        };
-    }
-    /**
-     * @inheritDoc
-     */
-    async stream(request, handleRedirect) {
-        // eslint-disable-next-line no-async-promise-executor
-        return new Promise(async (resolve, reject) => {
-            const stream = gotScraping({ ...request, isStream: true });
-            stream.on('redirect', (updatedOptions, redirectResponse) => {
-                handleRedirect?.(redirectResponse, updatedOptions);
-            });
-            // We need to end the stream for DELETE requests, otherwise it will hang.
-            if (request.method && ['DELETE', 'delete'].includes(request.method)) {
-                stream.end();
-            }
-            stream.on('error', reject);
-            stream.on('response', (response) => {
-                const result = {
-                    stream,
-                    request,
-                    redirectUrls: response.redirectUrls,
-                    url: response.url,
-                    ip: response.ip,
-                    statusCode: response.statusCode,
-                    headers: response.headers,
-                    trailers: response.trailers,
-                    complete: response.complete,
-                    get downloadProgress() {
-                        return stream.downloadProgress;
-                    },
-                    get uploadProgress() {
-                        return stream.uploadProgress;
-                    },
-                };
-                Object.assign(result, response); // TODO BC - remove in 4.0
-                resolve(result);
-                stream.on('end', () => {
-                    result.complete = response.complete;
-                    result.trailers ??= {};
-                    Object.assign(result.trailers, response.trailers);
-                    result.rawTrailers ??= []; // TODO BC - remove in 4.0
-                    Object.assign(result.rawTrailers, response.rawTrailers);
-                });
-            });
-        });
-    }
-}
-//# sourceMappingURL=got-scraping-http-client.js.map

package/http_clients/got-scraping-http-client.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"got-scraping-http-client.js","sourceRoot":"","sources":["../../src/http_clients/got-scraping-http-client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAW3C;;GAEG;AACH,MAAM,OAAO,qBAAqB;IAC9B;;OAEG;IACH,KAAK,CAAC,WAAW,CACb,OAAmC;QAEnC,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC;YAChC,GAAG,OAAO;YACV,KAAK,EAAE;gBACH,KAAK,EAAE,CAAC;gBACR,GAAI,OAAO,CAAC,KAA6C;aAC5D;SACJ,CAAC,CAAC;QAEH,OAAO;YACH,GAAG,SAAS;YACZ,IAAI,EAAE,SAAS,CAAC,IAAoC;YACpD,OAAO,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE;SACtD,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,OAAoB,EAAE,cAAgC;QAC/D,qDAAqD;QACrD,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;YACzC,MAAM,MAAM,GAAG,WAAW,CAAC,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YAE3D,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,cAAuB,EAAE,gBAA+B,EAAE,EAAE;gBAC/E,cAAc,EAAE,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAC;YACvD,CAAC,CAAC,CAAC;YAEH,yEAAyE;YACzE,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClE,MAAM,CAAC,GAAG,EAAE,CAAC;YACjB,CAAC;YAED,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAE3B,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,QAAuB,EAAE,EAAE;gBAC9C,MAAM,MAAM,GAA0B;oBAClC,MAAM;oBACN,OAAO;oBACP,YAAY,EAAE,QAAQ,CAAC,YAAY;oBACnC,GAAG,EAAE,QAAQ,CAAC,GAAG;oBACjB,EAAE,EAAE,QAAQ,CAAC,EAAE;oBACf,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,IAAI,gBAAgB;wBAChB,OAAO,MAAM,CAAC,gBAAgB,CAAC;oBACnC,CAAC;oBACD,IAAI,cAAc;wBACd,OAAO,MAAM,CAAC,cAAc,CAAC;oBACjC,CAAC;iBACJ,CAAC;gBAEF,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,0BAA0B;gBAE3D,OAAO,CAAC,MAAM,CAAC,CAAC;gBAEhB,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBAClB,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;oBAEpC,MAAM,CAAC,QAAQ,KAAK,EAAE,CAAC;oBACvB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;oBAEjD,MAAc,CAAC,WAAW,KAAK,EAAE,CAAC,CAAC,0BAA0B;oBAC9D,MAAM,CAAC,MAAM,CAAE,MAAc,CAAC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;gBACrE,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;CACJ"}

package/http_clients/index.d.ts

@@ -1,3 +0,0 @@
-export * from './base-http-client.js';
-export * from './got-scraping-http-client.js';
-//# sourceMappingURL=index.d.ts.map

package/http_clients/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/http_clients/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC"}

package/http_clients/index.js

@@ -1,3 +0,0 @@
-export * from './base-http-client.js';
-export * from './got-scraping-http-client.js';
-//# sourceMappingURL=index.js.map

package/http_clients/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/http_clients/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC"}

package/session_pool/events.d.ts

@@ -1,3 +0,0 @@
-/** @internal */
-export declare const EVENT_SESSION_RETIRED = "sessionRetired";
-//# sourceMappingURL=events.d.ts.map

package/session_pool/events.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/session_pool/events.ts"],"names":[],"mappings":"AAAA,gBAAgB;AAChB,eAAO,MAAM,qBAAqB,mBAAmB,CAAC"}

package/session_pool/events.js

@@ -1,3 +0,0 @@
-/** @internal */
-export const EVENT_SESSION_RETIRED = 'sessionRetired';
-//# sourceMappingURL=events.js.map

package/session_pool/events.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"events.js","sourceRoot":"","sources":["../../src/session_pool/events.ts"],"names":[],"mappings":"AAAA,gBAAgB;AAChB,MAAM,CAAC,MAAM,qBAAqB,GAAG,gBAAgB,CAAC"}

package/storages/storage_manager.d.ts

@@ -1,58 +0,0 @@
-import type { StorageClient } from '@crawlee/types';
-import { Configuration } from '../configuration.js';
-import type { ProxyConfiguration } from '../proxy_configuration.js';
-import type { Constructor } from '../typedefs.js';
-export interface IStorage {
-    id: string;
-    name?: string;
-}
-/**
- * StorageManager takes care of opening remote or local storages.
- * @ignore
- */
-export declare class StorageManager<T extends IStorage = IStorage> {
-    private readonly config;
-    private readonly name;
-    private readonly StorageConstructor;
-    private readonly cache;
-    private readonly storageOpenQueue;
-    constructor(StorageConstructor: Constructor<T>, config?: Configuration);
-    static openStorage<T extends IStorage>(storageClass: Constructor<T>, idOrName?: string, client?: StorageClient, config?: Configuration): Promise<T>;
-    static getManager<T extends IStorage>(storageClass: Constructor<T>, config?: Configuration): StorageManager<T>;
-    /** @internal */
-    static clearCache(config?: Configuration): void;
-    openStorage(idOrName?: string | null, client?: StorageClient): Promise<T>;
-    closeStorage(storage: {
-        id: string;
-        name?: string;
-    }): void;
-    /**
-     * Helper function that first requests storage by ID and if storage doesn't exist then gets it by name.
-     */
-// @ts-ignore optional peer dependency or compatibility with es2022
-    protected _getOrCreateStorage(storageIdOrName: string, storageConstructorName: string, apiClient: StorageClient): Promise<import("@crawlee/types").DatasetCollectionData>;
-    protected _getStorageClientFactories(client: StorageClient, storageConstructorName: string): {
-// @ts-ignore optional peer dependency or compatibility with es2022
-        createStorageClient: ((id: string) => import("@crawlee/types").DatasetClient) | ((id: string) => import("@crawlee/types").KeyValueStoreClient) | ((id: string, options?: import("@crawlee/types").RequestQueueOptions) => import("@crawlee/types").RequestQueueClient);
-// @ts-ignore optional peer dependency or compatibility with es2022
-        createStorageCollectionClient: (() => import("@crawlee/types").DatasetCollectionClient) | (() => import("@crawlee/types").KeyValueStoreCollectionClient) | (() => import("@crawlee/types").RequestQueueCollectionClient);
-    };
-    protected _addStorageToCache(storage: T): void;
-}
-export interface StorageManagerOptions {
-    /**
-     * SDK configuration instance, defaults to the static register.
-     */
-    config?: Configuration;
-    /**
-     * Optional storage client that should be used to open storages.
-     */
-    storageClient?: StorageClient;
-    /**
-     * Used to pass the proxy configuration for the `requestsFromUrl` objects.
-     * Takes advantage of the internal address rotation and authentication process.
-     * If undefined, the `requestsFromUrl` requests will be made without proxy.
-     */
-    proxyConfiguration?: ProxyConfiguration;
-}
-//# sourceMappingURL=storage_manager.d.ts.map

package/storages/storage_manager.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"storage_manager.d.ts","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGhE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAQlD,MAAM,WAAW,QAAQ;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,qBAAa,cAAc,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IAQjD,OAAO,CAAC,QAAQ,CAAC,MAAM;IAP3B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA+C;IACpE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAoC;IACvE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAwB;IAC9C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoB;gBAGjD,kBAAkB,EAAE,WAAW,CAAC,CAAC,CAAC,EACjB,MAAM,gBAAkC;WAMhD,WAAW,CAAC,CAAC,SAAS,QAAQ,EACvC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,EAC5B,QAAQ,CAAC,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,aAAa,EACtB,MAAM,gBAAkC,GACzC,OAAO,CAAC,CAAC,CAAC;IAIb,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,QAAQ,EAChC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,EAC5B,MAAM,gBAAkC,GACzC,cAAc,CAAC,CAAC,CAAC;IASpB,gBAAgB;IAChB,MAAM,CAAC,UAAU,CAAC,MAAM,gBAAkC,GAAG,IAAI;IAW3D,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC;IA6B/E,YAAY,CAAC,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAU1D;;OAEG;cACa,mBAAmB,CAC/B,eAAe,EAAE,MAAM,EACvB,sBAAsB,EAAE,MAAM,EAC9B,SAAS,EAAE,aAAa;IAe5B,SAAS,CAAC,0BAA0B,CAAC,MAAM,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM;;;;IAY1F,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI;CASjD;AAKD,MAAM,WAAW,qBAAqB;IAClC;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C"}

package/storages/storage_manager.js

@@ -1,105 +0,0 @@
-import { AsyncQueue } from '@sapphire/async-queue';
-import { Configuration } from '../configuration.js';
-const DEFAULT_ID_CONFIG_KEYS = {
-    Dataset: 'defaultDatasetId',
-    KeyValueStore: 'defaultKeyValueStoreId',
-    RequestQueue: 'defaultRequestQueueId',
-};
-/**
- * StorageManager takes care of opening remote or local storages.
- * @ignore
- */
-export class StorageManager {
-    config;
-    name;
-    StorageConstructor;
-    cache = new Map();
-    storageOpenQueue = new AsyncQueue();
-    constructor(StorageConstructor, config = Configuration.getGlobalConfig()) {
-        this.config = config;
-        this.StorageConstructor = StorageConstructor;
-        this.name = this.StorageConstructor.name;
-    }
-    static async openStorage(storageClass, idOrName, client, config = Configuration.getGlobalConfig()) {
-        return this.getManager(storageClass, config).openStorage(idOrName, client);
-    }
-    static getManager(storageClass, config = Configuration.getGlobalConfig()) {
-        if (!config.storageManagers.has(storageClass)) {
-            const manager = new StorageManager(storageClass, config);
-            config.storageManagers.set(storageClass, manager);
-        }
-        return config.storageManagers.get(storageClass);
-    }
-    /** @internal */
-    static clearCache(config = Configuration.getGlobalConfig()) {
-        config.storageManagers.forEach((manager) => {
-            if (manager.name === 'KeyValueStore') {
-                manager.cache.forEach((item) => {
-                    item.clearCache?.();
-                });
-            }
-        });
-        config.storageManagers.clear();
-    }
-    async openStorage(idOrName, client) {
-        await this.storageOpenQueue.wait();
-        if (!idOrName) {
-            const defaultIdConfigKey = DEFAULT_ID_CONFIG_KEYS[this.name];
-            idOrName = this.config.get(defaultIdConfigKey);
-        }
-        const cacheKey = idOrName;
-        let storage = this.cache.get(cacheKey);
-        if (!storage) {
-            client ??= this.config.getStorageClient();
-            const storageObject = await this._getOrCreateStorage(idOrName, this.name, client);
-            storage = new this.StorageConstructor({
-                id: storageObject.id,
-                name: storageObject.name,
-                storageObject,
-                client,
-            });
-            this._addStorageToCache(storage);
-        }
-        this.storageOpenQueue.shift();
-        return storage;
-    }
-    closeStorage(storage) {
-        const idKey = storage.id;
-        this.cache.delete(idKey);
-        if (storage.name) {
-            const nameKey = storage.name;
-            this.cache.delete(nameKey);
-        }
-    }
-    /**
-     * Helper function that first requests storage by ID and if storage doesn't exist then gets it by name.
-     */
-    async _getOrCreateStorage(storageIdOrName, storageConstructorName, apiClient) {
-        const { createStorageClient, createStorageCollectionClient } = this._getStorageClientFactories(apiClient, storageConstructorName);
-        const storageClient = createStorageClient(storageIdOrName);
-        const existingStorage = await storageClient.get();
-        if (existingStorage)
-            return existingStorage;
-        const storageCollectionClient = createStorageCollectionClient();
-        return storageCollectionClient.getOrCreate(storageIdOrName);
-    }
-    _getStorageClientFactories(client, storageConstructorName) {
-        // Dataset => dataset
-        const clientName = (storageConstructorName[0].toLowerCase() + storageConstructorName.slice(1));
-        // dataset => datasets
-        const collectionClientName = `${clientName}s`;
-        return {
-            createStorageClient: client[clientName].bind(client),
-            createStorageCollectionClient: client[collectionClientName].bind(client),
-        };
-    }
-    _addStorageToCache(storage) {
-        const idKey = storage.id;
-        this.cache.set(idKey, storage);
-        if (storage.name) {
-            const nameKey = storage.name;
-            this.cache.set(nameKey, storage);
-        }
-    }
-}
-//# sourceMappingURL=storage_manager.js.map

package/storages/storage_manager.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"storage_manager.js","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIpD,MAAM,sBAAsB,GAAG;IAC3B,OAAO,EAAE,kBAAkB;IAC3B,aAAa,EAAE,wBAAwB;IACvC,YAAY,EAAE,uBAAuB;CAC/B,CAAC;AAOX;;;GAGG;AACH,MAAM,OAAO,cAAc;IAQF;IAPJ,IAAI,CAA+C;IACnD,kBAAkB,CAAoC;IACtD,KAAK,GAAG,IAAI,GAAG,EAAa,CAAC;IAC7B,gBAAgB,GAAG,IAAI,UAAU,EAAE,CAAC;IAErD,YACI,kBAAkC,EACjB,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEzD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAoD,CAAC;IAC7F,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,WAAW,CACpB,YAA4B,EAC5B,QAAiB,EACjB,MAAsB,EACtB,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE;QAExC,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,CAAC,UAAU,CACb,YAA4B,EAC5B,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE;QAExC,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YACzD,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACtD,CAAC;QAED,OAAO,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAsB,CAAC;IACzE,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE;QACtD,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YACvC,IAAI,OAAO,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;gBACnC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC1B,IAAmB,CAAC,UAAU,EAAE,EAAE,CAAC;gBACxC,CAAC,CAAC,CAAC;YACP,CAAC;QACL,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAwB,EAAE,MAAsB;QAC9D,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAEnC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,MAAM,kBAAkB,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7D,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAW,CAAC;QAC7D,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC;QAC1B,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEvC,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC1C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAClF,OAAO,GAAG,IAAI,IAAI,CAAC,kBAAkB,CAAC;gBAClC,EAAE,EAAE,aAAa,CAAC,EAAE;gBACpB,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,aAAa;gBACb,MAAM;aACT,CAAC,CAAC;YAEH,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAE9B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,YAAY,CAAC,OAAsC;QAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEzB,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAC/B,eAAuB,EACvB,sBAA8B,EAC9B,SAAwB;QAExB,MAAM,EAAE,mBAAmB,EAAE,6BAA6B,EAAE,GAAG,IAAI,CAAC,0BAA0B,CAC1F,SAAS,EACT,sBAAsB,CACzB,CAAC;QAEF,MAAM,aAAa,GAAG,mBAAmB,CAAC,eAAe,CAAC,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,CAAC;QAClD,IAAI,eAAe;YAAE,OAAO,eAAe,CAAC;QAE5C,MAAM,uBAAuB,GAAG,6BAA6B,EAAE,CAAC;QAChE,OAAO,uBAAuB,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;IAChE,CAAC;IAES,0BAA0B,CAAC,MAAqB,EAAE,sBAA8B;QACtF,qBAAqB;QACrB,MAAM,UAAU,GAAG,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAgB,CAAC;QAC9G,sBAAsB;QACtB,MAAM,oBAAoB,GAAG,GAAG,UAAU,GAA4B,CAAC;QAEvE,OAAO;YACH,mBAAmB,EAAE,MAAM,CAAC,UAAW,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;YACrD,6BAA6B,EAAE,MAAM,CAAC,oBAAqB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SAC5E,CAAC;IACN,CAAC;IAES,kBAAkB,CAAC,OAAU;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAE/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACrC,CAAC;IACL,CAAC;CACJ"}