@crawlee/core 3.0.0-alpha.1 → 3.0.0-alpha.4

package/storages/request_list.d.ts

@@ -0,0 +1,461 @@
+/// <reference types="node" />
+import { Configuration } from '../configuration';
+import { Request, RequestOptions } from '../request';
+/** @internal */
+export declare const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
+/** @internal */
+export declare const REQUESTS_PERSISTENCE_KEY = "REQUEST_LIST_REQUESTS";
+export interface RequestListOptions {
+    /**
+     * An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,
+     * plain objects that define at least the `url` property, or an array of {@link Request} instances.
+     *
+     * **IMPORTANT:** The `sources` array will be consumed (left empty) after `RequestList` initializes.
+     * This is a measure to prevent memory leaks in situations when millions of sources are
+     * added.
+     *
+     * Additionally, the `requestsFromUrl` property may be used instead of `url`,
+     * which will instruct `RequestList` to download the source URLs from a given remote location.
+     * The URLs will be parsed from the received response.
+     *
+     * ```
+     * [
+     *     // A single URL
+     *     'http://example.com/a/b',
+     *
+     *     // Modify Request options
+     *     { method: PUT, 'https://example.com/put, payload: { foo: 'bar' }}
+     *
+     *     // Batch import of URLs from a file hosted on the web,
+     *     // where the URLs should be requested using the HTTP POST request
+     *     { method: 'POST', requestsFromUrl: 'http://example.com/urls.txt' },
+     *
+     *     // Batch import from remote file, using a specific regular expression to extract the URLs.
+     *     { requestsFromUrl: 'http://example.com/urls.txt', regex: /https:\/\/example.com\/.+/ },
+     *
+     *     // Get list of URLs from a Google Sheets document. Just add "/gviz/tq?tqx=out:csv" to the Google Sheet URL.
+     *     // For details, see https://help.apify.com/en/articles/2906022-scraping-a-list-of-urls-from-a-google-sheets-document
+     *     { requestsFromUrl: 'https://docs.google.com/spreadsheets/d/1GA5sSQhQjB_REes8I5IKg31S-TuRcznWOPjcpNqtxmU/gviz/tq?tqx=out:csv' }
+     * ]
+     * ```
+     */
+    sources?: Source[];
+    /**
+     * A function that will be called to get the sources for the `RequestList`, but only if `RequestList`
+     * was not able to fetch their persisted version (see {@link RequestListOptions.persistRequestsKey}).
+     * It must return an `Array` of {@link Request} or {@link RequestOptions}.
+     *
+     * This is very useful in a scenario when getting the sources is a resource intensive or time consuming
+     * task, such as fetching URLs from multiple sitemaps or parsing URLs from large datasets. Using the
+     * `sourcesFunction` in combination with `persistStateKey` and `persistRequestsKey` will allow you to
+     * fetch and parse those URLs only once, saving valuable time when your actor migrates or restarts.
+     *
+     * If both {@link RequestListOptions.sources} and {@link RequestListOptions.sourcesFunction} are provided,
+     * the sources returned by the function will be added after the `sources`.
+     *
+     * **Example:**
+     * ```javascript
+     * // Let's say we want to scrape URLs extracted from sitemaps.
+     *
+     * const sourcesFunction = async () => {
+     *     // With super large sitemaps, this operation could take very long
+     *     // and big websites typically have multiple sitemaps.
+     *     const sitemaps = await downloadHugeSitemaps();
+     *     return parseUrlsFromSitemaps(sitemaps);
+     * }
+     *
+     * // Sitemaps can change in real-time, so it's important to persist
+     * // the URLs we collected. Otherwise we might lose our scraping
+     * // state in case of an actor migration / failure / time-out.
+     * const requestList = new RequestList({
+     *     sourcesFunction,
+     *     persistStateKey: 'state-key',
+     *     persistRequestsKey: 'requests-key',
+     * })
+     *
+     * // The sourcesFunction is called now and the Requests are persisted.
+     * // If something goes wrong and we need to start again, RequestList
+     * // will load the persisted Requests from storage and will NOT
+     * // call the sourcesFunction again, saving time and resources.
+     * await requestList.initialize();
+     * ```
+     */
+    sourcesFunction?: RequestListSourcesFunction;
+    /**
+     * Identifies the key in the default key-value store under which `RequestList` periodically stores its
+     * state (i.e. which URLs were crawled and which not).
+     * If the actor is restarted, `RequestList` will read the state
+     * and continue where it left off.
+     *
+     * If `persistStateKey` is not set, `RequestList` will always start from the beginning,
+     * and all the source URLs will be crawled again.
+     */
+    persistStateKey?: string;
+    /**
+     * Identifies the key in the default key-value store under which the `RequestList` persists its
+     * Requests during the {@link RequestList.initialize} call.
+     * This is necessary if `persistStateKey` is set and the source URLs might potentially change,
+     * to ensure consistency of the source URLs and state object. However, it comes with some
+     * storage and performance overheads.
+     *
+     * If `persistRequestsKey` is not set, {@link RequestList.initialize} will always fetch the sources
+     * from their origin, check that they are consistent with the restored state (if any)
+     * and throw an error if they are not.
+     */
+    persistRequestsKey?: string;
+    /**
+     * The state object that the `RequestList` will be initialized from.
+     * It is in the form as returned by `RequestList.getState()`, such as follows:
+     *
+     * ```
+     * {
+     *     nextIndex: 5,
+     *     nextUniqueKey: 'unique-key-5'
+     *     inProgress: {
+     *       'unique-key-1': true,
+     *       'unique-key-4': true,
+     *     },
+     * }
+     * ```
+     *
+     * Note that the preferred (and simpler) way to persist the state of crawling of the `RequestList`
+     * is to use the `stateKeyPrefix` parameter instead.
+     */
+    state?: RequestListState;
+    /**
+     * By default, `RequestList` will deduplicate the provided URLs. Default deduplication is based
+     * on the `uniqueKey` property of passed source {@link Request} objects.
+     *
+     * If the property is not present, it is generated by normalizing the URL. If present, it is kept intact.
+     * In any case, only one request per `uniqueKey` is added to the `RequestList` resulting in removal
+     * of duplicate URLs / unique keys.
+     *
+     * Setting `keepDuplicateUrls` to `true` will append an additional identifier to the `uniqueKey`
+     * of each request that does not already include a `uniqueKey`. Therefore, duplicate
+     * URLs will be kept in the list. It does not protect the user from having duplicates in user set
+     * `uniqueKey`s however. It is the user's responsibility to ensure uniqueness of their unique keys
+     * if they wish to keep more than just a single copy in the `RequestList`.
+     * @default false
+     */
+    keepDuplicateUrls?: boolean;
+    /** @internal */
+    config?: Configuration;
+}
+/**
+ * Represents a static list of URLs to crawl.
+ * The URLs can be provided either in code or parsed from a text file hosted on the web.
+ * `RequestList` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
+ * and {@link PlaywrightCrawler} as a source of URLs to crawl.
+ *
+ * Each URL is represented using an instance of the {@link Request} class.
+ * The list can only contain unique URLs. More precisely, it can only contain `Request` instances
+ * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
+ * To add a single URL to the list multiple times, corresponding {@link Request} objects will need to have different
+ * `uniqueKey` properties. You can use the `keepDuplicateUrls` option to do this for you when initializing the
+ * `RequestList` from sources.
+ *
+ * Once you create an instance of `RequestList`, you need to call the {@link RequestList.initialize} function
+ * before the instance can be used. After that, no more URLs can be added to the list.
+ * Unlike {@link RequestQueue}, `RequestList` is static but it can contain even millions of URLs.
+ * > Note that `RequestList` can be used together with `RequestQueue` by the same crawler.
+ * > In such cases, each request from `RequestList` is enqueued into `RequestQueue` first and then consumed from the latter.
+ * > This is necessary to avoid the same URL being processed more than once (from the list first and then possibly from the queue).
+ * > In practical terms, such a combination can be useful when there is a large number of initial URLs,
+ * > but more URLs would be added dynamically by the crawler.
+ *
+ * `RequestList` has an internal state where it stores information about which requests were already handled,
+ * which are in progress and which were reclaimed. The state may be automatically persisted to the default
+ * {@link KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
+ * the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
+ * event that is periodically emitted by {@link events|Actor.events}.
+ *
+ * The internal state is closely tied to the provided sources (URLs). If the sources change on actor restart, the state will become corrupted and
+ * `RequestList` will raise an exception. This typically happens when the sources is a list of URLs downloaded from the web.
+ * In such case, use the `persistRequestsKey` option in conjunction with `persistStateKey`,
+ * to make the `RequestList` store the initial sources to the default key-value store and load them after restart,
+ * which will prevent any issues that a live list of URLs might cause.
+ *
+ * **Basic usage:**
+ * ```javascript
+ * // Use a helper function to simplify request list initialization.
+ * // State and sources are automatically persisted. This is a preferred usage.
+ * const requestList = await RequestList.open('my-request-list', [
+ *     'http://www.example.com/page-1',
+ *     { url: 'http://www.example.com/page-2', method: 'POST', userData: { foo: 'bar' }},
+ *     { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
+ * ]);
+ * ```
+ *
+ * **Advanced usage:**
+ * ```javascript
+ * // Use the constructor to get more control over the initialization.
+ * const requestList = new RequestList({
+ *     sources: [
+ *         // Separate requests
+ *         { url: 'http://www.example.com/page-1', method: 'GET', headers: { ... } },
+ *         { url: 'http://www.example.com/page-2', userData: { foo: 'bar' }},
+ *
+ *         // Bulk load of URLs from file `http://www.example.com/my-url-list.txt`
+ *         // Note that all URLs must start with http:// or https://
+ *         { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
+ *     ],
+ *
+ *     // Persist the state to avoid re-crawling which can lead to data duplications.
+ *     // Keep in mind that the sources have to be immutable or this will throw an error.
+ *     persistStateKey: 'my-state',
+ * });
+ *
+ * await requestList.initialize();
+ * ```
+ * @category Sources
+ */
+export declare class RequestList {
+    private log;
+    /**
+     * Array of all requests from all sources, in the order as they appeared in sources.
+     * All requests in the array have distinct uniqueKey!
+     * @internal
+     */
+    requests: Request[];
+    /** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */
+    private nextIndex;
+    /** Dictionary, key is Request.uniqueKey, value is corresponding index in the requests array. */
+    private uniqueKeyToIndex;
+    /**
+     * Set of `uniqueKey`s of requests that were returned by fetchNextRequest().
+     * @internal
+     */
+    inProgress: Set<string>;
+    /**
+     * Set of `uniqueKey`s of requests for which reclaimRequest() was called.
+     * @internal
+     */
+    reclaimed: Set<string>;
+    /**
+     * Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.
+     * @internal
+     */
+    isStatePersisted: boolean;
+    /**
+     * Starts as false because we don't know yet and sources might change in the meantime (eg. download from live list).
+     * @internal
+     */
+    areRequestsPersisted: boolean;
+    private isLoading;
+    private isInitialized;
+    private persistStateKey?;
+    private persistRequestsKey?;
+    private initialState?;
+    private store?;
+    private keepDuplicateUrls;
+    private sources;
+    private sourcesFunction?;
+    private events;
+    /**
+     * @param options All `RequestList` configuration options
+     */
+    constructor(options?: RequestListOptions);
+    /**
+     * Loads all remote sources of URLs and potentially starts periodic state persistence.
+     * This function must be called before you can start using the instance in a meaningful way.
+     */
+    initialize(): Promise<void>;
+    /**
+     * Adds previously persisted Requests, as retrieved from the key-value store.
+     * This needs to be done in a memory efficient way. We should update the input
+     * to a Stream once apify-client supports streams.
+     */
+    protected _addPersistedRequests(persistedRequests: Buffer): Promise<void>;
+    /**
+     * Add Requests from both options.sources and options.sourcesFunction.
+     * This function is called only when persisted sources were not loaded.
+     * We need to avoid keeping both sources and requests in memory
+     * to reduce memory footprint with very large sources.
+     */
+    protected _addRequestsFromSources(): Promise<void>;
+    /**
+     * Persists the current state of the `RequestList` into the default {@link KeyValueStore}.
+     * The state is persisted automatically in regular intervals, but calling this method manually
+     * is useful in cases where you want to have the most current state available after you pause
+     * or stop fetching its requests. For example after you pause or abort a crawl. Or just before
+     * a server migration.
+     */
+    persistState(): Promise<void>;
+    /**
+     * Unlike persistState(), this is used only internally, since the sources
+     * are automatically persisted at RequestList initialization (if the persistRequestsKey is set),
+     * but there's no reason to persist it again afterwards, because RequestList is immutable.
+     */
+    protected _persistRequests(): Promise<void>;
+    /**
+     * Restores RequestList state from a state object.
+     */
+    protected _restoreState(state?: RequestListState): void;
+    /**
+     * Attempts to load state and requests using the `RequestList` configuration
+     * and returns a tuple of [state, requests] where each may be null if not loaded.
+     */
+    protected _loadStateAndPersistedRequests(): Promise<[RequestListState, Buffer]>;
+    /**
+     * Returns an object representing the internal state of the `RequestList` instance.
+     * Note that the object's fields can change in future releases.
+     */
+    getState(): RequestListState;
+    /**
+     * Resolves to `true` if the next call to {@link RequestList.fetchNextRequest} function
+     * would return `null`, otherwise it resolves to `false`.
+     * Note that even if the list is empty, there might be some pending requests currently being processed.
+     */
+    isEmpty(): Promise<boolean>;
+    /**
+     * Returns `true` if all requests were already handled and there are no more left.
+     */
+    isFinished(): Promise<boolean>;
+    /**
+     * Gets the next {@link Request} to process. First, the function gets a request previously reclaimed
+     * using the {@link RequestList.reclaimRequest} function, if there is any.
+     * Otherwise it gets the next request from sources.
+     *
+     * The function's `Promise` resolves to `null` if there are no more
+     * requests to process.
+     */
+    fetchNextRequest(): Promise<Request | null>;
+    /**
+     * Marks request as handled after successful processing.
+     */
+    markRequestHandled(request: Request): Promise<void>;
+    /**
+     * Reclaims request to the list if its processing failed.
+     * The request will become available in the next `this.fetchNextRequest()`.
+     */
+    reclaimRequest(request: Request): Promise<void>;
+    /**
+     * Adds all fetched requests from a URL from a remote resource.
+     */
+    protected _addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[]): Promise<void>;
+    protected _getPersistedState<T>(key: string): Promise<T>;
+    /**
+     * Fetches URLs from requestsFromUrl and returns them in format of list of requests
+     */
+    protected _fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]>;
+    /**
+     * Adds given request.
+     * If the `source` parameter is a string or plain object and not an instance
+     * of a `Request`, then the function creates a `Request` instance.
+     */
+    protected _addRequest(source: Source): void;
+    /**
+     * Helper function that validates unique key.
+     * Throws an error if uniqueKey is not a non-empty string.
+     */
+    protected _ensureUniqueKeyValid(uniqueKey: string): void;
+    /**
+     * Checks that request is not reclaimed and throws an error if so.
+     */
+    protected _ensureInProgressAndNotReclaimed(uniqueKey: string): void;
+    /**
+     * Throws an error if request list wasn't initialized.
+     */
+    protected _ensureIsInitialized(): void;
+    /**
+     * Returns the total number of unique requests present in the `RequestList`.
+     */
+    length(): number;
+    /**
+     * Returns number of handled requests.
+     */
+    handledCount(): number;
+    /**
+     * Opens a request list and returns a promise resolving to an instance
+     * of the {@link RequestList} class that is already initialized.
+     *
+     * {@link RequestList} represents a list of URLs to crawl, which is always stored in memory.
+     * To enable picking up where left off after a process restart, the request list sources
+     * are persisted to the key-value store at initialization of the list. Then, while crawling,
+     * a small state object is regularly persisted to keep track of the crawling status.
+     *
+     * For more details and code examples, see the {@link RequestList} class.
+     *
+     * **Example usage:**
+     *
+     * ```javascript
+     * const sources = [
+     *     'https://www.example.com',
+     *     'https://www.google.com',
+     *     'https://www.bing.com'
+     * ];
+     *
+     * const requestList = await RequestList.open('my-name', sources);
+     * ```
+     *
+     * @param listName
+     *   Name of the request list to be opened. Setting a name enables the `RequestList`'s state to be persisted
+     *   in the key-value store. This is useful in case of a restart or migration. Since `RequestList` is only
+     *   stored in memory, a restart or migration wipes it clean. Setting a name will enable the `RequestList`'s
+     *   state to survive those situations and continue where it left off.
+     *
+     *   The name will be used as a prefix in key-value store, producing keys such as `NAME-REQUEST_LIST_STATE`
+     *   and `NAME-REQUEST_LIST_SOURCES`.
+     *
+     *   If `null`, the list will not be persisted and will only be stored in memory. Process restart
+     *   will then cause the list to be crawled again from the beginning. We suggest always using a name.
+     * @param sources
+     *  An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,
+     *  plain objects that define at least the `url` property, or an array of {@link Request} instances.
+     *
+     *  **IMPORTANT:** The `sources` array will be consumed (left empty) after {@link RequestList} initializes.
+     *  This is a measure to prevent memory leaks in situations when millions of sources are
+     *  added.
+     *
+     *  Additionally, the `requestsFromUrl` property may be used instead of `url`,
+     *  which will instruct {@link RequestList} to download the source URLs from a given remote location.
+     *  The URLs will be parsed from the received response. In this case you can limit the URLs
+     *  using `regex` parameter containing regular expression pattern for URLs to be included.
+     *
+     *  For details, see the {@link RequestListOptions.sources}
+     * @param [options]
+     *   The {@link RequestList} options. Note that the `listName` parameter supersedes
+     *   the {@link RequestListOptions.persistStateKey} and {@link RequestListOptions.persistRequestsKey}
+     *   options and the `sources` parameter supersedes the {@link RequestListOptions.sources} option.
+     */
+    static open(listName: string | null, sources: Source[], options?: RequestListOptions): Promise<RequestList>;
+    /**
+     * @internal wraps public utility for mocking purposes
+     */
+    private _downloadListOfUrls;
+}
+/**
+ * Represents state of a {@link RequestList}. It can be used to resume a {@link RequestList} which has been previously processed.
+ * You can obtain the state by calling {@link RequestList.getState} and receive an object with
+ * the following structure:
+ *
+ * ```
+ * {
+ *     nextIndex: 5,
+ *     nextUniqueKey: 'unique-key-5'
+ *     inProgress: {
+ *         'unique-key-1': true,
+ *         'unique-key-4': true
+ *     },
+ * }
+ * ```
+ */
+export interface RequestListState {
+    /** Position of the next request to be processed. */
+    nextIndex: number;
+    /** Key of the next request to be processed. */
+    nextUniqueKey: string | null;
+    /** Array of request keys representing those that being processed at the moment. */
+    inProgress: string[];
+}
+export declare type Source = string | (Partial<RequestOptions> & {
+    requestsFromUrl?: string;
+    regex?: RegExp;
+}) | Request;
+declare type InternalSource = {
+    requestsFromUrl: string;
+    regex?: RegExp;
+};
+export declare type RequestListSourcesFunction = () => Promise<Source[]>;
+export {};
+//# sourceMappingURL=request_list.d.ts.map

package/storages/request_list.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"request_list.d.ts","sourceRoot":"","sources":["../../src/storages/request_list.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAKrD,gBAAgB;AAChB,eAAO,MAAM,qBAAqB,uBAAuB,CAAC;AAE1D,gBAAgB;AAChB,eAAO,MAAM,wBAAwB,0BAA0B,CAAC;AAIhE,MAAM,WAAW,kBAAkB;IAC/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,eAAe,CAAC,EAAE,0BAA0B,CAAC;IAE7C;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CAAC,EAAE,gBAAgB,CAAC;IAEzB;;;;;;;;;;;;;;OAcG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmEG;AACH,qBAAa,WAAW;IACpB,OAAO,CAAC,GAAG,CAAwC;IAEnD;;;;OAIG;IACH,QAAQ,EAAE,OAAO,EAAE,CAAM;IAEzB,kHAAkH;IAClH,OAAO,CAAC,SAAS,CAAK;IAEtB,gGAAgG;IAChG,OAAO,CAAC,gBAAgB,CAA8B;IAEtD;;;OAGG;IACH,UAAU,cAAqB;IAE/B;;;OAGG;IACH,SAAS,cAAqB;IAE9B;;;OAGG;IACH,gBAAgB,UAAQ;IAExB;;;OAGG;IACH,oBAAoB,UAAS;IAE7B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,kBAAkB,CAAC,CAAS;IACpC,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,KAAK,CAAC,CAAgB;IAC9B,OAAO,CAAC,iBAAiB,CAAU;IACnC,OAAO,CAAC,OAAO,CAAW;IAC1B,OAAO,CAAC,eAAe,CAAC,CAA6B;IACrD,OAAO,CAAC,MAAM,CAAe;IAE7B;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IAwC5C;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAwBjC;;;;OAIG;cACa,qBAAqB,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAe/E;;;;;OAKG;cACa,uBAAuB,IAAI,OAAO,CAAC,IAAI,CAAC;IAmCxD;;;;;;OAMG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAenC;;;;OAIG;cACa,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAOjD;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,gBAAgB,GAAG,IAAI;IAuDvD;;;OAGG;cACa,8BAA8B,IAAI,OAAO,CAAC,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAoBrF;;;OAGG;IACH,QAAQ,IAAI,gBAAgB;IAY5B;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAMjC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;;;;;;OAOG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAuBjD;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzD;;;OAGG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAUrD;;OAEG;cACa,mBAAmB,CAAC,MAAM,EAAE,cAAc,EAAE,eAAe,EAAE,cAAc,EAAE;cAmB7E,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;IAO9D;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAoBxF;;;;OAIG;IACH,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM;IAgCpC;;;OAGG;IACH,SAAS,CAAC,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAMxD;;OAEG;IACH,SAAS,CAAC,gCAAgC,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IASnE;;OAEG;IACH,SAAS,CAAC,oBAAoB,IAAI,IAAI;IAMtC;;OAEG;IACH,MAAM,IAAI,MAAM;IAMhB;;OAEG;IACH,YAAY,IAAI,MAAM;IAMtB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoDG;WACU,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;IAgBrH;;OAEG;YACW,mBAAmB;CAGpC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,gBAAgB;IAE7B,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAElB,+CAA+C;IAC/C,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAE7B,mFAAmF;IACnF,UAAU,EAAE,MAAM,EAAE,CAAC;CAExB;AAED,oBAAY,MAAM,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC;AACjH,aAAK,cAAc,GAAG;IAAE,eAAe,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAClE,oBAAY,0BAA0B,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC"}