@crawlee/http 3.0.3-beta.13

package/internals/http-crawler.js

@@ -0,0 +1,629 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createHttpRouter = exports.HttpCrawler = void 0;
+const tslib_1 = require("tslib");
+const timeout_1 = require("@apify/timeout");
+const utilities_1 = require("@apify/utilities");
+const basic_1 = require("@crawlee/basic");
+const content_type_1 = tslib_1.__importDefault(require("content-type"));
+const mime_types_1 = tslib_1.__importDefault(require("mime-types"));
+const got_scraping_1 = require("got-scraping");
+const path_1 = require("path");
+const iconv_lite_1 = tslib_1.__importDefault(require("iconv-lite"));
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const node_util_1 = tslib_1.__importDefault(require("node:util"));
+/**
+ * Default mime types, which HttpScraper supports.
+ */
+const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml', 'application/xml'];
+const APPLICATION_JSON_MIME_TYPE = 'application/json';
+const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
+    desiredConcurrency: 10,
+    snapshotterOptions: {
+        eventLoopSnapshotIntervalSecs: 2,
+        maxBlockedMillis: 100,
+    },
+    systemStatusOptions: {
+        maxEventLoopOverloadedRatio: 0.7,
+    },
+};
+/**
+ * Provides a framework for the parallel crawling of web pages using plain HTTP requests.
+ * The URLs to crawl are fed either from a static list of URLs
+ * or from a dynamic queue of URLs enabling recursive crawling of websites.
+ *
+ * It is very fast and efficient on data bandwidth. However, if the target website requires JavaScript
+ * to display the content, you might need to use {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler} instead,
+ * because it loads the pages using full-featured headless Chrome browser.
+ *
+ * This crawler downloads each URL using a plain HTTP request and doesn't do any HTML parsing.
+ *
+ * The source URLs are represented using {@apilink Request} objects that are fed from
+ * {@apilink RequestList} or {@apilink RequestQueue} instances provided by the {@apilink HttpCrawlerOptions.requestList}
+ * or {@apilink HttpCrawlerOptions.requestQueue} constructor options, respectively.
+ *
+ * If both {@apilink HttpCrawlerOptions.requestList} and {@apilink HttpCrawlerOptions.requestQueue} are used,
+ * the instance first processes URLs from the {@apilink RequestList} and automatically enqueues all of them
+ * to {@apilink RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ *
+ * The crawler finishes when there are no more {@apilink Request} objects to crawl.
+ *
+ * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ *
+ * ```javascript
+ * preNavigationHooks: [
+ *     (crawlingContext, gotOptions) => {
+ *         // ...
+ *     },
+ * ]
+ * ```
+ *
+ * By default, this crawler only processes web pages with the `text/html`
+ * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
+ * and skips pages with other content types. If you want the crawler to process other content types,
+ * use the {@apilink HttpCrawlerOptions.additionalMimeTypes} constructor option.
+ * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
+ * For details, see {@apilink HttpCrawlerOptions.requestHandler}.
+ *
+ * New requests are only dispatched when there is enough free CPU and memory available,
+ * using the functionality provided by the {@apilink AutoscaledPool} class.
+ * All {@apilink AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
+ * parameter of the constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+ * {@apilink AutoscaledPool} options are available directly in the constructor.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * import { HttpCrawler, Dataset } from '@crawlee/http';
+ *
+ * const crawler = new HttpCrawler({
+ *     requestList,
+ *     async requestHandler({ request, response, body, contentType }) {
+ *         // Save the data to dataset.
+ *         await Dataset.pushData({
+ *             url: request.url,
+ *             html: body,
+ *         });
+ *     },
+ * });
+ *
+ * await crawler.run([
+ *     'http://www.example.com/page-1',
+ *     'http://www.example.com/page-2',
+ * ]);
+ * ```
+ * @category Crawlers
+ */
+class HttpCrawler extends basic_1.BasicCrawler {
+    /**
+     * All `HttpCrawlerOptions` parameters are passed via an options object.
+     */
+    constructor(options = {}, config = basic_1.Configuration.getGlobalConfig()) {
+        (0, ow_1.default)(options, 'HttpCrawlerOptions', ow_1.default.object.exactShape(HttpCrawler.optionsShape));
+        const { requestHandler, handlePageFunction, requestHandlerTimeoutSecs = 60, navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, proxyConfiguration, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], 
+        // Ignored
+        handleRequestFunction, 
+        // BasicCrawler
+        autoscaledPoolOptions = HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS, ...basicCrawlerOptions } = options;
+        super({
+            ...basicCrawlerOptions,
+            requestHandler,
+            autoscaledPoolOptions,
+            // We need to add some time for internal functions to finish,
+            // but not too much so that we would stall the crawler.
+            requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + basic_1.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
+        }, config);
+        Object.defineProperty(this, "config", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: config
+        });
+        /**
+         * A reference to the underlying {@apilink ProxyConfiguration} class that manages the crawler's proxies.
+         * Only available if used by the crawler.
+         */
+        Object.defineProperty(this, "proxyConfiguration", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "userRequestHandlerTimeoutMillis", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "preNavigationHooks", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "postNavigationHooks", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "persistCookiesPerSession", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "navigationTimeoutMillis", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "ignoreSslErrors", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "suggestResponseEncoding", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "forceResponseEncoding", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "supportedMimeTypes", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * @internal wraps public utility for mocking purposes
+         */
+        Object.defineProperty(this, "_requestAsBrowser", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: (options) => {
+                return new Promise((resolve, reject) => {
+                    const stream = (0, got_scraping_1.gotScraping)(options);
+                    stream.on('error', reject);
+                    stream.on('response', () => {
+                        resolve(addResponsePropertiesToStream(stream));
+                    });
+                });
+            }
+        });
+        this._handlePropertyNameChange({
+            newName: 'requestHandler',
+            oldName: 'handlePageFunction',
+            propertyKey: 'requestHandler',
+            newProperty: requestHandler,
+            oldProperty: handlePageFunction,
+            allowUndefined: true,
+        });
+        if (!this.requestHandler) {
+            this.requestHandler = this.router;
+        }
+        // Cookies should be persisted per session only if session pool is used
+        if (!this.useSessionPool && persistCookiesPerSession) {
+            throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
+        }
+        this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
+        if (additionalMimeTypes.length)
+            this._extendSupportedMimeTypes(additionalMimeTypes);
+        if (suggestResponseEncoding && forceResponseEncoding) {
+            this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
+        }
+        this.userRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
+        this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
+        this.ignoreSslErrors = ignoreSslErrors;
+        this.suggestResponseEncoding = suggestResponseEncoding;
+        this.forceResponseEncoding = forceResponseEncoding;
+        this.proxyConfiguration = proxyConfiguration;
+        this.preNavigationHooks = preNavigationHooks;
+        this.postNavigationHooks = [
+            ({ request, response }) => this._abortDownloadOfBody(request, response),
+            ...postNavigationHooks,
+        ];
+        if (this.useSessionPool) {
+            this.persistCookiesPerSession = persistCookiesPerSession ?? true;
+        }
+        else {
+            this.persistCookiesPerSession = false;
+        }
+    }
+    /**
+     * **EXPERIMENTAL**
+     * Function for attaching CrawlerExtensions such as the Unblockers.
+     * @param extension Crawler extension that overrides the crawler configuration.
+     */
+    use(extension) {
+        (0, ow_1.default)(extension, ow_1.default.object.instanceOf(basic_1.CrawlerExtension));
+        const className = this.constructor.name;
+        const extensionOptions = extension.getCrawlerOptions();
+        for (const [key, value] of Object.entries(extensionOptions)) {
+            const isConfigurable = this.hasOwnProperty(key);
+            const originalType = typeof this[key];
+            const extensionType = typeof value; // What if we want to null something? It is really needed?
+            const isSameType = originalType === extensionType || value == null; // fast track for deleting keys
+            const exists = this[key] != null;
+            if (!isConfigurable) { // Test if the property can be configured on the crawler
+                throw new Error(`${extension.name} tries to set property "${key}" that is not configurable on ${className} instance.`);
+            }
+            if (!isSameType && exists) { // Assuming that extensions will only add up configuration
+                throw new Error(`${extension.name} tries to set property of different type "${extensionType}". "${className}.${key}: ${originalType}".`);
+            }
+            this.log.warning(`${extension.name} is overriding "${className}.${key}: ${originalType}" with ${value}.`);
+            this[key] = value;
+        }
+    }
+    /**
+     * Wrapper around requestHandler that opens and closes pages etc.
+     */
+    async _runRequestHandler(crawlingContext) {
+        const { request, session } = crawlingContext;
+        if (this.proxyConfiguration) {
+            const sessionId = session ? session.id : undefined;
+            crawlingContext.proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId);
+        }
+        if (!request.skipNavigation) {
+            await this._handleNavigation(crawlingContext);
+            (0, timeout_1.tryCancel)();
+            const parsed = await this._parseResponse(request, crawlingContext.response, crawlingContext);
+            const response = parsed.response;
+            const contentType = parsed.contentType;
+            (0, timeout_1.tryCancel)();
+            if (this.useSessionPool) {
+                this._throwOnBlockedRequest(session, response.statusCode);
+            }
+            if (this.persistCookiesPerSession) {
+                session.setCookiesFromResponse(response);
+            }
+            request.loadedUrl = response.url;
+            Object.assign(crawlingContext, parsed);
+            Object.defineProperty(crawlingContext, 'json', {
+                get() {
+                    if (contentType.type !== APPLICATION_JSON_MIME_TYPE)
+                        return null;
+                    const jsonString = parsed.body.toString(contentType.encoding);
+                    return JSON.parse(jsonString);
+                },
+            });
+        }
+        return (0, timeout_1.addTimeoutToPromise)(() => Promise.resolve(this.requestHandler(crawlingContext)), this.userRequestHandlerTimeoutMillis, `requestHandler timed out after ${this.userRequestHandlerTimeoutMillis / 1000} seconds.`);
+    }
+    async _handleNavigation(crawlingContext) {
+        const gotOptions = {};
+        const { request, session } = crawlingContext;
+        const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
+        // Execute pre navigation hooks before applying session pool cookies,
+        // as they may also set cookies in the session
+        await this._executeHooks(this.preNavigationHooks, crawlingContext, gotOptions);
+        (0, timeout_1.tryCancel)();
+        const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
+        this._applyCookies(crawlingContext, gotOptions, preNavigationHooksCookies, postNavigationHooksCookies);
+        const proxyUrl = crawlingContext.proxyInfo?.url;
+        crawlingContext.response = await (0, timeout_1.addTimeoutToPromise)(() => this._requestFunction({ request, session, proxyUrl, gotOptions }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        (0, timeout_1.tryCancel)();
+        await this._executeHooks(this.postNavigationHooks, crawlingContext, gotOptions);
+        (0, timeout_1.tryCancel)();
+    }
+    /**
+     * Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.
+     */
+    _applyCookies({ session, request }, gotOptions, preHookCookies, postHookCookies) {
+        const sessionCookie = session?.getCookieString(request.url) ?? '';
+        let alteredGotOptionsCookies = (gotOptions.headers?.Cookie || gotOptions.headers?.cookie || '');
+        if (gotOptions.headers?.Cookie && gotOptions.headers?.cookie) {
+            const { Cookie: upperCaseHeader, cookie: lowerCaseHeader, } = gotOptions.headers;
+            // eslint-disable-next-line max-len
+            this.log.warning(`Encountered mixed casing for the cookie headers in the got options for request ${request.url} (${request.id}). Their values will be merged`);
+            const sourceCookies = [];
+            if (Array.isArray(lowerCaseHeader)) {
+                sourceCookies.push(...lowerCaseHeader);
+            }
+            else {
+                sourceCookies.push(lowerCaseHeader);
+            }
+            if (Array.isArray(upperCaseHeader)) {
+                sourceCookies.push(...upperCaseHeader);
+            }
+            else {
+                sourceCookies.push(upperCaseHeader);
+            }
+            alteredGotOptionsCookies = (0, basic_1.mergeCookies)(request.url, sourceCookies);
+        }
+        const sourceCookies = [
+            sessionCookie,
+            preHookCookies,
+        ];
+        if (Array.isArray(alteredGotOptionsCookies)) {
+            sourceCookies.push(...alteredGotOptionsCookies);
+        }
+        else {
+            sourceCookies.push(alteredGotOptionsCookies);
+        }
+        sourceCookies.push(postHookCookies);
+        const mergedCookie = (0, basic_1.mergeCookies)(request.url, sourceCookies);
+        gotOptions.headers ?? (gotOptions.headers = {});
+        Reflect.deleteProperty(gotOptions.headers, 'Cookie');
+        Reflect.deleteProperty(gotOptions.headers, 'cookie');
+        gotOptions.headers.Cookie = mergedCookie;
+    }
+    /**
+     * Function to make the HTTP request. It performs optimizations
+     * on the request such as only downloading the request body if the
+     * received content type matches text/html, application/xml, application/xhtml+xml.
+     */
+    async _requestFunction({ request, session, proxyUrl, gotOptions }) {
+        const opts = this._getRequestOptions(request, session, proxyUrl, gotOptions);
+        try {
+            return await this._requestAsBrowser(opts);
+        }
+        catch (e) {
+            if (e instanceof got_scraping_1.TimeoutError) {
+                this._handleRequestTimeout(session);
+                return undefined;
+            }
+            throw e;
+        }
+    }
+    /**
+     * Encodes and parses response according to the provided content type
+     */
+    async _parseResponse(request, responseStream, crawlingContext) {
+        const { statusCode } = responseStream;
+        const { type, charset } = parseContentTypeFromResponse(responseStream);
+        const { response, encoding } = this._encodeResponse(request, responseStream, charset);
+        const contentType = { type, encoding };
+        if (statusCode >= 500) {
+            const body = await (0, utilities_1.readStreamToString)(response, encoding);
+            // Errors are often sent as JSON, so attempt to parse them,
+            // despite Accept header being set to text/html.
+            if (type === APPLICATION_JSON_MIME_TYPE) {
+                const errorResponse = JSON.parse(body);
+                let { message } = errorResponse;
+                if (!message)
+                    message = node_util_1.default.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
+                throw new Error(`${statusCode} - ${message}`);
+            }
+            // It's not a JSON, so it's probably some text. Get the first 100 chars of it.
+            throw new Error(`${statusCode} - Internal Server Error: ${body.slice(0, 100)}`);
+        }
+        else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
+            const isXml = type.includes('xml');
+            const parsed = await this._parseHTML(response, isXml, crawlingContext);
+            return { ...parsed, isXml, response, contentType };
+        }
+        else {
+            const body = await (0, utilities_1.concatStreamToBuffer)(response);
+            return { body, response, contentType };
+        }
+    }
+    async _parseHTML(response, _isXml, _crawlingContext) {
+        return {
+            body: await (0, utilities_1.concatStreamToBuffer)(response),
+        };
+    }
+    /**
+     * Combines the provided `requestOptions` with mandatory (non-overridable) values.
+     */
+    _getRequestOptions(request, session, proxyUrl, gotOptions) {
+        const requestOptions = {
+            url: request.url,
+            method: request.method,
+            proxyUrl,
+            timeout: { request: this.navigationTimeoutMillis },
+            sessionToken: session,
+            ...gotOptions,
+            headers: { ...request.headers, ...gotOptions?.headers },
+            https: {
+                ...gotOptions?.https,
+                rejectUnauthorized: !this.ignoreSslErrors,
+            },
+            isStream: true,
+        };
+        // Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header
+        Reflect.deleteProperty(requestOptions.headers, 'cookie');
+        // TODO this is incorrect, the check for man in the middle needs to be done
+        //   on individual proxy level, not on the `proxyConfiguration` level,
+        //   because users can use normal + MITM proxies in a single configuration.
+        // Disable SSL verification for MITM proxies
+        if (this.proxyConfiguration && this.proxyConfiguration.isManInTheMiddle) {
+            requestOptions.https = {
+                ...requestOptions.https,
+                rejectUnauthorized: false,
+            };
+        }
+        if (/PATCH|POST|PUT/.test(request.method))
+            requestOptions.body = request.payload;
+        return requestOptions;
+    }
+    _encodeResponse(request, response, encoding) {
+        if (this.forceResponseEncoding) {
+            encoding = this.forceResponseEncoding;
+        }
+        else if (!encoding && this.suggestResponseEncoding) {
+            encoding = this.suggestResponseEncoding;
+        }
+        // Fall back to utf-8 if we still don't have encoding.
+        const utf8 = 'utf8';
+        if (!encoding)
+            return { response, encoding: utf8 };
+        // This means that the encoding is one of Node.js supported
+        // encodings and we don't need to re-encode it.
+        if (Buffer.isEncoding(encoding))
+            return { response, encoding };
+        // Try to re-encode a variety of unsupported encodings to utf-8
+        if (iconv_lite_1.default.encodingExists(encoding)) {
+            const encodeStream = iconv_lite_1.default.encodeStream(utf8);
+            const decodeStream = iconv_lite_1.default.decodeStream(encoding).on('error', (err) => encodeStream.emit('error', err));
+            response.on('error', (err) => decodeStream.emit('error', err));
+            const encodedResponse = response.pipe(decodeStream).pipe(encodeStream);
+            encodedResponse.statusCode = response.statusCode;
+            encodedResponse.headers = response.headers;
+            encodedResponse.url = response.url;
+            return {
+                response: encodedResponse,
+                encoding: utf8,
+            };
+        }
+        throw new Error(`Resource ${request.url} served with unsupported charset/encoding: ${encoding}`);
+    }
+    /**
+     * Checks and extends supported mime types
+     */
+    _extendSupportedMimeTypes(additionalMimeTypes) {
+        for (const mimeType of additionalMimeTypes) {
+            if (mimeType === '*/*') {
+                this.supportedMimeTypes.add(mimeType);
+                continue;
+            }
+            try {
+                const parsedType = content_type_1.default.parse(mimeType);
+                this.supportedMimeTypes.add(parsedType.type);
+            }
+            catch (err) {
+                throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
+            }
+        }
+    }
+    /**
+     * Handles timeout request
+     */
+    _handleRequestTimeout(session) {
+        session?.markBad();
+        throw new Error(`request timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds.`);
+    }
+    _abortDownloadOfBody(request, response) {
+        const { statusCode } = response;
+        const { type } = parseContentTypeFromResponse(response);
+        if (statusCode === 406) {
+            request.noRetry = true;
+            throw new Error(`Resource ${request.url} is not available in the format requested by the Accept header. Skipping resource.`);
+        }
+        if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && statusCode < 500) {
+            request.noRetry = true;
+            throw new Error(`Resource ${request.url} served Content-Type ${type}, `
+                + `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
+        }
+    }
+}
+exports.HttpCrawler = HttpCrawler;
+Object.defineProperty(HttpCrawler, "optionsShape", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: {
+        ...basic_1.BasicCrawler.optionsShape,
+        handlePageFunction: ow_1.default.optional.function,
+        navigationTimeoutSecs: ow_1.default.optional.number,
+        ignoreSslErrors: ow_1.default.optional.boolean,
+        additionalMimeTypes: ow_1.default.optional.array.ofType(ow_1.default.string),
+        suggestResponseEncoding: ow_1.default.optional.string,
+        forceResponseEncoding: ow_1.default.optional.string,
+        proxyConfiguration: ow_1.default.optional.object.validate(basic_1.validators.proxyConfiguration),
+        persistCookiesPerSession: ow_1.default.optional.boolean,
+        preNavigationHooks: ow_1.default.optional.array,
+        postNavigationHooks: ow_1.default.optional.array,
+    }
+});
+/**
+ * The stream object returned from got does not have the below properties.
+ * At the same time, you can't read data directly from the response stream,
+ * because they won't get emitted unless you also read from the primary
+ * got stream. To be able to work with only one stream, we move the expected props
+ * from the response stream to the got stream.
+ * @internal
+ */
+function addResponsePropertiesToStream(stream) {
+    const properties = [
+        'statusCode', 'statusMessage', 'headers',
+        'complete', 'httpVersion', 'rawHeaders',
+        'rawTrailers', 'trailers', 'url',
+        'request',
+    ];
+    const response = stream.response;
+    response.on('end', () => {
+        // @ts-expect-error
+        Object.assign(stream.rawTrailers, response.rawTrailers);
+        // @ts-expect-error
+        Object.assign(stream.trailers, response.trailers);
+        // @ts-expect-error
+        stream.complete = response.complete;
+    });
+    for (const prop of properties) {
+        if (!(prop in stream)) {
+            // @ts-expect-error
+            stream[prop] = response[prop];
+        }
+    }
+    return stream;
+}
+/**
+ * Gets parsed content type from response object
+ * @param response HTTP response object
+ */
+function parseContentTypeFromResponse(response) {
+    (0, ow_1.default)(response, ow_1.default.object.partialShape({
+        url: ow_1.default.string.url,
+        headers: ow_1.default.object,
+    }));
+    const { url, headers } = response;
+    let parsedContentType;
+    if (headers['content-type']) {
+        try {
+            parsedContentType = content_type_1.default.parse(headers['content-type']);
+        }
+        catch {
+            // Can not parse content type from Content-Type header. Try to parse it from file extension.
+        }
+    }
+    // Parse content type from file extension as fallback
+    if (!parsedContentType) {
+        const parsedUrl = new URL(url);
+        const contentTypeFromExtname = mime_types_1.default.contentType((0, path_1.extname)(parsedUrl.pathname))
+            || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
+        parsedContentType = content_type_1.default.parse(contentTypeFromExtname);
+    }
+    return {
+        type: parsedContentType.type,
+        charset: parsedContentType.parameters.charset,
+    };
+}
+/**
+ * Creates new {@apilink Router} instance that works based on request labels.
+ * This instance can then serve as a `requestHandler` of your {@apilink HttpCrawler}.
+ * Defaults to the {@apilink HttpCrawlingContext}.
+ *
+ * > Serves as a shortcut for using `Router.create<HttpCrawlingContext>()`.
+ *
+ * ```ts
+ * import { HttpCrawler, createHttpRouter } from 'crawlee';
+ *
+ * const router = createHttpRouter();
+ * router.addHandler('label-a', async (ctx) => {
+ *    ctx.log.info('...');
+ * });
+ * router.addDefaultHandler(async (ctx) => {
+ *    ctx.log.info('...');
+ * });
+ *
+ * const crawler = new HttpCrawler({
+ *     requestHandler: router,
+ * });
+ * await crawler.run();
+ * ```
+ */
+function createHttpRouter() {
+    return basic_1.Router.create();
+}
+exports.createHttpRouter = createHttpRouter;
+//# sourceMappingURL=http-crawler.js.map

package/internals/http-crawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":";;;;AAAA,4CAAgE;AAChE,gDAA4E;AAW5E,0CAQwB;AAGxB,wEAA6C;AAC7C,oEAA8B;AAE9B,+CAAyD;AACzD,+BAA+B;AAE/B,oEAA+B;AAC/B,oDAAoB;AACpB,kEAA6B;AAE7B;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AAuJF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAa,WAAyF,SAAQ,oBAAqB;IAiC/H;;OAEG;IACH,YAAY,UAAuC,EAAE,EAAoB,SAAS,qBAAa,CAAC,eAAe,EAAE;QAC7G,IAAA,YAAE,EAAC,OAAO,EAAE,oBAAoB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,cAAc,EACd,kBAAkB,EAElB,yBAAyB,GAAG,EAAE,EAC9B,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,kBAAkB,EAClB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE;QAExB,UAAU;QACV,qBAAqB;QAErB,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,cAAc;YACd,qBAAqB;YACrB,6DAA6D;YAC7D,uDAAuD;YACvD,yBAAyB,EAAE,qBAAqB,GAAG,yBAAyB,GAAG,yCAAiC;SACnH,EAAE,MAAM,CAAC,CAAC;;;;;mBAjC0D;;QAnCzE;;;WAGG;QACH;;;;;WAAwC;QAExC;;;;;WAAkD;QAClD;;;;;WAA0D;QAC1D;;;;;WAA2D;QAC3D;;;;;WAA4C;QAC5C;;;;;WAA0C;QAC1C;;;;;WAAmC;QACnC;;;;;WAA2C;QAC3C;;;;;WAAyC;QACzC;;;;;WAAmD;QA2bnD;;WAEG;QACH;;;;mBAA4B,CAAC,OAAyC,EAAE,EAAE;gBACtE,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBACpD,MAAM,MAAM,GAAG,IAAA,0BAAW,EAAC,OAAO,CAAC,CAAC;oBAEpC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;oBAC3B,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE;wBACvB,OAAO,CAAC,6BAA6B,CAAC,MAAM,CAAC,CAAC,CAAC;oBACnD,CAAC,CAAC,CAAC;gBACP,CAAC,CAAC,CAAC;YACP,CAAC;WAAC;QA/YE,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,gBAAgB;YACzB,OAAO,EAAE,oBAAoB;YAC7B,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,cAAc;YAC3B,WAAW,EAAE,kBAAkB;YAC/B,cAAc,EAAE,IAAI;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;YACtB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;SACrC;QAED,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE;YAClD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;SACtG;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE;YAClD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,sGAAsG,CAAC,CAAC;SAC5H;QAED,IAAI,CAAC,+BAA+B,GAAG,yBAAyB,GAAG,IAAI,CAAC;QACxE,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,IAAI,IAAI,CAAC;SACpE;aAAM;YACH,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;SACzC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAAC,SAA2B;QAC3B,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAgB,CAAC,CAAC,CAAC;QAEtD,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAExC,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,CAAC;QAEvD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE;YACzD,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAChD,MAAM,YAAY,GAAG,OAAO,IAAI,CAAC,GAAiB,CAAC,CAAC;YACpD,MAAM,aAAa,GAAG,OAAO,KAAK,CAAC,CAAC,0DAA0D;YAC9F,MAAM,UAAU,GAAG,YAAY,KAAK,aAAa,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,+BAA+B;YACnG,MAAM,MAAM,GAAG,IAAI,CAAC,GAAiB,CAAC,IAAI,IAAI,CAAC;YAE/C,IAAI,CAAC,cAAc,EAAE,EAAE,wDAAwD;gBAC3E,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,CAAC,IAAI,2BAA2B,GAAG,iCAAiC,SAAS,YAAY,CAAC,CAAC;aAC1H;YAED,IAAI,CAAC,UAAU,IAAI,MAAM,EAAE,EAAE,0DAA0D;gBACnF,MAAM,IAAI,KAAK,CACX,GAAG,SAAS,CAAC,IAAI,6CAA6C,aAAa,OAAO,SAAS,IAAI,GAAG,KAAK,YAAY,IAAI,CAC1H,CAAC;aACL;YAED,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,IAAI,mBAAmB,SAAS,IAAI,GAAG,KAAK,YAAY,UAAU,KAAK,GAAG,CAAC,CAAC;YAE1G,IAAI,CAAC,GAAiB,CAAC,GAAG,KAAyB,CAAC;SACvD;IACL,CAAC;IAED;;OAEG;IACgB,KAAK,CAAC,kBAAkB,CAAC,eAAwB;QAChE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,eAAe,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;SACrF;QACD,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE;YACzB,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAA,mBAAS,GAAE,CAAC;YAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,eAAe,CAAC,QAAS,EAAE,eAAe,CAAC,CAAC;YAC9F,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;YACxC,IAAA,mBAAS,GAAE,CAAC;YAEZ,IAAI,IAAI,CAAC,cAAc,EAAE;gBACrB,IAAI,CAAC,sBAAsB,CAAC,OAAQ,EAAE,QAAQ,CAAC,UAAW,CAAC,CAAC;aAC/D;YAED,IAAI,IAAI,CAAC,wBAAwB,EAAE;gBAC/B,OAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;aAC7C;YAED,OAAO,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC;YAEjC,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;YAEvC,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE;gBAC3C,GAAG;oBACC,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;wBAAE,OAAO,IAAI,CAAC;oBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;oBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBAClC,CAAC;aACJ,CAAC,CAAC;SACN;QAED,OAAO,IAAA,6BAAmB,EACtB,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC,EAC3D,IAAI,CAAC,+BAA+B,EACpC,kCAAkC,IAAI,CAAC,+BAA+B,GAAG,IAAI,WAAW,CAC3F,CAAC;IACN,CAAC;IAES,KAAK,CAAC,iBAAiB,CAAC,eAAwB;QACtD,MAAM,UAAU,GAAG,EAAiB,CAAC;QACrC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAC7C,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE5E,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAC/E,IAAA,mBAAS,GAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE7E,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,UAAU,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEvG,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,eAAe,CAAC,QAAQ,GAAG,MAAM,IAAA,6BAAmB,EAChD,GAAG,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EACvE,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,IAAA,mBAAS,GAAE,CAAC;QAEZ,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAChF,IAAA,mBAAS,GAAE,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,EAAE,OAAO,EAAE,OAAO,EAAmB,EAAE,UAAuB,EAAE,cAAsB,EAAE,eAAuB;QACjI,MAAM,aAAa,GAAG,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAClE,IAAI,wBAAwB,GAAG,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;QAEhG,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE;YAC1D,MAAM,EACF,MAAM,EAAE,eAAe,EACvB,MAAM,EAAE,eAAe,GAC1B,GAAG,UAAU,CAAC,OAAO,CAAC;YAEvB,mCAAmC;YACnC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,kFAAkF,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,gCAAgC,CAAC,CAAC;YAE/J,MAAM,aAAa,GAAG,EAAE,CAAC;YAEzB,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE;gBAChC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;aAC1C;iBAAM;gBACH,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;aACvC;YAED,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE;gBAChC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;aAC1C;iBAAM;gBACH,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;aACvC;YAED,wBAAwB,GAAG,IAAA,oBAAY,EAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;SACvE;QAED,MAAM,aAAa,GAAG;YAClB,aAAa;YACb,cAAc;SACjB,CAAC;QAEF,IAAI,KAAK,CAAC,OAAO,CAAC,wBAAwB,CAAC,EAAE;YACzC,aAAa,CAAC,IAAI,CAAC,GAAG,wBAAwB,CAAC,CAAC;SACnD;aAAM;YACH,aAAa,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;SAChD;QAED,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpC,MAAM,YAAY,GAAG,IAAA,oBAAY,EAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAE9D,UAAU,CAAC,OAAO,KAAlB,UAAU,CAAC,OAAO,GAAK,EAAE,EAAC;QAC1B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACrD,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACrD,UAAU,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAA0B;QAC/F,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE7E,IAAI;YACA,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;SAC7C;QAAC,OAAO,CAAC,EAAE;YACR,IAAI,CAAC,YAAY,2BAAY,EAAE;gBAC3B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,SAAuC,CAAC;aAClD;YAED,MAAM,CAAC,CAAC;SACX;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAgB,EAAE,cAA+B,EAAE,eAAwB;QACtG,MAAM,EAAE,UAAU,EAAE,GAAG,cAAc,CAAC;QACtC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,cAAc,CAAC,CAAC;QACvE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QACtF,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,UAAW,IAAI,GAAG,EAAE;YACpB,MAAM,IAAI,GAAG,MAAM,IAAA,8BAAkB,EAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE1D,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE;gBACrC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,mBAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,MAAM,OAAO,EAAE,CAAC,CAAC;aACjD;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;SACnF;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC;YACvE,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;SACtD;aAAM;YACH,MAAM,IAAI,GAAG,MAAM,IAAA,gCAAoB,EAAC,QAAQ,CAAC,CAAC;YAClD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;SAC1C;IACL,CAAC;IAES,KAAK,CAAC,UAAU,CAAC,QAAyB,EAAE,MAAe,EAAE,gBAAyB;QAC5F,OAAO;YACH,IAAI,EAAE,MAAM,IAAA,gCAAoB,EAAC,QAAQ,CAAC;SACzB,CAAC;IAC1B,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAgB,EAAE,OAAiB,EAAE,QAAiB,EAAE,UAAwB;QACzG,MAAM,cAAc,GAAqC;YACrD,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAgB;YAChC,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE;YAClD,YAAY,EAAE,OAAO;YACrB,GAAG,UAAU;YACb,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE;YACvD,KAAK,EAAE;gBACH,GAAG,UAAU,EAAE,KAAK;gBACpB,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,QAAQ,EAAE,IAAI;SACjB,CAAC;QAEF,yHAAyH;QACzH,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,OAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1D,2EAA2E;QAC3E,sEAAsE;QACtE,2EAA2E;QAC3E,4CAA4C;QAC5C,IAAI,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;YACrE,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;SACL;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;QAEjF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CAAC,OAAgB,EAAE,QAAyB,EAAE,QAAwB;QAI3F,IAAI,IAAI,CAAC,qBAAqB,EAAE;YAC5B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;SAC3D;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE;YAClD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;SAC7D;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,oBAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE;YAChC,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACtE,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,YAAY,CAIpE,CAAC;YACF,eAAe,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;YACjD,eAAe,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC3C,eAAe,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;YACnC,OAAO;gBACH,QAAQ,EAAE,eAAsB;gBAChC,QAAQ,EAAE,IAAI;aACjB,CAAC;SACL;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE;YACxC,IAAI,QAAQ,KAAK,KAAK,EAAE;gBACpB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;aACZ;YAED,IAAI;gBACA,MAAM,UAAU,GAAG,sBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;aAChD;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;aAC9F;SACJ;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,EAAE,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,2BAA2B,GAAG,IAAI,WAAW,CAAC,CAAC;IACnG,CAAC;IAEO,oBAAoB,CAAC,OAAgB,EAAE,QAAyB;QACpE,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC;QAChC,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,UAAU,KAAK,GAAG,EAAE;YACpB,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,oFAAoF,CAAC,CAAC;SAChI;QAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,UAAW,GAAG,GAAG,EAAE;YAChG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;kBACjE,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;SACvG;IACL,CAAC;;AAxcL,kCAudC;AAtcG;;;;WAAyC;QACrC,GAAG,oBAAY,CAAC,YAAY;QAC5B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAExC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAU,CAAC,kBAAkB,CAAC;QAC9E,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;KACzC;GAAC;AAicN;;;;;;;GAOG;AACH,SAAS,6BAA6B,CAAC,MAAkB;IACrD,MAAM,UAAU,GAAG;QACf,YAAY,EAAE,eAAe,EAAE,SAAS;QACxC,UAAU,EAAE,aAAa,EAAE,YAAY;QACvC,aAAa,EAAE,UAAU,EAAE,KAAK;QAChC,SAAS;KACZ,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;IAElC,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACpB,mBAAmB;QACnB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;QACxD,mBAAmB;QACnB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAElD,mBAAmB;QACnB,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE;QAC3B,IAAI,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE;YACnB,mBAAmB;YACnB,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAA6B,CAAC,CAAC;SAC1D;KACJ;IAED,OAAO,MAAoC,CAAC;AAChD,CAAC;AAED;;;GAGG;AACH,SAAS,4BAA4B,CAAC,QAAyB;IAC3D,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QAChC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;QAClB,OAAO,EAAE,YAAE,CAAC,MAAM;KACrB,CAAC,CAAC,CAAC;IAEJ,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IAClC,IAAI,iBAAiB,CAAC;IAEtB,IAAI,OAAO,CAAC,cAAc,CAAC,EAAE;QACzB,IAAI;YACA,iBAAiB,GAAG,sBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;SACxE;QAAC,MAAM;YACJ,4FAA4F;SAC/F;KACJ;IAED,qDAAqD;IACrD,IAAI,CAAC,iBAAiB,EAAE;QACpB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,sBAAsB,GAAG,oBAAI,CAAC,WAAW,CAAC,IAAA,cAAO,EAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;eACrE,yCAAyC,CAAC,CAAC,0FAA0F;QAC5I,iBAAiB,GAAG,sBAAiB,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;KACvE;IAED,OAAO;QACH,IAAI,EAAE,iBAAiB,CAAC,IAAI;QAC5B,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,OAAyB;KAClE,CAAC;AACN,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAgB,gBAAgB;IAC5B,OAAO,cAAM,CAAC,MAAM,EAAW,CAAC;AACpC,CAAC;AAFD,4CAEC"}