@crawlee/puppeteer 3.0.0-alpha.2 → 3.0.0-alpha.3

package/dist/internals/utils/puppeteer_utils.d.ts

@@ -0,0 +1,298 @@
+/**
+ * A namespace that contains various utilities for
+ * [Puppeteer](https://github.com/puppeteer/puppeteer) - the headless Chrome Node API.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const Apify = require('apify');
+ * const { puppeteer } = Actor.utils;
+ *
+ * // Open https://www.example.com in Puppeteer
+ * const browser = await Actor.launchPuppeteer();
+ * const page = await browser.newPage();
+ * await page.goto('https://www.example.com');
+ *
+ * // Inject jQuery into a page
+ * await puppeteer.injectJQuery(page);
+ * ```
+ * @module puppeteerUtils
+ */
+import { Page, HTTPResponse, ResponseForRequest } from 'puppeteer';
+import { Request } from '@crawlee/core';
+import { Dictionary } from '@crawlee/utils';
+import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements';
+import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer_request_interception';
+export interface DirectNavigationOptions {
+    /**
+     * Maximum operation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The
+     * default value can be changed by using the browserContext.setDefaultNavigationTimeout(timeout),
+     * browserContext.setDefaultTimeout(timeout), page.setDefaultNavigationTimeout(timeout) or
+     * page.setDefaultTimeout(timeout) methods.
+     */
+    timeout?: number;
+    /**
+     * When to consider operation succeeded, defaults to `load`. Events can be either:
+     * - `'domcontentloaded'` - consider operation to be finished when the `DOMContentLoaded` event is fired.
+     * - `'load'` - consider operation to be finished when the `load` event is fired.
+     * - `'networkidle'` - consider operation to be finished when there are no network connections for at least `500` ms.
+     */
+    waitUntil?: 'domcontentloaded' | 'load' | 'networkidle';
+    /**
+     * Referer header value. If provided it will take preference over the referer header value set by page.setExtraHTTPHeaders(headers).
+     */
+    referer?: string;
+}
+export interface InjectFileOptions {
+    /**
+     * Enables the injected script to survive page navigations and reloads without need to be re-injected manually.
+     * This does not mean, however, that internal state will be preserved. Just that it will be automatically
+     * re-injected on each navigation before any other scripts get the chance to execute.
+     */
+    surviveNavigations?: boolean;
+}
+export interface BlockRequestsOptions {
+    /**
+     * The patterns of URLs to block from being loaded by the browser.
+     * Only `*` can be used as a wildcard. It is also automatically added to the beginning
+     * and end of the pattern. This limitation is enforced by the DevTools protocol.
+     * `.png` is the same as `*.png*`.
+     */
+    urlPatterns?: string[];
+    /**
+     * If you just want to append to the default blocked patterns, use this property.
+     */
+    extraUrlPatterns?: string[];
+}
+export interface CompiledScriptParams {
+    page: Page;
+    request: Request;
+}
+export declare type CompiledScriptFunction = (params: CompiledScriptParams) => Promise<unknown>;
+/**
+ * Injects a JavaScript file into a Puppeteer page.
+ * Unlike Puppeteer's `addScriptTag` function, this function works on pages
+ * with arbitrary Cross-Origin Resource Sharing (CORS) policies.
+ *
+ * File contents are cached for up to 10 files to limit file system access.
+ *
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param filePath File path
+ * @param [options]
+ */
+export declare function injectFile(page: Page, filePath: string, options?: InjectFileOptions): Promise<unknown>;
+/**
+ * Injects the [jQuery](https://jquery.com/) library into a Puppeteer page.
+ * jQuery is often useful for various web scraping and crawling tasks.
+ * For example, it can help extract text from HTML elements using CSS selectors.
+ *
+ * Beware that the injected jQuery object will be set to the `window.$` variable and thus it might cause conflicts with
+ * other libraries included by the page that use the same variable name (e.g. another version of jQuery).
+ * This can affect functionality of page's scripts.
+ *
+ * The injected jQuery will survive page navigations and reloads.
+ *
+ * **Example usage:**
+ * ```javascript
+ * await Actor.utils.puppeteer.injectJQuery(page);
+ * const title = await page.evaluate(() => {
+ *   return $('head title').text();
+ * });
+ * ```
+ *
+ * Note that `injectJQuery()` does not affect the Puppeteer's
+ * [`page.$()`](https://pptr.dev/#?product=Puppeteer&show=api-pageselector)
+ * function in any way.
+ *
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ */
+export declare function injectJQuery(page: Page): Promise<unknown>;
+/**
+ * Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
+ * This is useful to speed up crawling of websites, since it reduces the amount
+ * of data that needs to be downloaded from the web, but it may break some websites
+ * or unexpectedly prevent loading of resources.
+ *
+ * By default, the function will block all URLs including the following patterns:
+ *
+ * ```json
+ * [".css", ".jpg", ".jpeg", ".png", ".svg", ".gif", ".woff", ".pdf", ".zip"]
+ * ```
+ *
+ * If you want to extend this list further, use the `extraUrlPatterns` option,
+ * which will keep blocking the default patterns, as well as add your custom ones.
+ * If you would like to block only specific patterns, use the `urlPatterns` option,
+ * which will override the defaults and block only URLs with your custom patterns.
+ *
+ * This function does not use Puppeteer's request interception and therefore does not interfere
+ * with browser cache. It's also faster than blocking requests using interception,
+ * because the blocking happens directly in the browser without the round-trip to Node.js,
+ * but it does not provide the extra benefits of request interception.
+ *
+ * The function will never block main document loads and their respective redirects.
+ *
+ * **Example usage**
+ * ```javascript
+ * const Apify = require('apify');
+ *
+ * const browser = await Actor.launchPuppeteer();
+ * const page = await browser.newPage();
+ *
+ * // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.
+ * await Actor.utils.puppeteer.blockRequests(page, {
+ *     extraUrlPatterns: ['adsbygoogle.js'],
+ * });
+ *
+ * await page.goto('https://cnn.com');
+ * ```
+ *
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param [options]
+ */
+export declare function blockRequests(page: Page, options?: BlockRequestsOptions): Promise<void>;
+/**
+ * `blockResources()` has a high impact on performance in recent versions of Puppeteer.
+ * Until this resolves, please use `Actor.utils.puppeteer.blockRequests()`.
+ * @deprecated
+ */
+export declare const blockResources: (page: Page, resourceTypes?: string[]) => Promise<void>;
+/**
+ * *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal
+ * performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.
+ *
+ * Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
+ * *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
+ *   This issue should be resolved or atleast mitigated in future iterations of this feature.
+ * @param page
+ *   Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param cache
+ *   Object in which responses are stored
+ * @param responseUrlRules
+ *   List of rules that are used to check if the response should be cached.
+ *   String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
+ * @deprecated
+ */
+export declare function cacheResponses(page: Page, cache: Dictionary<Partial<ResponseForRequest>>, responseUrlRules: (string | RegExp)[]): Promise<void>;
+/**
+ * Compiles a Puppeteer script into an async function that may be executed at any time
+ * by providing it with the following object:
+ * ```
+ * {
+ *    page: Page,
+ *    request: Request,
+ * }
+ * ```
+ * Where `page` is a Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page)
+ * and `request` is a {@link Request}.
+ *
+ * The function is compiled by using the `scriptString` parameter as the function's body,
+ * so any limitations to function bodies apply. Return value of the compiled function
+ * is the return value of the function body = the `scriptString` parameter.
+ *
+ * As a security measure, no globals such as `process` or `require` are accessible
+ * from within the function body. Note that the function does not provide a safe
+ * sandbox and even though globals are not easily accessible, malicious code may
+ * still execute in the main process via prototype manipulation. Therefore you
+ * should only use this function to execute sanitized or safe code.
+ *
+ * Custom context may also be provided using the `context` parameter. To improve security,
+ * make sure to only pass the really necessary objects to the context. Preferably making
+ * secured copies beforehand.
+ *
+ */
+export declare function compileScript(scriptString: string, context?: Dictionary): CompiledScriptFunction;
+/**
+ * Extended version of Puppeteer's `page.goto()` allowing to perform requests with HTTP method other than GET,
+ * with custom headers and POST payload. URL, method, headers and payload are taken from
+ * request parameter that must be an instance of Request class.
+ *
+ * *NOTE:* In recent versions of Puppeteer using requests other than GET, overriding headers and adding payloads disables
+ * browser cache which degrades performance.
+ *
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param request
+ * @param [gotoOptions] Custom options for `page.goto()`.
+ */
+export declare function gotoExtended(page: Page, request: Request, gotoOptions?: DirectNavigationOptions): Promise<HTTPResponse | null>;
+export interface InfiniteScrollOptions {
+    /**
+     * How many seconds to scroll for. If 0, will scroll until bottom of page.
+     * @default 1
+     */
+    timeoutSecs?: number;
+    /**
+     * How many seconds to wait for no new content to load before exit.
+     * @default 4
+     */
+    waitForSecs?: number;
+    /**
+     * If true, it will scroll up a bit after each scroll down. This is required on some websites for the scroll to work.
+     * @default false
+     */
+    scrollDownAndUp?: boolean;
+    /**
+     * Optionally checks and clicks a button if it appears while scrolling. This is required on some websites for the scroll to work.
+     */
+    buttonSelector?: string;
+    /**
+     * This function is called after every scroll and stops the scrolling process if it returns `true`. The function can be `async`.
+     */
+    stopScrollCallback?: () => unknown | Promise<unknown>;
+}
+/**
+ * Scrolls to the bottom of a page, or until it times out.
+ * Loads dynamic content when it hits the bottom of a page, and then continues scrolling.
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param [options]
+ */
+export declare function infiniteScroll(page: Page, options?: InfiniteScrollOptions): Promise<void>;
+export interface SaveSnapshotOptions {
+    /**
+     * Key under which the screenshot and HTML will be saved. `.jpg` will be appended for screenshot and `.html` for HTML.
+     * @default 'SNAPSHOT'
+     */
+    key?: string;
+    /**
+     * The quality of the image, between 0-100. Higher quality images have bigger size and require more storage.
+     * @default 50
+     */
+    screenshotQuality?: number;
+    /**
+     * If true, it will save a full screenshot of the current page as a record with `key` appended by `.jpg`.
+     * @default true
+     */
+    saveScreenshot?: boolean;
+    /**
+     * If true, it will save a full HTML of the current page as a record with `key` appended by `.html`.
+     * @default true
+     */
+    saveHtml?: boolean;
+    /**
+     * Name or id of the Key-Value store where snapshot is saved. By default it is saved to default Key-Value store.
+     * @default null
+     */
+    keyValueStoreName?: string | null;
+}
+/**
+ * Saves a full screenshot and HTML of the current page into a Key-Value store.
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param [options]
+ */
+export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions): Promise<void>;
+export { enqueueLinksByClickingElements, addInterceptRequestHandler, removeInterceptRequestHandler, };
+/** @internal */
+export declare const puppeteerUtils: {
+    injectFile: typeof injectFile;
+    injectJQuery: typeof injectJQuery;
+    enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
+    blockRequests: typeof blockRequests;
+    blockResources: (page: Page, resourceTypes?: string[]) => Promise<void>;
+    cacheResponses: typeof cacheResponses;
+    compileScript: typeof compileScript;
+    gotoExtended: typeof gotoExtended;
+    addInterceptRequestHandler: typeof addInterceptRequestHandler;
+    removeInterceptRequestHandler: typeof removeInterceptRequestHandler;
+    infiniteScroll: typeof infiniteScroll;
+    saveSnapshot: typeof saveSnapshot;
+};
+//# sourceMappingURL=puppeteer_utils.d.ts.map

package/dist/internals/utils/puppeteer_utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"puppeteer_utils.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/puppeteer_utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAOH,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,kBAAkB,EAAmC,MAAM,WAAW,CAAC;AAEpG,OAAO,EAAiB,OAAO,EAAc,MAAM,eAAe,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,8BAA8B,EAAE,MAAM,iCAAiC,CAAC;AACjF,OAAO,EAAE,0BAA0B,EAAE,6BAA6B,EAAE,MAAM,kCAAkC,CAAC;AAU7G,MAAM,WAAW,uBAAuB;IACpC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,kBAAkB,GAAG,MAAM,GAAG,aAAa,CAAC;IAExD;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAC9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACjC;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,OAAO,CAAC;CACpB;AAED,oBAAY,sBAAsB,GAAG,CAAC,MAAM,EAAE,oBAAoB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAOxF;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAgBhH;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAGzD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAgBjG;AAED;;;;GAIG;AACH,eAAO,MAAM,cAAc,SAAgB,IAAI,4CAQ9C,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA4CrJ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE,UAAgC,GAAG,sBAAsB,CAcrH;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,GAAE,uBAA4B,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAuCxI;AAED,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzD;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAqGnG;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAmC/F;AAED,OAAO,EACH,8BAA8B,EAC9B,0BAA0B,EAC1B,6BAA6B,GAChC,CAAC;AAEF,gBAAgB;AAChB,eAAO,MAAM,cAAc;;;;;2BApYgB,IAAI;;;;;;;;CAiZ9C,CAAC"}