@crawlee/puppeteer 4.0.0-beta.134 → 4.0.0-beta.136

package/internals/enqueue-links/click-elements.js

@@ -1,5 +1,6 @@
 import { URL } from 'node:url';
-import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, filterRequestOptionsByPatterns, parseArgument, urlPatternSchema, Request, schemas, serviceLocator, } from '@crawlee/browser';
+import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, filterRequestOptionsByPatterns, urlPatternSchema, Request, serviceLocator, } from '@crawlee/browser';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { z } from 'zod';
 import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer_request_interception.js';
 const STARTING_Z_INDEX = 2147400000;

package/internals/puppeteer-crawler.d.ts

@@ -1,13 +1,11 @@
-import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
+import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, ContextPipeline, CrawlingContext, GetUserDataFromRequest, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import { BrowserCrawler } from '@crawlee/browser';
 import type { Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { HTTPResponse, LaunchOptions, Page } from 'puppeteer';
 import { z } from 'zod';
-import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js';
 import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
-import type { InterceptHandler } from './utils/puppeteer_request_interception.js';
-import type { BlockRequestsOptions, DirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, PuppeteerContextUtils, SaveSnapshotOptions } from './utils/puppeteer_utils.js';
+import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer_utils.js';
 export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
 export interface PuppeteerCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
 }
@@ -123,6 +121,9 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
  * @category Crawlers
  */
 export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, HTTPResponse, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+    /**
+     * @internal
+     */
     protected static optionsShape: {
         headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
         navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
@@ -181,6 +182,7 @@ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, Exte
         statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         id: z.ZodOptional<z.ZodString>;
     };
+    /** @internal */
     protected static optionsSchema: z.ZodObject<{
         headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
         navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
@@ -243,24 +245,7 @@ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, Exte
      * All `PuppeteerCrawler` parameters are passed via an options object.
      */
     constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
-// @ts-ignore optional peer dependency or compatibility with es2022
-    protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, HTTPResponse, Dictionary, Dictionary> & {
-        injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
-        injectJQuery: () => Promise<void>;
-        waitForSelector: (selector: string, timeoutMs?: number) => Promise<void>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<import("@crawlee/browser").CheerioAPI>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        enqueueLinksByClickingElements: (options: Omit<EnqueueLinksByClickingElementsOptions, "page" | "requestManager">) => Promise<import("@crawlee/types").BatchAddRequestsResult>;
-        blockRequests: (options?: BlockRequestsOptions) => Promise<void>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        compileScript: (scriptString: string, ctx?: Dictionary) => import("./utils/puppeteer_utils.js").CompiledScriptFunction;
-        addInterceptRequestHandler: (handler: InterceptHandler) => Promise<void>;
-        removeInterceptRequestHandler: (handler: InterceptHandler) => Promise<void>;
-        infiniteScroll: (options?: InfiniteScrollOptions) => Promise<void>;
-        saveSnapshot: (options?: SaveSnapshotOptions) => Promise<void>;
-        closeCookieModals: () => Promise<void>;
-    }>;
+    protected buildContextPipeline(): ContextPipeline<CrawlingContext, PuppeteerCrawlingContext>;
     private enhanceContext;
     protected navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<HTTPResponse | null>;
 }

package/internals/puppeteer-crawler.js

@@ -1,5 +1,6 @@
 import { assertBrowserPoolNotConfigured, BrowserCrawler, RequestState, Router } from '@crawlee/browser';
-import { parseArgument, serviceLocator } from '@crawlee/core';
+import { serviceLocator } from '@crawlee/core';
+import { parseArgument } from '@crawlee/utils/internal';
 import { z } from 'zod';
 import { puppeteerBrowserPool, remotePuppeteerBrowserPool } from './puppeteer-browser-pool.js';
 import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
@@ -69,12 +70,16 @@ import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
  * @category Crawlers
  */
 export class PuppeteerCrawler extends BrowserCrawler {
+    /**
+     * @internal
+     */
     static optionsShape = {
         ...BrowserCrawler.optionsShape,
         // Deliberately looser than the declared type: Puppeteer's own accepted string values have moved over
         // time (`'new'`/`'old'`, now `'shell'`), and the value is forwarded to it verbatim.
         headless: z.union([z.boolean(), z.string()]).optional(),
     };
+    /** @internal */
     static optionsSchema = z.strictObject(PuppeteerCrawler.optionsShape);
     /**
      * All `PuppeteerCrawler` parameters are passed via an options object.

package/internals/puppeteer-launcher.d.ts

@@ -68,6 +68,9 @@ export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPl
  */
 export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown> {
     readonly configuration: Configuration;
+    /**
+     * @internal
+     */
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
         launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
@@ -81,6 +84,7 @@ export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin,
         launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
         userAgent: z.ZodOptional<z.ZodString>;
     };
+    /** @internal */
     protected static optionsSchema: z.ZodObject<{
 // @ts-ignore optional peer dependency or compatibility with es2022
         launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;

package/internals/puppeteer-launcher.js

@@ -1,5 +1,6 @@
-import { BrowserLauncher, Configuration, parseArgument, schemas } from '@crawlee/browser';
+import { BrowserLauncher, Configuration } from '@crawlee/browser';
 import { PuppeteerPlugin } from '@crawlee/browser-pool';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { z } from 'zod';
 /**
  * `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
@@ -7,10 +8,14 @@ import { z } from 'zod';
  */
 export class PuppeteerLauncher extends BrowserLauncher {
     configuration;
+    /**
+     * @internal
+     */
     static optionsShape = {
         ...BrowserLauncher.optionsShape,
         launcher: schemas.anyObject.optional(),
     };
+    /** @internal */
     static optionsSchema = z.strictObject(PuppeteerLauncher.optionsShape);
     /**
      * All `PuppeteerLauncher` parameters are passed via an launchContext object.

package/internals/utils/puppeteer_request_interception.js

@@ -1,5 +1,6 @@
 import { EventEmitter } from 'node:events';
-import { parseArgument, schemas, serviceLocator } from '@crawlee/browser';
+import { serviceLocator } from '@crawlee/browser';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 const pageSchema = schemas.objectWithKeys(['goto', 'evaluate']);
 // We use weak maps here so that the content gets discarded after page gets closed.
 const pageInterceptRequestHandlersMap = new WeakMap(); // Maps page to an array of request interception handlers.

package/internals/utils/puppeteer_utils.d.ts

@@ -20,7 +20,7 @@
 import type { Request } from '@crawlee/browser';
 import { Configuration } from '@crawlee/browser';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils/internal';
+import type { CheerioAPI } from 'cheerio';
 import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { HTTPResponse, Page, ResponseForRequest } from 'puppeteer';
@@ -129,7 +129,7 @@ export declare function injectJQuery(page: Page, options?: {
  * @param page Puppeteer [`Page`](https://pptr.dev/api/puppeteer.page) object.
  * @param ignoreShadowRoots
  */
-export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;
+export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioAPI>;
 /**
  * Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
  * This is useful to speed up crawling of websites, since it reduces the amount
@@ -377,7 +377,7 @@ export interface PuppeteerContextUtils {
      * });
      * ```
      */
-    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
     /**
      * The function finds elements matching a specific CSS selector in a Puppeteer page,
      * clicks all those elements using a mouse move and a left mouse button click and intercepts

package/internals/utils/puppeteer_utils.js

@@ -20,8 +20,9 @@
 import { readFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import vm from 'node:vm';
-import { Configuration, KeyValueStore, parseArgument, schemas, serviceLocator, validators } from '@crawlee/browser';
+import { Configuration, KeyValueStore, serviceLocator, validators } from '@crawlee/browser';
 import { expandShadowRoots, sleep } from '@crawlee/utils';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/puppeteer",
-    "version": "4.0.0-beta.134",
+    "version": "4.0.0-beta.136",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -48,11 +48,11 @@
     },
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
-        "@crawlee/browser": "4.0.0-beta.134",
-        "@crawlee/browser-pool": "4.0.0-beta.134",
-        "@crawlee/core": "4.0.0-beta.134",
-        "@crawlee/types": "4.0.0-beta.134",
-        "@crawlee/utils": "4.0.0-beta.134",
+        "@crawlee/browser": "4.0.0-beta.136",
+        "@crawlee/browser-pool": "4.0.0-beta.136",
+        "@crawlee/core": "4.0.0-beta.136",
+        "@crawlee/types": "4.0.0-beta.136",
+        "@crawlee/utils": "4.0.0-beta.136",
         "cheerio": "^1.0.0",
         "devtools-protocol": "*",
         "jquery": "^3.7.1",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "0f269d5b24ec2829f3d44c19a35973fa20d95999"
+    "gitHead": "d425bb9841885b4b71b84ae6fb0bafbdc4a7e978"
 }