@crawlee/browser 4.0.0-beta.134 → 4.0.0-beta.135
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -254,6 +254,9 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
|
|
|
254
254
|
launchContext: BrowserLaunchContext<LaunchOptions, unknown>;
|
|
255
255
|
protected readonly ignoreShadowRoots: boolean;
|
|
256
256
|
protected readonly ignoreIframes: boolean;
|
|
257
|
+
/**
|
|
258
|
+
* @internal
|
|
259
|
+
*/
|
|
257
260
|
protected static optionsShape: {
|
|
258
261
|
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
259
262
|
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
@@ -311,6 +314,7 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
|
|
|
311
314
|
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
312
315
|
id: z.ZodOptional<z.ZodString>;
|
|
313
316
|
};
|
|
317
|
+
/** @internal */
|
|
314
318
|
protected static optionsSchema: z.ZodObject<{
|
|
315
319
|
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
316
320
|
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, OwnedOrInjected,
|
|
2
|
-
import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
|
|
1
|
+
import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, validators, } from '@crawlee/basic';
|
|
2
|
+
import { CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
|
|
3
3
|
import { sleep } from '@crawlee/utils';
|
|
4
4
|
import { z } from 'zod';
|
|
5
5
|
import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
|
|
@@ -87,6 +87,9 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
87
87
|
#preNavigationHooks;
|
|
88
88
|
#postNavigationHooks;
|
|
89
89
|
#saveResponseCookies;
|
|
90
|
+
/**
|
|
91
|
+
* @internal
|
|
92
|
+
*/
|
|
90
93
|
static optionsShape = {
|
|
91
94
|
...BasicCrawler.optionsShape,
|
|
92
95
|
navigationTimeoutSecs: schemas.anyNumber
|
|
@@ -103,6 +106,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
103
106
|
ignoreIframes: z.boolean().default(false),
|
|
104
107
|
ignoreShadowRoots: z.boolean().default(false),
|
|
105
108
|
};
|
|
109
|
+
/** @internal */
|
|
106
110
|
static optionsSchema = z.strictObject(BrowserCrawler.optionsShape);
|
|
107
111
|
/**
|
|
108
112
|
* All `BrowserCrawler` parameters are passed via an options object.
|
|
@@ -92,6 +92,9 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
|
|
|
92
92
|
otherLaunchContextProps: Dictionary;
|
|
93
93
|
Plugin: T;
|
|
94
94
|
userAgent?: string;
|
|
95
|
+
/**
|
|
96
|
+
* @internal
|
|
97
|
+
*/
|
|
95
98
|
protected static optionsShape: {
|
|
96
99
|
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
97
100
|
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
@@ -102,6 +105,7 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
|
|
|
102
105
|
launchOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
103
106
|
userAgent: z.ZodOptional<z.ZodString>;
|
|
104
107
|
};
|
|
108
|
+
/** @internal */
|
|
105
109
|
protected static optionsSchema: z.ZodObject<{
|
|
106
110
|
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
107
111
|
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import { createRequire } from 'node:module';
|
|
3
3
|
import os from 'node:os';
|
|
4
|
-
import { Configuration,
|
|
4
|
+
import { Configuration, serviceLocator } from '@crawlee/basic';
|
|
5
5
|
import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
|
|
6
|
+
import { schemas } from '@crawlee/utils/internal';
|
|
6
7
|
import { z } from 'zod';
|
|
7
8
|
const DEFAULT_VIEWPORT = {
|
|
8
9
|
width: 1366,
|
|
@@ -23,6 +24,9 @@ export class BrowserLauncher {
|
|
|
23
24
|
// to be provided by child classes;
|
|
24
25
|
Plugin;
|
|
25
26
|
userAgent;
|
|
27
|
+
/**
|
|
28
|
+
* @internal
|
|
29
|
+
*/
|
|
26
30
|
static optionsShape = {
|
|
27
31
|
proxyUrl: z.url().optional(),
|
|
28
32
|
useChrome: z.boolean().optional(),
|
|
@@ -33,6 +37,7 @@ export class BrowserLauncher {
|
|
|
33
37
|
launchOptions: schemas.anyObject.optional(),
|
|
34
38
|
userAgent: z.string().optional(),
|
|
35
39
|
};
|
|
40
|
+
/** @internal */
|
|
36
41
|
static optionsSchema = z.strictObject(BrowserLauncher.optionsShape);
|
|
37
42
|
static requireLauncherOrThrow(launcher, apifyImageName) {
|
|
38
43
|
try {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.135",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -48,10 +48,10 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^0.4.4",
|
|
51
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
53
|
-
"@crawlee/types": "4.0.0-beta.
|
|
54
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/basic": "4.0.0-beta.135",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-beta.135",
|
|
53
|
+
"@crawlee/types": "4.0.0-beta.135",
|
|
54
|
+
"@crawlee/utils": "4.0.0-beta.135",
|
|
55
55
|
"tslib": "^2.8.1",
|
|
56
56
|
"type-fest": "^4.41.0",
|
|
57
57
|
"zod": "^4.4.3"
|
|
@@ -75,5 +75,5 @@
|
|
|
75
75
|
}
|
|
76
76
|
}
|
|
77
77
|
},
|
|
78
|
-
"gitHead": "
|
|
78
|
+
"gitHead": "ca0325880bbcc753d97506796e9bda3b35bca201"
|
|
79
79
|
}
|