@crawlee/playwright 4.0.0-beta.121 → 4.0.0-beta.122
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/adaptive-playwright-crawler.js +5 -4
- package/internals/enqueue-links/click-elements.js +21 -21
- package/internals/playwright-crawler.d.ts +117 -98
- package/internals/playwright-crawler.js +9 -10
- package/internals/playwright-launcher.d.ts +24 -15
- package/internals/playwright-launcher.js +10 -8
- package/internals/utils/playwright-utils.js +48 -45
- package/package.json +10 -11
|
@@ -2,9 +2,9 @@ import { isDeepStrictEqual } from 'node:util';
|
|
|
2
2
|
import { BasicCrawler } from '@crawlee/basic';
|
|
3
3
|
import { extractUrlsFromPage } from '@crawlee/browser';
|
|
4
4
|
import { CheerioCrawler } from '@crawlee/cheerio';
|
|
5
|
-
import { createStorageTransaction, OwnedOrInjected, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/core';
|
|
5
|
+
import { createStorageTransaction, OwnedOrInjected, parseArgument, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/core';
|
|
6
6
|
import { extractUrlsFromCheerio } from '@crawlee/utils/internal';
|
|
7
|
-
import
|
|
7
|
+
import { z } from 'zod';
|
|
8
8
|
import { addTimeoutToPromise } from '@apify/timeout';
|
|
9
9
|
import { PlaywrightCrawler } from './playwright-crawler.js';
|
|
10
10
|
import { RenderingTypePredictor, } from './utils/rendering-type-prediction.js';
|
|
@@ -100,8 +100,9 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
100
100
|
#teardownHooks = [];
|
|
101
101
|
constructor(options = {}) {
|
|
102
102
|
const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statistics, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, contextPipelineBuilder, transactionalStorage, ...rest } = options;
|
|
103
|
-
// The user's value is replaced by `false` in the `super` call below — validate it separately
|
|
104
|
-
|
|
103
|
+
// The user's value is replaced by `false` in the `super` call below — validate it separately,
|
|
104
|
+
// wrapped in an object so the error still names the field.
|
|
105
|
+
parseArgument({ transactionalStorage }, z.object({ transactionalStorage: BasicCrawler.optionsShape.transactionalStorage }), 'AdaptivePlaywrightCrawlerOptions');
|
|
105
106
|
// Per-attempt buffering is load-bearing here: the handler runs up to twice per request and the
|
|
106
107
|
// losing attempt's writes must be discardable.
|
|
107
108
|
if (transactionalStorage === false) {
|
|
@@ -1,8 +1,24 @@
|
|
|
1
1
|
import { URL } from 'node:url';
|
|
2
|
-
import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, filterRequestOptionsByPatterns, Request as CrawleeRequest, serviceLocator, } from '@crawlee/browser';
|
|
3
|
-
import
|
|
2
|
+
import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, filterRequestOptionsByPatterns, parseArgument, urlPatternSchema, Request as CrawleeRequest, schemas, serviceLocator, } from '@crawlee/browser';
|
|
3
|
+
import { z } from 'zod';
|
|
4
4
|
const STARTING_Z_INDEX = 2147400000;
|
|
5
5
|
const getLog = () => serviceLocator.getChildLog('Playwright Click Elements');
|
|
6
|
+
const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
|
|
7
|
+
page: schemas.objectWithKeys(['goto', 'evaluate']),
|
|
8
|
+
requestManager: schemas.objectWithKeys(['fetchNextRequest', 'addRequestsBatched']),
|
|
9
|
+
selector: z.string(),
|
|
10
|
+
userData: schemas.anyObject.optional(),
|
|
11
|
+
clickOptions: schemas.anyObject.optional(),
|
|
12
|
+
include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
|
|
13
|
+
exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
|
|
14
|
+
transformRequestFunction: schemas.anyFunction.optional(),
|
|
15
|
+
waitForPageIdleSecs: schemas.anyNumber.default(1),
|
|
16
|
+
maxWaitForPageIdleSecs: schemas.anyNumber.default(5),
|
|
17
|
+
label: z.string().optional(),
|
|
18
|
+
forefront: z.boolean().optional(),
|
|
19
|
+
skipNavigation: z.boolean().optional(),
|
|
20
|
+
onSkippedRequest: schemas.anyFunction.optional(),
|
|
21
|
+
});
|
|
6
22
|
/**
|
|
7
23
|
* The function finds elements matching a specific CSS selector in a Playwright page,
|
|
8
24
|
* clicks all those elements using a mouse move and a left mouse button click and intercepts
|
|
@@ -44,24 +60,8 @@ const getLog = () => serviceLocator.getChildLog('Playwright Click Elements');
|
|
|
44
60
|
* @returns Promise that resolves to {@link BatchAddRequestsResult} object.
|
|
45
61
|
*/
|
|
46
62
|
export async function enqueueLinksByClickingElements(options) {
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
page: ow.object.hasKeys('goto', 'evaluate'),
|
|
50
|
-
requestManager: ow.object.hasKeys('fetchNextRequest', 'addRequestsBatched'),
|
|
51
|
-
selector: ow.string,
|
|
52
|
-
userData: ow.optional.object,
|
|
53
|
-
clickOptions: ow.optional.object,
|
|
54
|
-
include: ow.optional.array.ofType(urlPatternValidator),
|
|
55
|
-
exclude: ow.optional.array.ofType(urlPatternValidator),
|
|
56
|
-
transformRequestFunction: ow.optional.function,
|
|
57
|
-
waitForPageIdleSecs: ow.optional.number,
|
|
58
|
-
maxWaitForPageIdleSecs: ow.optional.number,
|
|
59
|
-
label: ow.optional.string,
|
|
60
|
-
forefront: ow.optional.boolean,
|
|
61
|
-
skipNavigation: ow.optional.boolean,
|
|
62
|
-
onSkippedRequest: ow.optional.function,
|
|
63
|
-
}));
|
|
64
|
-
const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, onSkippedRequest, } = options;
|
|
63
|
+
const parsedOptions = parseArgument(options, enqueueLinksByClickingElementsOptionsSchema, 'EnqueueLinksByClickingElementsOptions');
|
|
64
|
+
const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
|
|
65
65
|
const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
|
|
66
66
|
const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
|
|
67
67
|
const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
|
|
@@ -73,7 +73,7 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
73
73
|
maxWaitForPageIdleMillis,
|
|
74
74
|
clickOptions,
|
|
75
75
|
});
|
|
76
|
-
const requestOptions = createRequestOptions(interceptedRequests,
|
|
76
|
+
const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
|
|
77
77
|
const skippedByFilters = [];
|
|
78
78
|
let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (url) => skippedByFilters.push(url));
|
|
79
79
|
if (onSkippedRequest && skippedByFilters.length > 0) {
|
|
@@ -4,6 +4,7 @@ import type { PlaywrightPlugin } from '@crawlee/browser-pool';
|
|
|
4
4
|
import type { Dictionary } from '@crawlee/types';
|
|
5
5
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
6
6
|
import type { Download, LaunchOptions, Page, Response } from 'playwright';
|
|
7
|
+
import { z } from 'zod';
|
|
7
8
|
import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js';
|
|
8
9
|
import type { PlaywrightLaunchContext } from './playwright-launcher.js';
|
|
9
10
|
import type { BlockRequestsOptions, DirectNavigationOptions, HandleCloudflareChallengeOptions, InfiniteScrollOptions, InjectFileOptions, PlaywrightContextUtils, SaveSnapshotOptions } from './utils/playwright-utils.js';
|
|
@@ -145,105 +146,123 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
|
|
|
145
146
|
browserPlugins: [PlaywrightPlugin];
|
|
146
147
|
}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes> {
|
|
147
148
|
protected static optionsShape: {
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
206
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
207
|
-
statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
208
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
209
|
-
statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
210
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
211
|
-
additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
212
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
213
|
-
ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
214
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
215
|
-
blockedStatusCodes: import("ow").ArrayPredicate<number>;
|
|
216
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
217
|
-
retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
218
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
219
|
-
respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
|
|
220
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
221
|
-
transactionalStorage: import("ow").BasePredicate<boolean | Partial<import("@crawlee/browser").StorageWritePolicy> | undefined>;
|
|
222
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
223
|
-
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
224
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
225
|
-
httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
226
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
227
|
-
configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
228
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
229
|
-
storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
230
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
231
|
-
eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
232
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
233
|
-
logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
234
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
235
|
-
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
236
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
237
|
-
maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
238
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
239
|
-
maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
240
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
241
|
-
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
242
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
243
|
-
statistics: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
244
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
245
|
-
id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
149
|
+
browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
150
|
+
launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
151
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
152
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
153
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
154
|
+
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
155
|
+
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
156
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
157
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
158
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
159
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
160
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
161
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
162
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
163
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
164
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
165
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
166
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
167
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
168
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
169
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
170
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
171
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
172
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
173
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
174
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
175
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
176
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
177
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
178
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
179
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
180
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
181
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
182
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
183
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
184
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
185
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
186
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
187
|
+
deferred: "deferred";
|
|
188
|
+
writeThrough: "writeThrough";
|
|
189
|
+
}>>;
|
|
190
|
+
}, z.core.$strict>]>>;
|
|
191
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
192
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
193
|
+
httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
|
|
194
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
195
|
+
configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").Configuration, import("@crawlee/browser").Configuration>>;
|
|
196
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
197
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
198
|
+
eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").EventManager, import("@crawlee/browser").EventManager>>;
|
|
199
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
200
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
201
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
202
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
203
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
204
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
205
|
+
id: z.ZodOptional<z.ZodString>;
|
|
246
206
|
};
|
|
207
|
+
protected static optionsSchema: z.ZodObject<{
|
|
208
|
+
browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
209
|
+
launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
210
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
211
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
212
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
213
|
+
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
214
|
+
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
215
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
216
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
217
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
218
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
219
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
220
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
221
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
222
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
223
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
224
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
225
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
226
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
227
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
228
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
229
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
230
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
231
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
232
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
233
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
234
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
235
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
236
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
237
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
238
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
239
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
240
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
241
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
242
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
243
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
244
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
245
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
246
|
+
deferred: "deferred";
|
|
247
|
+
writeThrough: "writeThrough";
|
|
248
|
+
}>>;
|
|
249
|
+
}, z.core.$strict>]>>;
|
|
250
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
251
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
252
|
+
httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
|
|
253
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
254
|
+
configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").Configuration, import("@crawlee/browser").Configuration>>;
|
|
255
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
256
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
257
|
+
eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").EventManager, import("@crawlee/browser").EventManager>>;
|
|
258
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
259
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
260
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
261
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
262
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
263
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
264
|
+
id: z.ZodOptional<z.ZodString>;
|
|
265
|
+
}, z.core.$strict>;
|
|
247
266
|
/**
|
|
248
267
|
* All `PlaywrightCrawler` parameters are passed via an options object.
|
|
249
268
|
*/
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { BrowserCrawler, RequestState, Router, serviceLocator } from '@crawlee/browser';
|
|
2
|
-
import
|
|
1
|
+
import { BrowserCrawler, parseArgument, RequestState, Router, schemas, serviceLocator } from '@crawlee/browser';
|
|
2
|
+
import { z } from 'zod';
|
|
3
3
|
import { PlaywrightLauncher } from './playwright-launcher.js';
|
|
4
4
|
import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
|
|
5
5
|
/**
|
|
@@ -70,19 +70,18 @@ import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
|
|
|
70
70
|
export class PlaywrightCrawler extends BrowserCrawler {
|
|
71
71
|
static optionsShape = {
|
|
72
72
|
...BrowserCrawler.optionsShape,
|
|
73
|
-
browserPoolOptions:
|
|
74
|
-
launcher:
|
|
75
|
-
ignoreIframes: ow.optional.boolean,
|
|
76
|
-
ignoreShadowRoots: ow.optional.boolean,
|
|
73
|
+
browserPoolOptions: schemas.anyObject.optional(),
|
|
74
|
+
launcher: schemas.anyObject.optional(),
|
|
77
75
|
};
|
|
76
|
+
static optionsSchema = z.strictObject(PlaywrightCrawler.optionsShape);
|
|
78
77
|
/**
|
|
79
78
|
* All `PlaywrightCrawler` parameters are passed via an options object.
|
|
80
79
|
*/
|
|
81
80
|
constructor(options = {}) {
|
|
82
|
-
|
|
83
|
-
const { launchContext
|
|
81
|
+
const parsedOptions = parseArgument(options, PlaywrightCrawler.optionsSchema, 'PlaywrightCrawlerOptions');
|
|
82
|
+
const { launchContext, headless, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
|
|
84
83
|
const browserPoolOptions = {
|
|
85
|
-
...
|
|
84
|
+
...parsedOptions.browserPoolOptions,
|
|
86
85
|
};
|
|
87
86
|
if (launchContext.proxyUrl) {
|
|
88
87
|
throw new Error('PlaywrightCrawlerOptions.launchContext.proxyUrl is not allowed in PlaywrightCrawler.' +
|
|
@@ -97,7 +96,7 @@ export class PlaywrightCrawler extends BrowserCrawler {
|
|
|
97
96
|
launchContext.launchOptions ??= {};
|
|
98
97
|
launchContext.launchOptions.headless = headless;
|
|
99
98
|
}
|
|
100
|
-
const playwrightLauncher = new PlaywrightLauncher(launchContext,
|
|
99
|
+
const playwrightLauncher = new PlaywrightLauncher(launchContext, parsedOptions.configuration);
|
|
101
100
|
browserPoolOptions.browserPlugins = [playwrightLauncher.createBrowserPlugin()];
|
|
102
101
|
super({
|
|
103
102
|
...browserCrawlerOptions,
|
|
@@ -3,6 +3,7 @@ import { BrowserLauncher, Configuration } from '@crawlee/browser';
|
|
|
3
3
|
import { PlaywrightPlugin } from '@crawlee/browser-pool';
|
|
4
4
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
5
5
|
import type { Browser, BrowserType, LaunchOptions } from 'playwright';
|
|
6
|
+
import { z } from 'zod';
|
|
6
7
|
/**
|
|
7
8
|
* Apify extends the launch options of Playwright.
|
|
8
9
|
* You can use any of the Playwright compatible
|
|
@@ -73,26 +74,34 @@ export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin
|
|
|
73
74
|
readonly configuration: Configuration;
|
|
74
75
|
protected static optionsShape: {
|
|
75
76
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
76
|
-
launcher: import("
|
|
77
|
+
launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
77
78
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
78
|
-
launchContextOptions: import("
|
|
79
|
+
launchContextOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
80
|
+
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
81
|
+
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
82
|
+
useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
|
|
83
|
+
browserPerProxy: z.ZodOptional<z.ZodBoolean>;
|
|
84
|
+
ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
|
|
85
|
+
userDataDir: z.ZodOptional<z.ZodString>;
|
|
79
86
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
85
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
86
|
-
browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
87
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
88
|
-
ignoreProxyCertificate: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
87
|
+
launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
88
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
89
|
+
};
|
|
90
|
+
protected static optionsSchema: z.ZodObject<{
|
|
89
91
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
90
|
-
|
|
92
|
+
launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
91
93
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
92
|
-
|
|
94
|
+
launchContextOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
95
|
+
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
96
|
+
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
97
|
+
useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
|
|
98
|
+
browserPerProxy: z.ZodOptional<z.ZodBoolean>;
|
|
99
|
+
ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
|
|
100
|
+
userDataDir: z.ZodOptional<z.ZodString>;
|
|
93
101
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
94
|
-
|
|
95
|
-
|
|
102
|
+
launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
103
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
104
|
+
}, z.core.$strict>;
|
|
96
105
|
/**
|
|
97
106
|
* All `PlaywrightLauncher` parameters are passed via this launchContext object.
|
|
98
107
|
*/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { BrowserLauncher, Configuration } from '@crawlee/browser';
|
|
1
|
+
import { BrowserLauncher, Configuration, parseArgument, schemas } from '@crawlee/browser';
|
|
2
2
|
import { PlaywrightPlugin } from '@crawlee/browser-pool';
|
|
3
|
-
import
|
|
3
|
+
import { z } from 'zod';
|
|
4
4
|
/**
|
|
5
5
|
* `PlaywrightLauncher` is based on the `BrowserLauncher`. It launches `playwright` browser instance.
|
|
6
6
|
* @ignore
|
|
@@ -9,21 +9,23 @@ export class PlaywrightLauncher extends BrowserLauncher {
|
|
|
9
9
|
configuration;
|
|
10
10
|
static optionsShape = {
|
|
11
11
|
...BrowserLauncher.optionsShape,
|
|
12
|
-
launcher
|
|
13
|
-
|
|
12
|
+
// Passthrough schemas — the launcher module object must keep its prototype through parsing.
|
|
13
|
+
launcher: schemas.anyObject.optional(),
|
|
14
|
+
launchContextOptions: schemas.anyObject.optional(),
|
|
14
15
|
};
|
|
16
|
+
static optionsSchema = z.strictObject(PlaywrightLauncher.optionsShape);
|
|
15
17
|
/**
|
|
16
18
|
* All `PlaywrightLauncher` parameters are passed via this launchContext object.
|
|
17
19
|
*/
|
|
18
20
|
constructor(launchContext = {}, configuration = Configuration.getGlobalConfiguration()) {
|
|
19
|
-
|
|
20
|
-
const { launcher = BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium, } =
|
|
21
|
-
const { launchOptions = {}, ...rest } =
|
|
21
|
+
const parsedContext = parseArgument(launchContext, PlaywrightLauncher.optionsSchema, 'PlaywrightLaunchContext');
|
|
22
|
+
const { launcher = BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium, } = parsedContext;
|
|
23
|
+
const { launchOptions = {}, ...rest } = parsedContext;
|
|
22
24
|
super({
|
|
23
25
|
...rest,
|
|
24
26
|
launchOptions: {
|
|
25
27
|
...launchOptions,
|
|
26
|
-
executablePath: getDefaultExecutablePath(
|
|
28
|
+
executablePath: getDefaultExecutablePath(parsedContext, configuration),
|
|
27
29
|
},
|
|
28
30
|
launcher,
|
|
29
31
|
}, configuration);
|
|
@@ -20,9 +20,9 @@
|
|
|
20
20
|
import { readFile } from 'node:fs/promises';
|
|
21
21
|
import { createRequire } from 'node:module';
|
|
22
22
|
import vm from 'node:vm';
|
|
23
|
-
import { Configuration, KeyValueStore, serviceLocator, SessionError, validators } from '@crawlee/browser';
|
|
23
|
+
import { Configuration, KeyValueStore, parseArgument, schemas, serviceLocator, SessionError, validators, } from '@crawlee/browser';
|
|
24
24
|
import { expandShadowRoots, sleep } from '@crawlee/utils';
|
|
25
|
-
import
|
|
25
|
+
import { z } from 'zod';
|
|
26
26
|
import { LruCache } from '@apify/datastructures';
|
|
27
27
|
import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
|
|
28
28
|
import { RenderingTypePredictor } from './rendering-type-prediction.js';
|
|
@@ -31,6 +31,36 @@ const require = createRequire(import.meta.url);
|
|
|
31
31
|
const jqueryPath = require.resolve('jquery');
|
|
32
32
|
const MAX_INJECT_FILE_CACHE_SIZE = 10;
|
|
33
33
|
const DEFAULT_BLOCK_REQUEST_URL_PATTERNS = ['.css', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip'];
|
|
34
|
+
const filePathSchema = z.string();
|
|
35
|
+
const injectFileOptionsSchema = z.strictObject({
|
|
36
|
+
surviveNavigations: z.boolean().optional(),
|
|
37
|
+
});
|
|
38
|
+
const gotoExtendedRequestSchema = z.looseObject({
|
|
39
|
+
url: z.url(),
|
|
40
|
+
method: z.string().optional(),
|
|
41
|
+
headers: schemas.anyObject.optional(),
|
|
42
|
+
payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
|
|
43
|
+
});
|
|
44
|
+
const blockRequestsOptionsSchema = z.strictObject({
|
|
45
|
+
urlPatterns: schemas.arrayOf(z.string(), 'strings').default(DEFAULT_BLOCK_REQUEST_URL_PATTERNS),
|
|
46
|
+
extraUrlPatterns: schemas.arrayOf(z.string(), 'strings').default(() => []),
|
|
47
|
+
});
|
|
48
|
+
const infiniteScrollOptionsSchema = z.strictObject({
|
|
49
|
+
timeoutSecs: schemas.anyNumber.default(0),
|
|
50
|
+
maxScrollHeight: schemas.anyNumber.default(0),
|
|
51
|
+
waitForSecs: schemas.anyNumber.default(4),
|
|
52
|
+
scrollDownAndUp: z.boolean().default(false),
|
|
53
|
+
buttonSelector: z.string().optional(),
|
|
54
|
+
stopScrollCallback: schemas.anyFunction.optional(),
|
|
55
|
+
});
|
|
56
|
+
const saveSnapshotOptionsSchema = z.strictObject({
|
|
57
|
+
key: z.string().min(1).default('SNAPSHOT'),
|
|
58
|
+
screenshotQuality: schemas.anyNumber.default(50),
|
|
59
|
+
saveScreenshot: z.boolean().default(true),
|
|
60
|
+
saveHtml: z.boolean().default(true),
|
|
61
|
+
keyValueStoreName: z.string().optional(),
|
|
62
|
+
configuration: schemas.anyObject.optional(),
|
|
63
|
+
});
|
|
34
64
|
/**
|
|
35
65
|
* Cache contents of previously injected files to limit file system access.
|
|
36
66
|
*/
|
|
@@ -47,18 +77,16 @@ const injectedFilesCache = new LruCache({ maxLength: MAX_INJECT_FILE_CACHE_SIZE
|
|
|
47
77
|
* @param [options]
|
|
48
78
|
*/
|
|
49
79
|
export async function injectFile(page, filePath, options = {}) {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
surviveNavigations: ow.optional.boolean,
|
|
54
|
-
}));
|
|
80
|
+
parseArgument(page, validators.browserPage);
|
|
81
|
+
parseArgument(filePath, filePathSchema);
|
|
82
|
+
const { surviveNavigations } = parseArgument(options, injectFileOptionsSchema);
|
|
55
83
|
let contents = injectedFilesCache.get(filePath);
|
|
56
84
|
if (!contents) {
|
|
57
85
|
contents = await readFile(filePath, 'utf8');
|
|
58
86
|
injectedFilesCache.add(filePath, contents);
|
|
59
87
|
}
|
|
60
88
|
const evalP = page.evaluate(contents);
|
|
61
|
-
if (
|
|
89
|
+
if (surviveNavigations) {
|
|
62
90
|
page.on('framenavigated', async () => page
|
|
63
91
|
.evaluate(contents)
|
|
64
92
|
.catch((error) => getLog().warning('An error occurred during the script injection!', { error })));
|
|
@@ -92,7 +120,7 @@ export async function injectFile(page, filePath, options = {}) {
|
|
|
92
120
|
* @param [options.surviveNavigations] Opt-out option to disable the JQuery reinjection after navigation.
|
|
93
121
|
*/
|
|
94
122
|
export async function injectJQuery(page, options) {
|
|
95
|
-
|
|
123
|
+
parseArgument(page, validators.browserPage);
|
|
96
124
|
return injectFile(page, jqueryPath, { surviveNavigations: options?.surviveNavigations ?? true });
|
|
97
125
|
}
|
|
98
126
|
/**
|
|
@@ -108,14 +136,9 @@ export async function injectJQuery(page, options) {
|
|
|
108
136
|
* @param [gotoOptions] Custom options for `page.goto()`.
|
|
109
137
|
*/
|
|
110
138
|
export async function gotoExtended(page, request, gotoOptions = {}) {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
method: ow.optional.string,
|
|
115
|
-
headers: ow.optional.object,
|
|
116
|
-
payload: ow.optional.any(ow.string, ow.uint8Array),
|
|
117
|
-
}));
|
|
118
|
-
ow(gotoOptions, ow.object);
|
|
139
|
+
parseArgument(page, validators.browserPage);
|
|
140
|
+
parseArgument(request, gotoExtendedRequestSchema);
|
|
141
|
+
parseArgument(gotoOptions, schemas.anyObject);
|
|
119
142
|
const { url, method, headers, payload } = request;
|
|
120
143
|
const isEmpty = (o) => !o || Object.keys(o).length === 0;
|
|
121
144
|
if (method !== 'GET' || payload) {
|
|
@@ -200,12 +223,8 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
|
|
|
200
223
|
* @param [options]
|
|
201
224
|
*/
|
|
202
225
|
export async function blockRequests(page, options = {}) {
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
urlPatterns: ow.optional.array.ofType(ow.string),
|
|
206
|
-
extraUrlPatterns: ow.optional.array.ofType(ow.string),
|
|
207
|
-
}));
|
|
208
|
-
const { urlPatterns = DEFAULT_BLOCK_REQUEST_URL_PATTERNS, extraUrlPatterns = [] } = options;
|
|
226
|
+
parseArgument(page, validators.browserPage);
|
|
227
|
+
const { urlPatterns, extraUrlPatterns } = parseArgument(options, blockRequestsOptionsSchema);
|
|
209
228
|
const patternsToBlock = [...urlPatterns, ...extraUrlPatterns];
|
|
210
229
|
try {
|
|
211
230
|
const client = await page.context().newCDPSession(page);
|
|
@@ -263,16 +282,8 @@ export function compileScript(scriptString, context = Object.create(null)) {
|
|
|
263
282
|
* @param [options]
|
|
264
283
|
*/
|
|
265
284
|
export async function infiniteScroll(page, options = {}) {
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
timeoutSecs: ow.optional.number,
|
|
269
|
-
maxScrollHeight: ow.optional.number,
|
|
270
|
-
waitForSecs: ow.optional.number,
|
|
271
|
-
scrollDownAndUp: ow.optional.boolean,
|
|
272
|
-
buttonSelector: ow.optional.string,
|
|
273
|
-
stopScrollCallback: ow.optional.function,
|
|
274
|
-
}));
|
|
275
|
-
const { timeoutSecs = 0, maxScrollHeight = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback, } = options;
|
|
285
|
+
parseArgument(page, validators.browserPage);
|
|
286
|
+
const { timeoutSecs, maxScrollHeight, waitForSecs, scrollDownAndUp, buttonSelector, stopScrollCallback } = parseArgument(options, infiniteScrollOptionsSchema);
|
|
276
287
|
let finished;
|
|
277
288
|
const startTime = Date.now();
|
|
278
289
|
const CHECK_INTERVAL_MILLIS = 1000;
|
|
@@ -349,16 +360,8 @@ export async function infiniteScroll(page, options = {}) {
|
|
|
349
360
|
* @param [options]
|
|
350
361
|
*/
|
|
351
362
|
export async function saveSnapshot(page, options = {}) {
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
key: ow.optional.string.nonEmpty,
|
|
355
|
-
screenshotQuality: ow.optional.number,
|
|
356
|
-
saveScreenshot: ow.optional.boolean,
|
|
357
|
-
saveHtml: ow.optional.boolean,
|
|
358
|
-
keyValueStoreName: ow.optional.string,
|
|
359
|
-
configuration: ow.optional.object,
|
|
360
|
-
}));
|
|
361
|
-
const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, configuration, } = options;
|
|
363
|
+
parseArgument(page, validators.browserPage);
|
|
364
|
+
const { key, screenshotQuality, saveScreenshot, saveHtml, keyValueStoreName, configuration } = parseArgument(options, saveSnapshotOptionsSchema);
|
|
362
365
|
try {
|
|
363
366
|
const store = await KeyValueStore.open(keyValueStoreName ? { name: keyValueStoreName } : null, {
|
|
364
367
|
configuration: configuration ?? Configuration.getGlobalConfiguration(),
|
|
@@ -396,7 +399,7 @@ export async function saveSnapshot(page, options = {}) {
|
|
|
396
399
|
* @param ignoreShadowRoots
|
|
397
400
|
*/
|
|
398
401
|
export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIframes = false) {
|
|
399
|
-
|
|
402
|
+
parseArgument(page, validators.browserPage);
|
|
400
403
|
const html = ignoreShadowRoots
|
|
401
404
|
? null
|
|
402
405
|
: (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
|
|
@@ -457,7 +460,7 @@ ${error.message}
|
|
|
457
460
|
return idcacPlaywright;
|
|
458
461
|
}
|
|
459
462
|
export async function closeCookieModals(page) {
|
|
460
|
-
|
|
463
|
+
parseArgument(page, validators.browserPage);
|
|
461
464
|
const idcac = await getIdcacPlaywright();
|
|
462
465
|
if (idcac?.getInjectableScript()) {
|
|
463
466
|
await page.evaluate(idcac.getInjectableScript());
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/playwright",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.122",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -49,22 +49,21 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^2.0.3",
|
|
51
51
|
"@apify/timeout": "^0.4.4",
|
|
52
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
53
|
-
"@crawlee/browser": "4.0.0-beta.
|
|
54
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
55
|
-
"@crawlee/cheerio": "4.0.0-beta.
|
|
56
|
-
"@crawlee/core": "4.0.0-beta.
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
52
|
+
"@crawlee/basic": "4.0.0-beta.122",
|
|
53
|
+
"@crawlee/browser": "4.0.0-beta.122",
|
|
54
|
+
"@crawlee/browser-pool": "4.0.0-beta.122",
|
|
55
|
+
"@crawlee/cheerio": "4.0.0-beta.122",
|
|
56
|
+
"@crawlee/core": "4.0.0-beta.122",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.122",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.122",
|
|
59
59
|
"cheerio": "^1.0.0",
|
|
60
60
|
"jquery": "^3.7.1",
|
|
61
61
|
"ml-logistic-regression": "^2.0.0",
|
|
62
62
|
"ml-matrix": "^6.12.1",
|
|
63
|
-
"ow": "^2.0.0",
|
|
64
63
|
"string-comparison": "^1.3.0",
|
|
65
64
|
"tslib": "^2.8.1",
|
|
66
65
|
"type-fest": "^4.0.0",
|
|
67
|
-
"zod": "^4.
|
|
66
|
+
"zod": "^4.4.3"
|
|
68
67
|
},
|
|
69
68
|
"peerDependencies": {
|
|
70
69
|
"idcac-playwright": "^0.2.0",
|
|
@@ -85,5 +84,5 @@
|
|
|
85
84
|
}
|
|
86
85
|
}
|
|
87
86
|
},
|
|
88
|
-
"gitHead": "
|
|
87
|
+
"gitHead": "2c3e87fefdb9e1fca4c144f03167d8524cfdc2e5"
|
|
89
88
|
}
|