@crawlee/playwright 4.0.0-beta.121 → 4.0.0-beta.123
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/adaptive-playwright-crawler.d.ts +1 -0
- package/internals/adaptive-playwright-crawler.js +22 -22
- package/internals/enqueue-links/click-elements.js +21 -21
- package/internals/playwright-crawler.d.ts +117 -98
- package/internals/playwright-crawler.js +9 -10
- package/internals/playwright-launcher.d.ts +24 -15
- package/internals/playwright-launcher.js +10 -8
- package/internals/utils/playwright-utils.js +48 -45
- package/package.json +10 -11
|
@@ -190,6 +190,7 @@ export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<nev
|
|
|
190
190
|
readonly querySelectorAll: AdaptivePlaywrightCrawlerContext["querySelectorAll"];
|
|
191
191
|
readonly waitForSelector: AdaptivePlaywrightCrawlerContext["waitForSelector"];
|
|
192
192
|
readonly parseWithCheerio: AdaptivePlaywrightCrawlerContext["parseWithCheerio"];
|
|
193
|
+
readonly enqueueLinks: AdaptivePlaywrightCrawlerContext["enqueueLinks"];
|
|
193
194
|
}>;
|
|
194
195
|
private adaptCheerioContext;
|
|
195
196
|
private adaptPlaywrightContext;
|
|
@@ -2,9 +2,9 @@ import { isDeepStrictEqual } from 'node:util';
|
|
|
2
2
|
import { BasicCrawler } from '@crawlee/basic';
|
|
3
3
|
import { extractUrlsFromPage } from '@crawlee/browser';
|
|
4
4
|
import { CheerioCrawler } from '@crawlee/cheerio';
|
|
5
|
-
import { createStorageTransaction, OwnedOrInjected, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/core';
|
|
5
|
+
import { createStorageTransaction, EnqueueStrategy, OwnedOrInjected, parseArgument, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/core';
|
|
6
6
|
import { extractUrlsFromCheerio } from '@crawlee/utils/internal';
|
|
7
|
-
import
|
|
7
|
+
import { z } from 'zod';
|
|
8
8
|
import { addTimeoutToPromise } from '@apify/timeout';
|
|
9
9
|
import { PlaywrightCrawler } from './playwright-crawler.js';
|
|
10
10
|
import { RenderingTypePredictor, } from './utils/rendering-type-prediction.js';
|
|
@@ -100,8 +100,9 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
100
100
|
#teardownHooks = [];
|
|
101
101
|
constructor(options = {}) {
|
|
102
102
|
const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statistics, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, contextPipelineBuilder, transactionalStorage, ...rest } = options;
|
|
103
|
-
// The user's value is replaced by `false` in the `super` call below — validate it separately
|
|
104
|
-
|
|
103
|
+
// The user's value is replaced by `false` in the `super` call below — validate it separately,
|
|
104
|
+
// wrapped in an object so the error still names the field.
|
|
105
|
+
parseArgument({ transactionalStorage }, z.object({ transactionalStorage: BasicCrawler.optionsShape.transactionalStorage }), 'AdaptivePlaywrightCrawlerOptions');
|
|
105
106
|
// Per-attempt buffering is load-bearing here: the handler runs up to twice per request and the
|
|
106
107
|
// losing attempt's writes must be discardable.
|
|
107
108
|
if (transactionalStorage === false) {
|
|
@@ -217,6 +218,9 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
217
218
|
get parseWithCheerio() {
|
|
218
219
|
throw new Error(errorMessage('parseWithCheerio'));
|
|
219
220
|
},
|
|
221
|
+
get enqueueLinks() {
|
|
222
|
+
throw new Error(errorMessage('enqueueLinks'));
|
|
223
|
+
},
|
|
220
224
|
}),
|
|
221
225
|
});
|
|
222
226
|
}
|
|
@@ -232,9 +236,8 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
232
236
|
return cheerioContext.$(selector);
|
|
233
237
|
},
|
|
234
238
|
enqueueLinks: async (options = {}) => {
|
|
235
|
-
const urls = options.
|
|
236
|
-
|
|
237
|
-
return (await this.enqueueLinks({ ...options, urls }, cheerioContext.request));
|
|
239
|
+
const urls = extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
|
|
240
|
+
return (await this.enqueueLinks(urls, options, cheerioContext.request));
|
|
238
241
|
},
|
|
239
242
|
response: cheerioContext.response,
|
|
240
243
|
};
|
|
@@ -262,19 +265,11 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
262
265
|
},
|
|
263
266
|
enqueueLinks: async (options = {}, timeoutMs = 5000) => {
|
|
264
267
|
// TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplication
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
urls =
|
|
271
|
-
options.urls ??
|
|
272
|
-
(await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl));
|
|
273
|
-
}
|
|
274
|
-
else {
|
|
275
|
-
urls = options.urls;
|
|
276
|
-
}
|
|
277
|
-
return (await this.enqueueLinks({ ...options, urls }, playwrightContext.request));
|
|
268
|
+
const selector = options.selector ?? 'a';
|
|
269
|
+
const locator = playwrightContext.page.locator(selector).first();
|
|
270
|
+
await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
|
|
271
|
+
const urls = await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl);
|
|
272
|
+
return (await this.enqueueLinks(urls, options, playwrightContext.request));
|
|
278
273
|
},
|
|
279
274
|
};
|
|
280
275
|
}
|
|
@@ -422,16 +417,21 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
422
417
|
}
|
|
423
418
|
}
|
|
424
419
|
}
|
|
425
|
-
async enqueueLinks(options, request) {
|
|
420
|
+
async enqueueLinks(urls, options, request) {
|
|
426
421
|
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
|
|
427
422
|
enqueueStrategy: options?.strategy,
|
|
428
423
|
finalRequestUrl: request.loadedUrl,
|
|
429
424
|
originalRequestUrl: request.url,
|
|
430
425
|
userProvidedBaseUrl: options?.baseUrl,
|
|
431
426
|
});
|
|
427
|
+
const requestsWithDepth = this.addCrawlDepthRequestGenerator(urls, request.crawlDepth + 1);
|
|
432
428
|
// The per-attempt transaction buffers these (the queue policy defaults to `deferred` here),
|
|
433
429
|
// so a discarded attempt's enqueues never reach the queue.
|
|
434
|
-
return await this.
|
|
430
|
+
return await this.addRequests(requestsWithDepth, {
|
|
431
|
+
...options,
|
|
432
|
+
baseUrl,
|
|
433
|
+
strategy: options.strategy ?? EnqueueStrategy.SameHostname,
|
|
434
|
+
});
|
|
435
435
|
}
|
|
436
436
|
createLogProxy(log, logs) {
|
|
437
437
|
return new Proxy(log, {
|
|
@@ -1,8 +1,24 @@
|
|
|
1
1
|
import { URL } from 'node:url';
|
|
2
|
-
import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, filterRequestOptionsByPatterns, Request as CrawleeRequest, serviceLocator, } from '@crawlee/browser';
|
|
3
|
-
import
|
|
2
|
+
import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, filterRequestOptionsByPatterns, parseArgument, urlPatternSchema, Request as CrawleeRequest, schemas, serviceLocator, } from '@crawlee/browser';
|
|
3
|
+
import { z } from 'zod';
|
|
4
4
|
const STARTING_Z_INDEX = 2147400000;
|
|
5
5
|
const getLog = () => serviceLocator.getChildLog('Playwright Click Elements');
|
|
6
|
+
const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
|
|
7
|
+
page: schemas.objectWithKeys(['goto', 'evaluate']),
|
|
8
|
+
requestManager: schemas.objectWithKeys(['fetchNextRequest', 'addRequestsBatched']),
|
|
9
|
+
selector: z.string(),
|
|
10
|
+
userData: schemas.anyObject.optional(),
|
|
11
|
+
clickOptions: schemas.anyObject.optional(),
|
|
12
|
+
include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
|
|
13
|
+
exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
|
|
14
|
+
transformRequestFunction: schemas.anyFunction.optional(),
|
|
15
|
+
waitForPageIdleSecs: schemas.anyNumber.default(1),
|
|
16
|
+
maxWaitForPageIdleSecs: schemas.anyNumber.default(5),
|
|
17
|
+
label: z.string().optional(),
|
|
18
|
+
forefront: z.boolean().optional(),
|
|
19
|
+
skipNavigation: z.boolean().optional(),
|
|
20
|
+
onSkippedRequest: schemas.anyFunction.optional(),
|
|
21
|
+
});
|
|
6
22
|
/**
|
|
7
23
|
* The function finds elements matching a specific CSS selector in a Playwright page,
|
|
8
24
|
* clicks all those elements using a mouse move and a left mouse button click and intercepts
|
|
@@ -44,24 +60,8 @@ const getLog = () => serviceLocator.getChildLog('Playwright Click Elements');
|
|
|
44
60
|
* @returns Promise that resolves to {@link BatchAddRequestsResult} object.
|
|
45
61
|
*/
|
|
46
62
|
export async function enqueueLinksByClickingElements(options) {
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
page: ow.object.hasKeys('goto', 'evaluate'),
|
|
50
|
-
requestManager: ow.object.hasKeys('fetchNextRequest', 'addRequestsBatched'),
|
|
51
|
-
selector: ow.string,
|
|
52
|
-
userData: ow.optional.object,
|
|
53
|
-
clickOptions: ow.optional.object,
|
|
54
|
-
include: ow.optional.array.ofType(urlPatternValidator),
|
|
55
|
-
exclude: ow.optional.array.ofType(urlPatternValidator),
|
|
56
|
-
transformRequestFunction: ow.optional.function,
|
|
57
|
-
waitForPageIdleSecs: ow.optional.number,
|
|
58
|
-
maxWaitForPageIdleSecs: ow.optional.number,
|
|
59
|
-
label: ow.optional.string,
|
|
60
|
-
forefront: ow.optional.boolean,
|
|
61
|
-
skipNavigation: ow.optional.boolean,
|
|
62
|
-
onSkippedRequest: ow.optional.function,
|
|
63
|
-
}));
|
|
64
|
-
const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, onSkippedRequest, } = options;
|
|
63
|
+
const parsedOptions = parseArgument(options, enqueueLinksByClickingElementsOptionsSchema, 'EnqueueLinksByClickingElementsOptions');
|
|
64
|
+
const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
|
|
65
65
|
const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
|
|
66
66
|
const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
|
|
67
67
|
const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
|
|
@@ -73,7 +73,7 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
73
73
|
maxWaitForPageIdleMillis,
|
|
74
74
|
clickOptions,
|
|
75
75
|
});
|
|
76
|
-
const requestOptions = createRequestOptions(interceptedRequests,
|
|
76
|
+
const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
|
|
77
77
|
const skippedByFilters = [];
|
|
78
78
|
let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (url) => skippedByFilters.push(url));
|
|
79
79
|
if (onSkippedRequest && skippedByFilters.length > 0) {
|
|
@@ -4,6 +4,7 @@ import type { PlaywrightPlugin } from '@crawlee/browser-pool';
|
|
|
4
4
|
import type { Dictionary } from '@crawlee/types';
|
|
5
5
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
6
6
|
import type { Download, LaunchOptions, Page, Response } from 'playwright';
|
|
7
|
+
import { z } from 'zod';
|
|
7
8
|
import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js';
|
|
8
9
|
import type { PlaywrightLaunchContext } from './playwright-launcher.js';
|
|
9
10
|
import type { BlockRequestsOptions, DirectNavigationOptions, HandleCloudflareChallengeOptions, InfiniteScrollOptions, InjectFileOptions, PlaywrightContextUtils, SaveSnapshotOptions } from './utils/playwright-utils.js';
|
|
@@ -145,105 +146,123 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
|
|
|
145
146
|
browserPlugins: [PlaywrightPlugin];
|
|
146
147
|
}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes> {
|
|
147
148
|
protected static optionsShape: {
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
206
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
207
|
-
statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
208
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
209
|
-
statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
210
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
211
|
-
additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
212
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
213
|
-
ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
214
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
215
|
-
blockedStatusCodes: import("ow").ArrayPredicate<number>;
|
|
216
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
217
|
-
retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
218
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
219
|
-
respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
|
|
220
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
221
|
-
transactionalStorage: import("ow").BasePredicate<boolean | Partial<import("@crawlee/browser").StorageWritePolicy> | undefined>;
|
|
222
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
223
|
-
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
224
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
225
|
-
httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
226
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
227
|
-
configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
228
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
229
|
-
storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
230
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
231
|
-
eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
232
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
233
|
-
logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
234
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
235
|
-
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
236
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
237
|
-
maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
238
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
239
|
-
maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
240
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
241
|
-
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
242
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
243
|
-
statistics: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
244
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
245
|
-
id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
149
|
+
browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
150
|
+
launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
151
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
152
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
153
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
154
|
+
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
155
|
+
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
156
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
157
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
158
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
159
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
160
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
161
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
162
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
163
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
164
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
165
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
166
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
167
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
168
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
169
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
170
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
171
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
172
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
173
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
174
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
175
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
176
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
177
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
178
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
179
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
180
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
181
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
182
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
183
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
184
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
185
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
186
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
187
|
+
deferred: "deferred";
|
|
188
|
+
writeThrough: "writeThrough";
|
|
189
|
+
}>>;
|
|
190
|
+
}, z.core.$strict>]>>;
|
|
191
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
192
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
193
|
+
httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
|
|
194
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
195
|
+
configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").Configuration, import("@crawlee/browser").Configuration>>;
|
|
196
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
197
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
198
|
+
eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").EventManager, import("@crawlee/browser").EventManager>>;
|
|
199
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
200
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
201
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
202
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
203
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
204
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
205
|
+
id: z.ZodOptional<z.ZodString>;
|
|
246
206
|
};
|
|
207
|
+
protected static optionsSchema: z.ZodObject<{
|
|
208
|
+
browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
209
|
+
launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
210
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
211
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
212
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
213
|
+
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
214
|
+
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
215
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
216
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
217
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
218
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
219
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
220
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
221
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
222
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
223
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
224
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
225
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
226
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
227
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
228
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
229
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
230
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
231
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
232
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
233
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
234
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
235
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
236
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
237
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
238
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
239
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
240
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
241
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
242
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
243
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
244
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
245
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
246
|
+
deferred: "deferred";
|
|
247
|
+
writeThrough: "writeThrough";
|
|
248
|
+
}>>;
|
|
249
|
+
}, z.core.$strict>]>>;
|
|
250
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
251
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
252
|
+
httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
|
|
253
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
254
|
+
configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").Configuration, import("@crawlee/browser").Configuration>>;
|
|
255
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
256
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
257
|
+
eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").EventManager, import("@crawlee/browser").EventManager>>;
|
|
258
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
259
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
260
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
261
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
262
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
263
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
264
|
+
id: z.ZodOptional<z.ZodString>;
|
|
265
|
+
}, z.core.$strict>;
|
|
247
266
|
/**
|
|
248
267
|
* All `PlaywrightCrawler` parameters are passed via an options object.
|
|
249
268
|
*/
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { BrowserCrawler, RequestState, Router, serviceLocator } from '@crawlee/browser';
|
|
2
|
-
import
|
|
1
|
+
import { BrowserCrawler, parseArgument, RequestState, Router, schemas, serviceLocator } from '@crawlee/browser';
|
|
2
|
+
import { z } from 'zod';
|
|
3
3
|
import { PlaywrightLauncher } from './playwright-launcher.js';
|
|
4
4
|
import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
|
|
5
5
|
/**
|
|
@@ -70,19 +70,18 @@ import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
|
|
|
70
70
|
export class PlaywrightCrawler extends BrowserCrawler {
|
|
71
71
|
static optionsShape = {
|
|
72
72
|
...BrowserCrawler.optionsShape,
|
|
73
|
-
browserPoolOptions:
|
|
74
|
-
launcher:
|
|
75
|
-
ignoreIframes: ow.optional.boolean,
|
|
76
|
-
ignoreShadowRoots: ow.optional.boolean,
|
|
73
|
+
browserPoolOptions: schemas.anyObject.optional(),
|
|
74
|
+
launcher: schemas.anyObject.optional(),
|
|
77
75
|
};
|
|
76
|
+
static optionsSchema = z.strictObject(PlaywrightCrawler.optionsShape);
|
|
78
77
|
/**
|
|
79
78
|
* All `PlaywrightCrawler` parameters are passed via an options object.
|
|
80
79
|
*/
|
|
81
80
|
constructor(options = {}) {
|
|
82
|
-
|
|
83
|
-
const { launchContext
|
|
81
|
+
const parsedOptions = parseArgument(options, PlaywrightCrawler.optionsSchema, 'PlaywrightCrawlerOptions');
|
|
82
|
+
const { launchContext, headless, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
|
|
84
83
|
const browserPoolOptions = {
|
|
85
|
-
...
|
|
84
|
+
...parsedOptions.browserPoolOptions,
|
|
86
85
|
};
|
|
87
86
|
if (launchContext.proxyUrl) {
|
|
88
87
|
throw new Error('PlaywrightCrawlerOptions.launchContext.proxyUrl is not allowed in PlaywrightCrawler.' +
|
|
@@ -97,7 +96,7 @@ export class PlaywrightCrawler extends BrowserCrawler {
|
|
|
97
96
|
launchContext.launchOptions ??= {};
|
|
98
97
|
launchContext.launchOptions.headless = headless;
|
|
99
98
|
}
|
|
100
|
-
const playwrightLauncher = new PlaywrightLauncher(launchContext,
|
|
99
|
+
const playwrightLauncher = new PlaywrightLauncher(launchContext, parsedOptions.configuration);
|
|
101
100
|
browserPoolOptions.browserPlugins = [playwrightLauncher.createBrowserPlugin()];
|
|
102
101
|
super({
|
|
103
102
|
...browserCrawlerOptions,
|
|
@@ -3,6 +3,7 @@ import { BrowserLauncher, Configuration } from '@crawlee/browser';
|
|
|
3
3
|
import { PlaywrightPlugin } from '@crawlee/browser-pool';
|
|
4
4
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
5
5
|
import type { Browser, BrowserType, LaunchOptions } from 'playwright';
|
|
6
|
+
import { z } from 'zod';
|
|
6
7
|
/**
|
|
7
8
|
* Apify extends the launch options of Playwright.
|
|
8
9
|
* You can use any of the Playwright compatible
|
|
@@ -73,26 +74,34 @@ export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin
|
|
|
73
74
|
readonly configuration: Configuration;
|
|
74
75
|
protected static optionsShape: {
|
|
75
76
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
76
|
-
launcher: import("
|
|
77
|
+
launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
77
78
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
78
|
-
launchContextOptions: import("
|
|
79
|
+
launchContextOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
80
|
+
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
81
|
+
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
82
|
+
useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
|
|
83
|
+
browserPerProxy: z.ZodOptional<z.ZodBoolean>;
|
|
84
|
+
ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
|
|
85
|
+
userDataDir: z.ZodOptional<z.ZodString>;
|
|
79
86
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
85
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
86
|
-
browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
87
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
88
|
-
ignoreProxyCertificate: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
87
|
+
launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
88
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
89
|
+
};
|
|
90
|
+
protected static optionsSchema: z.ZodObject<{
|
|
89
91
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
90
|
-
|
|
92
|
+
launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
91
93
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
92
|
-
|
|
94
|
+
launchContextOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
95
|
+
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
96
|
+
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
97
|
+
useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
|
|
98
|
+
browserPerProxy: z.ZodOptional<z.ZodBoolean>;
|
|
99
|
+
ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
|
|
100
|
+
userDataDir: z.ZodOptional<z.ZodString>;
|
|
93
101
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
94
|
-
|
|
95
|
-
|
|
102
|
+
launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
103
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
104
|
+
}, z.core.$strict>;
|
|
96
105
|
/**
|
|
97
106
|
* All `PlaywrightLauncher` parameters are passed via this launchContext object.
|
|
98
107
|
*/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { BrowserLauncher, Configuration } from '@crawlee/browser';
|
|
1
|
+
import { BrowserLauncher, Configuration, parseArgument, schemas } from '@crawlee/browser';
|
|
2
2
|
import { PlaywrightPlugin } from '@crawlee/browser-pool';
|
|
3
|
-
import
|
|
3
|
+
import { z } from 'zod';
|
|
4
4
|
/**
|
|
5
5
|
* `PlaywrightLauncher` is based on the `BrowserLauncher`. It launches `playwright` browser instance.
|
|
6
6
|
* @ignore
|
|
@@ -9,21 +9,23 @@ export class PlaywrightLauncher extends BrowserLauncher {
|
|
|
9
9
|
configuration;
|
|
10
10
|
static optionsShape = {
|
|
11
11
|
...BrowserLauncher.optionsShape,
|
|
12
|
-
launcher
|
|
13
|
-
|
|
12
|
+
// Passthrough schemas — the launcher module object must keep its prototype through parsing.
|
|
13
|
+
launcher: schemas.anyObject.optional(),
|
|
14
|
+
launchContextOptions: schemas.anyObject.optional(),
|
|
14
15
|
};
|
|
16
|
+
static optionsSchema = z.strictObject(PlaywrightLauncher.optionsShape);
|
|
15
17
|
/**
|
|
16
18
|
* All `PlaywrightLauncher` parameters are passed via this launchContext object.
|
|
17
19
|
*/
|
|
18
20
|
constructor(launchContext = {}, configuration = Configuration.getGlobalConfiguration()) {
|
|
19
|
-
|
|
20
|
-
const { launcher = BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium, } =
|
|
21
|
-
const { launchOptions = {}, ...rest } =
|
|
21
|
+
const parsedContext = parseArgument(launchContext, PlaywrightLauncher.optionsSchema, 'PlaywrightLaunchContext');
|
|
22
|
+
const { launcher = BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium, } = parsedContext;
|
|
23
|
+
const { launchOptions = {}, ...rest } = parsedContext;
|
|
22
24
|
super({
|
|
23
25
|
...rest,
|
|
24
26
|
launchOptions: {
|
|
25
27
|
...launchOptions,
|
|
26
|
-
executablePath: getDefaultExecutablePath(
|
|
28
|
+
executablePath: getDefaultExecutablePath(parsedContext, configuration),
|
|
27
29
|
},
|
|
28
30
|
launcher,
|
|
29
31
|
}, configuration);
|
|
@@ -20,9 +20,9 @@
|
|
|
20
20
|
import { readFile } from 'node:fs/promises';
|
|
21
21
|
import { createRequire } from 'node:module';
|
|
22
22
|
import vm from 'node:vm';
|
|
23
|
-
import { Configuration, KeyValueStore, serviceLocator, SessionError, validators } from '@crawlee/browser';
|
|
23
|
+
import { Configuration, KeyValueStore, parseArgument, schemas, serviceLocator, SessionError, validators, } from '@crawlee/browser';
|
|
24
24
|
import { expandShadowRoots, sleep } from '@crawlee/utils';
|
|
25
|
-
import
|
|
25
|
+
import { z } from 'zod';
|
|
26
26
|
import { LruCache } from '@apify/datastructures';
|
|
27
27
|
import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
|
|
28
28
|
import { RenderingTypePredictor } from './rendering-type-prediction.js';
|
|
@@ -31,6 +31,36 @@ const require = createRequire(import.meta.url);
|
|
|
31
31
|
const jqueryPath = require.resolve('jquery');
|
|
32
32
|
const MAX_INJECT_FILE_CACHE_SIZE = 10;
|
|
33
33
|
const DEFAULT_BLOCK_REQUEST_URL_PATTERNS = ['.css', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip'];
|
|
34
|
+
const filePathSchema = z.string();
|
|
35
|
+
const injectFileOptionsSchema = z.strictObject({
|
|
36
|
+
surviveNavigations: z.boolean().optional(),
|
|
37
|
+
});
|
|
38
|
+
const gotoExtendedRequestSchema = z.looseObject({
|
|
39
|
+
url: z.url(),
|
|
40
|
+
method: z.string().optional(),
|
|
41
|
+
headers: schemas.anyObject.optional(),
|
|
42
|
+
payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
|
|
43
|
+
});
|
|
44
|
+
const blockRequestsOptionsSchema = z.strictObject({
|
|
45
|
+
urlPatterns: schemas.arrayOf(z.string(), 'strings').default(DEFAULT_BLOCK_REQUEST_URL_PATTERNS),
|
|
46
|
+
extraUrlPatterns: schemas.arrayOf(z.string(), 'strings').default(() => []),
|
|
47
|
+
});
|
|
48
|
+
const infiniteScrollOptionsSchema = z.strictObject({
|
|
49
|
+
timeoutSecs: schemas.anyNumber.default(0),
|
|
50
|
+
maxScrollHeight: schemas.anyNumber.default(0),
|
|
51
|
+
waitForSecs: schemas.anyNumber.default(4),
|
|
52
|
+
scrollDownAndUp: z.boolean().default(false),
|
|
53
|
+
buttonSelector: z.string().optional(),
|
|
54
|
+
stopScrollCallback: schemas.anyFunction.optional(),
|
|
55
|
+
});
|
|
56
|
+
const saveSnapshotOptionsSchema = z.strictObject({
|
|
57
|
+
key: z.string().min(1).default('SNAPSHOT'),
|
|
58
|
+
screenshotQuality: schemas.anyNumber.default(50),
|
|
59
|
+
saveScreenshot: z.boolean().default(true),
|
|
60
|
+
saveHtml: z.boolean().default(true),
|
|
61
|
+
keyValueStoreName: z.string().optional(),
|
|
62
|
+
configuration: schemas.anyObject.optional(),
|
|
63
|
+
});
|
|
34
64
|
/**
|
|
35
65
|
* Cache contents of previously injected files to limit file system access.
|
|
36
66
|
*/
|
|
@@ -47,18 +77,16 @@ const injectedFilesCache = new LruCache({ maxLength: MAX_INJECT_FILE_CACHE_SIZE
|
|
|
47
77
|
* @param [options]
|
|
48
78
|
*/
|
|
49
79
|
export async function injectFile(page, filePath, options = {}) {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
surviveNavigations: ow.optional.boolean,
|
|
54
|
-
}));
|
|
80
|
+
parseArgument(page, validators.browserPage);
|
|
81
|
+
parseArgument(filePath, filePathSchema);
|
|
82
|
+
const { surviveNavigations } = parseArgument(options, injectFileOptionsSchema);
|
|
55
83
|
let contents = injectedFilesCache.get(filePath);
|
|
56
84
|
if (!contents) {
|
|
57
85
|
contents = await readFile(filePath, 'utf8');
|
|
58
86
|
injectedFilesCache.add(filePath, contents);
|
|
59
87
|
}
|
|
60
88
|
const evalP = page.evaluate(contents);
|
|
61
|
-
if (
|
|
89
|
+
if (surviveNavigations) {
|
|
62
90
|
page.on('framenavigated', async () => page
|
|
63
91
|
.evaluate(contents)
|
|
64
92
|
.catch((error) => getLog().warning('An error occurred during the script injection!', { error })));
|
|
@@ -92,7 +120,7 @@ export async function injectFile(page, filePath, options = {}) {
|
|
|
92
120
|
* @param [options.surviveNavigations] Opt-out option to disable the JQuery reinjection after navigation.
|
|
93
121
|
*/
|
|
94
122
|
export async function injectJQuery(page, options) {
|
|
95
|
-
|
|
123
|
+
parseArgument(page, validators.browserPage);
|
|
96
124
|
return injectFile(page, jqueryPath, { surviveNavigations: options?.surviveNavigations ?? true });
|
|
97
125
|
}
|
|
98
126
|
/**
|
|
@@ -108,14 +136,9 @@ export async function injectJQuery(page, options) {
|
|
|
108
136
|
* @param [gotoOptions] Custom options for `page.goto()`.
|
|
109
137
|
*/
|
|
110
138
|
export async function gotoExtended(page, request, gotoOptions = {}) {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
method: ow.optional.string,
|
|
115
|
-
headers: ow.optional.object,
|
|
116
|
-
payload: ow.optional.any(ow.string, ow.uint8Array),
|
|
117
|
-
}));
|
|
118
|
-
ow(gotoOptions, ow.object);
|
|
139
|
+
parseArgument(page, validators.browserPage);
|
|
140
|
+
parseArgument(request, gotoExtendedRequestSchema);
|
|
141
|
+
parseArgument(gotoOptions, schemas.anyObject);
|
|
119
142
|
const { url, method, headers, payload } = request;
|
|
120
143
|
const isEmpty = (o) => !o || Object.keys(o).length === 0;
|
|
121
144
|
if (method !== 'GET' || payload) {
|
|
@@ -200,12 +223,8 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
|
|
|
200
223
|
* @param [options]
|
|
201
224
|
*/
|
|
202
225
|
export async function blockRequests(page, options = {}) {
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
urlPatterns: ow.optional.array.ofType(ow.string),
|
|
206
|
-
extraUrlPatterns: ow.optional.array.ofType(ow.string),
|
|
207
|
-
}));
|
|
208
|
-
const { urlPatterns = DEFAULT_BLOCK_REQUEST_URL_PATTERNS, extraUrlPatterns = [] } = options;
|
|
226
|
+
parseArgument(page, validators.browserPage);
|
|
227
|
+
const { urlPatterns, extraUrlPatterns } = parseArgument(options, blockRequestsOptionsSchema);
|
|
209
228
|
const patternsToBlock = [...urlPatterns, ...extraUrlPatterns];
|
|
210
229
|
try {
|
|
211
230
|
const client = await page.context().newCDPSession(page);
|
|
@@ -263,16 +282,8 @@ export function compileScript(scriptString, context = Object.create(null)) {
|
|
|
263
282
|
* @param [options]
|
|
264
283
|
*/
|
|
265
284
|
export async function infiniteScroll(page, options = {}) {
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
timeoutSecs: ow.optional.number,
|
|
269
|
-
maxScrollHeight: ow.optional.number,
|
|
270
|
-
waitForSecs: ow.optional.number,
|
|
271
|
-
scrollDownAndUp: ow.optional.boolean,
|
|
272
|
-
buttonSelector: ow.optional.string,
|
|
273
|
-
stopScrollCallback: ow.optional.function,
|
|
274
|
-
}));
|
|
275
|
-
const { timeoutSecs = 0, maxScrollHeight = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback, } = options;
|
|
285
|
+
parseArgument(page, validators.browserPage);
|
|
286
|
+
const { timeoutSecs, maxScrollHeight, waitForSecs, scrollDownAndUp, buttonSelector, stopScrollCallback } = parseArgument(options, infiniteScrollOptionsSchema);
|
|
276
287
|
let finished;
|
|
277
288
|
const startTime = Date.now();
|
|
278
289
|
const CHECK_INTERVAL_MILLIS = 1000;
|
|
@@ -349,16 +360,8 @@ export async function infiniteScroll(page, options = {}) {
|
|
|
349
360
|
* @param [options]
|
|
350
361
|
*/
|
|
351
362
|
export async function saveSnapshot(page, options = {}) {
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
key: ow.optional.string.nonEmpty,
|
|
355
|
-
screenshotQuality: ow.optional.number,
|
|
356
|
-
saveScreenshot: ow.optional.boolean,
|
|
357
|
-
saveHtml: ow.optional.boolean,
|
|
358
|
-
keyValueStoreName: ow.optional.string,
|
|
359
|
-
configuration: ow.optional.object,
|
|
360
|
-
}));
|
|
361
|
-
const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, configuration, } = options;
|
|
363
|
+
parseArgument(page, validators.browserPage);
|
|
364
|
+
const { key, screenshotQuality, saveScreenshot, saveHtml, keyValueStoreName, configuration } = parseArgument(options, saveSnapshotOptionsSchema);
|
|
362
365
|
try {
|
|
363
366
|
const store = await KeyValueStore.open(keyValueStoreName ? { name: keyValueStoreName } : null, {
|
|
364
367
|
configuration: configuration ?? Configuration.getGlobalConfiguration(),
|
|
@@ -396,7 +399,7 @@ export async function saveSnapshot(page, options = {}) {
|
|
|
396
399
|
* @param ignoreShadowRoots
|
|
397
400
|
*/
|
|
398
401
|
export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIframes = false) {
|
|
399
|
-
|
|
402
|
+
parseArgument(page, validators.browserPage);
|
|
400
403
|
const html = ignoreShadowRoots
|
|
401
404
|
? null
|
|
402
405
|
: (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
|
|
@@ -457,7 +460,7 @@ ${error.message}
|
|
|
457
460
|
return idcacPlaywright;
|
|
458
461
|
}
|
|
459
462
|
export async function closeCookieModals(page) {
|
|
460
|
-
|
|
463
|
+
parseArgument(page, validators.browserPage);
|
|
461
464
|
const idcac = await getIdcacPlaywright();
|
|
462
465
|
if (idcac?.getInjectableScript()) {
|
|
463
466
|
await page.evaluate(idcac.getInjectableScript());
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/playwright",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.123",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -49,22 +49,21 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^2.0.3",
|
|
51
51
|
"@apify/timeout": "^0.4.4",
|
|
52
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
53
|
-
"@crawlee/browser": "4.0.0-beta.
|
|
54
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
55
|
-
"@crawlee/cheerio": "4.0.0-beta.
|
|
56
|
-
"@crawlee/core": "4.0.0-beta.
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
52
|
+
"@crawlee/basic": "4.0.0-beta.123",
|
|
53
|
+
"@crawlee/browser": "4.0.0-beta.123",
|
|
54
|
+
"@crawlee/browser-pool": "4.0.0-beta.123",
|
|
55
|
+
"@crawlee/cheerio": "4.0.0-beta.123",
|
|
56
|
+
"@crawlee/core": "4.0.0-beta.123",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.123",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.123",
|
|
59
59
|
"cheerio": "^1.0.0",
|
|
60
60
|
"jquery": "^3.7.1",
|
|
61
61
|
"ml-logistic-regression": "^2.0.0",
|
|
62
62
|
"ml-matrix": "^6.12.1",
|
|
63
|
-
"ow": "^2.0.0",
|
|
64
63
|
"string-comparison": "^1.3.0",
|
|
65
64
|
"tslib": "^2.8.1",
|
|
66
65
|
"type-fest": "^4.0.0",
|
|
67
|
-
"zod": "^4.
|
|
66
|
+
"zod": "^4.4.3"
|
|
68
67
|
},
|
|
69
68
|
"peerDependencies": {
|
|
70
69
|
"idcac-playwright": "^0.2.0",
|
|
@@ -85,5 +84,5 @@
|
|
|
85
84
|
}
|
|
86
85
|
}
|
|
87
86
|
},
|
|
88
|
-
"gitHead": "
|
|
87
|
+
"gitHead": "f77648095c6a3f5ed8815c7620ea765db430ae44"
|
|
89
88
|
}
|