@crawlee/playwright 4.0.0-beta.148 → 4.0.0-beta.149
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -149,7 +149,6 @@ export class PlaywrightCrawler extends BrowserCrawler {
|
|
|
149
149
|
requestManager: this.requestManager,
|
|
150
150
|
}),
|
|
151
151
|
compileScript: (scriptString, ctx) => playwrightUtils.compileScript(scriptString, ctx),
|
|
152
|
-
closeCookieModals: async () => playwrightUtils.closeCookieModals(context.page),
|
|
153
152
|
handleCloudflareChallenge: async (options) => {
|
|
154
153
|
return playwrightUtils.handleCloudflareChallenge(context.page, context.request.url, options);
|
|
155
154
|
},
|
|
@@ -288,7 +288,6 @@ export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions):
|
|
|
288
288
|
* @param ignoreShadowRoots
|
|
289
289
|
*/
|
|
290
290
|
export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioAPI>;
|
|
291
|
-
export declare function closeCookieModals(page: Page): Promise<void>;
|
|
292
291
|
export interface HandleCloudflareChallengeOptions {
|
|
293
292
|
/** Logging defaults to the `debug` level, use this flag to log to `info` level instead. */
|
|
294
293
|
verbose?: boolean;
|
|
@@ -513,19 +512,6 @@ export interface PlaywrightContextUtils {
|
|
|
513
512
|
* secured copies beforehand.
|
|
514
513
|
*/
|
|
515
514
|
compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
|
|
516
|
-
/**
|
|
517
|
-
* Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
|
|
518
|
-
*
|
|
519
|
-
* Note that this method requires the idcac-playwright package to be installed.
|
|
520
|
-
* Crawlee does not include it by default due to licensing issues.
|
|
521
|
-
*
|
|
522
|
-
* To use this method, please install the package manually by running:
|
|
523
|
-
*
|
|
524
|
-
* ```bash
|
|
525
|
-
* npm install idcac-playwright
|
|
526
|
-
* ```
|
|
527
|
-
*/
|
|
528
|
-
closeCookieModals(): Promise<void>;
|
|
529
515
|
/**
|
|
530
516
|
* This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
|
|
531
517
|
* It will try to detect the Cloudflare page, click on the checkbox, and wait for 10 seconds (configurable
|
|
@@ -583,7 +569,6 @@ export declare const playwrightUtils: {
|
|
|
583
569
|
infiniteScroll: typeof infiniteScroll;
|
|
584
570
|
saveSnapshot: typeof saveSnapshot;
|
|
585
571
|
compileScript: typeof compileScript;
|
|
586
|
-
closeCookieModals: typeof closeCookieModals;
|
|
587
572
|
RenderingTypePredictor: typeof RenderingTypePredictor;
|
|
588
573
|
handleCloudflareChallenge: typeof handleCloudflareChallenge;
|
|
589
574
|
};
|
|
@@ -438,35 +438,6 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
|
|
|
438
438
|
}
|
|
439
439
|
return $;
|
|
440
440
|
}
|
|
441
|
-
let idcacPlaywright = null;
|
|
442
|
-
async function getIdcacPlaywright() {
|
|
443
|
-
if (idcacPlaywright)
|
|
444
|
-
return idcacPlaywright;
|
|
445
|
-
try {
|
|
446
|
-
idcacPlaywright = await import('idcac-playwright');
|
|
447
|
-
}
|
|
448
|
-
catch (error) {
|
|
449
|
-
getLog().warning(`Failed to import 'idcac-playwright'.
|
|
450
|
-
|
|
451
|
-
We recently made idcac-playwright an optional dependency due to licensing issues.
|
|
452
|
-
To use this feature, please install it manually by running
|
|
453
|
-
|
|
454
|
-
npm install idcac-playwright
|
|
455
|
-
|
|
456
|
-
Original error message follows:
|
|
457
|
-
|
|
458
|
-
${error.message}
|
|
459
|
-
`);
|
|
460
|
-
}
|
|
461
|
-
return idcacPlaywright;
|
|
462
|
-
}
|
|
463
|
-
export async function closeCookieModals(page) {
|
|
464
|
-
parseArgument(page, validators.browserPage);
|
|
465
|
-
const idcac = await getIdcacPlaywright();
|
|
466
|
-
if (idcac?.getInjectableScript()) {
|
|
467
|
-
await page.evaluate(idcac.getInjectableScript());
|
|
468
|
-
}
|
|
469
|
-
}
|
|
470
441
|
/**
|
|
471
442
|
* This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
|
|
472
443
|
* It will try to detect the Cloudflare page, click on the checkbox, and wait for 10 seconds (configurable
|
|
@@ -588,7 +559,6 @@ export const playwrightUtils = {
|
|
|
588
559
|
infiniteScroll,
|
|
589
560
|
saveSnapshot,
|
|
590
561
|
compileScript,
|
|
591
|
-
closeCookieModals,
|
|
592
562
|
RenderingTypePredictor,
|
|
593
563
|
handleCloudflareChallenge,
|
|
594
564
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/playwright",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.149",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -49,13 +49,13 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^2.0.3",
|
|
51
51
|
"@apify/timeout": "^0.4.4",
|
|
52
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
53
|
-
"@crawlee/browser": "4.0.0-beta.
|
|
54
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
55
|
-
"@crawlee/cheerio": "4.0.0-beta.
|
|
56
|
-
"@crawlee/core": "4.0.0-beta.
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
52
|
+
"@crawlee/basic": "4.0.0-beta.149",
|
|
53
|
+
"@crawlee/browser": "4.0.0-beta.149",
|
|
54
|
+
"@crawlee/browser-pool": "4.0.0-beta.149",
|
|
55
|
+
"@crawlee/cheerio": "4.0.0-beta.149",
|
|
56
|
+
"@crawlee/core": "4.0.0-beta.149",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.149",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.149",
|
|
59
59
|
"cheerio": "^1.0.0",
|
|
60
60
|
"jquery": "^3.7.1",
|
|
61
61
|
"ml-logistic-regression": "^2.0.0",
|
|
@@ -66,13 +66,9 @@
|
|
|
66
66
|
"zod": "^4.4.3"
|
|
67
67
|
},
|
|
68
68
|
"peerDependencies": {
|
|
69
|
-
"idcac-playwright": "^0.2.0",
|
|
70
69
|
"playwright": "*"
|
|
71
70
|
},
|
|
72
71
|
"peerDependenciesMeta": {
|
|
73
|
-
"idcac-playwright": {
|
|
74
|
-
"optional": true
|
|
75
|
-
},
|
|
76
72
|
"playwright": {
|
|
77
73
|
"optional": true
|
|
78
74
|
}
|
|
@@ -84,5 +80,5 @@
|
|
|
84
80
|
}
|
|
85
81
|
}
|
|
86
82
|
},
|
|
87
|
-
"gitHead": "
|
|
83
|
+
"gitHead": "c88124c43b4c366d4b0ade01da5e90843d59c582"
|
|
88
84
|
}
|