@crawlee/puppeteer 4.0.0-beta.147 → 4.0.0-beta.149
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -147,7 +147,6 @@ export class PuppeteerCrawler extends BrowserCrawler {
|
|
|
147
147
|
...options,
|
|
148
148
|
configuration: serviceLocator.getConfiguration(),
|
|
149
149
|
}),
|
|
150
|
-
closeCookieModals: async () => puppeteerUtils.closeCookieModals(context.page),
|
|
151
150
|
};
|
|
152
151
|
}
|
|
153
152
|
async navigationHandler(crawlingContext, gotoOptions) {
|
|
@@ -315,7 +315,6 @@ export interface SaveSnapshotOptions {
|
|
|
315
315
|
* @param [options]
|
|
316
316
|
*/
|
|
317
317
|
export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions): Promise<void>;
|
|
318
|
-
export declare function closeCookieModals(page: Page): Promise<void>;
|
|
319
318
|
export interface PuppeteerContextUtils {
|
|
320
319
|
/**
|
|
321
320
|
* Injects a JavaScript file into current `page`.
|
|
@@ -548,19 +547,6 @@ export interface PuppeteerContextUtils {
|
|
|
548
547
|
* Saves a full screenshot and HTML of the current page into a Key-Value store.
|
|
549
548
|
*/
|
|
550
549
|
saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
|
|
551
|
-
/**
|
|
552
|
-
* Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
|
|
553
|
-
*
|
|
554
|
-
* Note that this method requires the idcac-playwright package to be installed.
|
|
555
|
-
* Crawlee does not include it by default due to licensing issues.
|
|
556
|
-
*
|
|
557
|
-
* To use this method, please install the package manually by running:
|
|
558
|
-
*
|
|
559
|
-
* ```bash
|
|
560
|
-
* npm install idcac-playwright
|
|
561
|
-
* ```
|
|
562
|
-
*/
|
|
563
|
-
closeCookieModals(): Promise<void>;
|
|
564
550
|
}
|
|
565
551
|
export { enqueueLinksByClickingElements, addInterceptRequestHandler, removeInterceptRequestHandler };
|
|
566
552
|
/** @internal */
|
|
@@ -576,5 +562,4 @@ export declare const puppeteerUtils: {
|
|
|
576
562
|
infiniteScroll: typeof infiniteScroll;
|
|
577
563
|
saveSnapshot: typeof saveSnapshot;
|
|
578
564
|
parseWithCheerio: typeof parseWithCheerio;
|
|
579
|
-
closeCookieModals: typeof closeCookieModals;
|
|
580
565
|
};
|
|
@@ -546,35 +546,6 @@ export async function saveSnapshot(page, options = {}) {
|
|
|
546
546
|
throw new Error(`saveSnapshot with key ${key} failed.\nCause:${err.message}`);
|
|
547
547
|
}
|
|
548
548
|
}
|
|
549
|
-
let idcacPlaywright = null;
|
|
550
|
-
async function getIdcacPlaywright() {
|
|
551
|
-
if (idcacPlaywright)
|
|
552
|
-
return idcacPlaywright;
|
|
553
|
-
try {
|
|
554
|
-
idcacPlaywright = await import('idcac-playwright');
|
|
555
|
-
}
|
|
556
|
-
catch (error) {
|
|
557
|
-
getLog().warning(`Failed to import 'idcac-playwright'.
|
|
558
|
-
|
|
559
|
-
We recently made idcac-playwright an optional dependency due to licensing issues.
|
|
560
|
-
To use this feature, please install it manually by running
|
|
561
|
-
|
|
562
|
-
npm install idcac-playwright
|
|
563
|
-
|
|
564
|
-
Original error message follows:
|
|
565
|
-
|
|
566
|
-
${error.message}
|
|
567
|
-
`);
|
|
568
|
-
}
|
|
569
|
-
return idcacPlaywright;
|
|
570
|
-
}
|
|
571
|
-
export async function closeCookieModals(page) {
|
|
572
|
-
parseArgument(page, validators.browserPage);
|
|
573
|
-
const idcac = await getIdcacPlaywright();
|
|
574
|
-
if (idcac?.getInjectableScript()) {
|
|
575
|
-
await page.evaluate(idcac.getInjectableScript());
|
|
576
|
-
}
|
|
577
|
-
}
|
|
578
549
|
export { enqueueLinksByClickingElements, addInterceptRequestHandler, removeInterceptRequestHandler };
|
|
579
550
|
/** @internal */
|
|
580
551
|
export const puppeteerUtils = {
|
|
@@ -589,5 +560,4 @@ export const puppeteerUtils = {
|
|
|
589
560
|
infiniteScroll,
|
|
590
561
|
saveSnapshot,
|
|
591
562
|
parseWithCheerio,
|
|
592
|
-
closeCookieModals,
|
|
593
563
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/puppeteer",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.149",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -48,11 +48,11 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^2.0.3",
|
|
51
|
-
"@crawlee/browser": "4.0.0-beta.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
53
|
-
"@crawlee/core": "4.0.0-beta.
|
|
54
|
-
"@crawlee/types": "4.0.0-beta.
|
|
55
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/browser": "4.0.0-beta.149",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-beta.149",
|
|
53
|
+
"@crawlee/core": "4.0.0-beta.149",
|
|
54
|
+
"@crawlee/types": "4.0.0-beta.149",
|
|
55
|
+
"@crawlee/utils": "4.0.0-beta.149",
|
|
56
56
|
"cheerio": "^1.0.0",
|
|
57
57
|
"devtools-protocol": "*",
|
|
58
58
|
"jquery": "^3.7.1",
|
|
@@ -60,13 +60,9 @@
|
|
|
60
60
|
"zod": "^4.4.3"
|
|
61
61
|
},
|
|
62
62
|
"peerDependencies": {
|
|
63
|
-
"idcac-playwright": "^0.2.0",
|
|
64
63
|
"puppeteer": "*"
|
|
65
64
|
},
|
|
66
65
|
"peerDependenciesMeta": {
|
|
67
|
-
"idcac-playwright": {
|
|
68
|
-
"optional": true
|
|
69
|
-
},
|
|
70
66
|
"puppeteer": {
|
|
71
67
|
"optional": true
|
|
72
68
|
}
|
|
@@ -78,5 +74,5 @@
|
|
|
78
74
|
}
|
|
79
75
|
}
|
|
80
76
|
},
|
|
81
|
-
"gitHead": "
|
|
77
|
+
"gitHead": "c88124c43b4c366d4b0ade01da5e90843d59c582"
|
|
82
78
|
}
|