@crawlee/puppeteer 3.0.0-alpha.5 → 3.0.0-alpha.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/index.d.ts +0 -2
- package/index.d.ts.map +1 -1
- package/index.js +1 -2
- package/index.js.map +1 -1
- package/index.mjs +0 -5
- package/internals/enqueue-links/click-elements.d.ts +3 -2
- package/internals/enqueue-links/click-elements.d.ts.map +1 -1
- package/internals/enqueue-links/click-elements.js +12 -13
- package/internals/enqueue-links/click-elements.js.map +1 -1
- package/internals/puppeteer-crawler.d.ts +8 -9
- package/internals/puppeteer-crawler.d.ts.map +1 -1
- package/internals/puppeteer-crawler.js +6 -17
- package/internals/puppeteer-crawler.js.map +1 -1
- package/internals/puppeteer-launcher.d.ts +1 -28
- package/internals/puppeteer-launcher.d.ts.map +1 -1
- package/internals/puppeteer-launcher.js +1 -77
- package/internals/puppeteer-launcher.js.map +1 -1
- package/internals/utils/puppeteer_utils.d.ts +20 -3
- package/internals/utils/puppeteer_utils.d.ts.map +1 -1
- package/internals/utils/puppeteer_utils.js +31 -10
- package/internals/utils/puppeteer_utils.js.map +1 -1
- package/package.json +4 -3
- package/tsconfig.build.tsbuildinfo +1 -1
- package/internals/stealth/hiding_tricks.d.ts +0 -10
- package/internals/stealth/hiding_tricks.d.ts.map +0 -1
- package/internals/stealth/hiding_tricks.js +0 -334
- package/internals/stealth/hiding_tricks.js.map +0 -1
- package/internals/stealth/index.d.ts +0 -40
- package/internals/stealth/index.d.ts.map +0 -1
- package/internals/stealth/index.js +0 -108
- package/internals/stealth/index.js.map +0 -1
package/README.md
CHANGED
|
@@ -18,6 +18,8 @@ running on the [Apify Cloud](https://apify.com/).
|
|
|
18
18
|
|
|
19
19
|
**View full documentation, guides and examples on the [Apify SDK project website](https://sdk.apify.com)**
|
|
20
20
|
|
|
21
|
+
> Would you like to work with us on Apify SDK or similar projects? [We are hiring!](https://apify.com/jobs#senior-node.js-engineer)
|
|
22
|
+
|
|
21
23
|
## Motivation
|
|
22
24
|
|
|
23
25
|
Thanks to tools like [Playwright](https://github.com/microsoft/playwright), [Puppeteer](https://github.com/puppeteer/puppeteer) or
|
package/index.d.ts
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
export * from '@crawlee/browser';
|
|
2
2
|
export * from './internals/puppeteer-crawler';
|
|
3
3
|
export * from './internals/puppeteer-launcher';
|
|
4
|
-
export * as puppeteerStealth from './internals/stealth';
|
|
5
|
-
export type { StealthOptions } from './internals/stealth';
|
|
6
4
|
export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception';
|
|
7
5
|
export type { InterceptHandler } from './internals/utils/puppeteer_request_interception';
|
|
8
6
|
export * as puppeteerUtils from './internals/utils/puppeteer_utils';
|
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAE/C,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAE/C,OAAO,KAAK,4BAA4B,MAAM,kDAAkD,CAAC;AACjG,YAAY,EAAE,gBAAgB,EAAE,MAAM,kDAAkD,CAAC;AAEzF,OAAO,KAAK,cAAc,MAAM,mCAAmC,CAAC;AACpE,YAAY,EACR,oBAAoB,EACpB,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,IAAI,gCAAgC,EAC3D,qBAAqB,EACrB,iBAAiB,EACjB,mBAAmB,GACtB,MAAM,mCAAmC,CAAC;AAE3C,OAAO,KAAK,sBAAsB,MAAM,0CAA0C,CAAC;AACnF,YAAY,EAAE,qCAAqC,EAAE,MAAM,0CAA0C,CAAC"}
|
package/index.js
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.puppeteerClickElements = exports.puppeteerUtils = exports.puppeteerRequestInterception =
|
|
3
|
+
exports.puppeteerClickElements = exports.puppeteerUtils = exports.puppeteerRequestInterception = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
tslib_1.__exportStar(require("@crawlee/browser"), exports);
|
|
6
6
|
tslib_1.__exportStar(require("./internals/puppeteer-crawler"), exports);
|
|
7
7
|
tslib_1.__exportStar(require("./internals/puppeteer-launcher"), exports);
|
|
8
|
-
exports.puppeteerStealth = tslib_1.__importStar(require("./internals/stealth"));
|
|
9
8
|
exports.puppeteerRequestInterception = tslib_1.__importStar(require("./internals/utils/puppeteer_request_interception"));
|
|
10
9
|
exports.puppeteerUtils = tslib_1.__importStar(require("./internals/utils/puppeteer_utils"));
|
|
11
10
|
exports.puppeteerClickElements = tslib_1.__importStar(require("./internals/enqueue-links/click-elements"));
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,2DAAiC;AACjC,wEAA8C;AAC9C,yEAA+C;AAE/C,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,2DAAiC;AACjC,wEAA8C;AAC9C,yEAA+C;AAE/C,yHAAiG;AAGjG,4FAAoE;AAWpE,2GAAmF"}
|
package/index.mjs
CHANGED
|
@@ -61,16 +61,11 @@ export const launchPuppeteer = mod.launchPuppeteer;
|
|
|
61
61
|
export const log = mod.log;
|
|
62
62
|
export const maybeStringify = mod.maybeStringify;
|
|
63
63
|
export const mergeCookies = mod.mergeCookies;
|
|
64
|
-
export const openDataset = mod.openDataset;
|
|
65
64
|
export const puppeteerClickElements = mod.puppeteerClickElements;
|
|
66
65
|
export const puppeteerRequestInterception = mod.puppeteerRequestInterception;
|
|
67
|
-
export const puppeteerStealth = mod.puppeteerStealth;
|
|
68
66
|
export const puppeteerUtils = mod.puppeteerUtils;
|
|
69
|
-
export const pushData = mod.pushData;
|
|
70
67
|
export const resolveBaseUrl = mod.resolveBaseUrl;
|
|
71
68
|
export const serializeArray = mod.serializeArray;
|
|
72
|
-
export const storage = mod.storage;
|
|
73
|
-
export const throwOnBlockedRequest = mod.throwOnBlockedRequest;
|
|
74
69
|
export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
|
|
75
70
|
export const validateGlobPattern = mod.validateGlobPattern;
|
|
76
71
|
export const validators = mod.validators;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, RequestQueue
|
|
1
|
+
import { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, RequestQueue } from '@crawlee/browser';
|
|
2
2
|
import { Dictionary } from '@crawlee/utils';
|
|
3
3
|
import { ClickOptions, Page, Target } from 'puppeteer';
|
|
4
|
+
import { BatchAddRequestsResult } from '@crawlee/types';
|
|
4
5
|
export interface EnqueueLinksByClickingElementsOptions {
|
|
5
6
|
/**
|
|
6
7
|
* Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
|
|
@@ -153,7 +154,7 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
153
154
|
*
|
|
154
155
|
* @returns Promise that resolves to {@link BatchAddRequestsResult} object.
|
|
155
156
|
*/
|
|
156
|
-
export declare function enqueueLinksByClickingElements(options: EnqueueLinksByClickingElementsOptions): Promise<
|
|
157
|
+
export declare function enqueueLinksByClickingElements(options: EnqueueLinksByClickingElementsOptions): Promise<BatchAddRequestsResult>;
|
|
157
158
|
interface WaitForPageIdleOptions {
|
|
158
159
|
page: Page;
|
|
159
160
|
waitForPageIdleMillis?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"click-elements.d.ts","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":"AAAA,OAAO,EAMH,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAEhB,YAAY,
|
|
1
|
+
{"version":3,"file":"click-elements.d.ts","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":"AAAA,OAAO,EAMH,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAEhB,YAAY,EAEf,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE5C,OAAO,EAGH,YAAY,EAGZ,IAAI,EAEJ,MAAM,EACT,MAAM,WAAW,CAAC;AAEnB,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAMxD,MAAM,WAAW,qCAAqC;IAClD;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;IAEX;;OAEG;IACH,YAAY,EAAE,YAAY,CAAC;IAE3B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;;OAkBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACnC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAsB,8BAA8B,CAAC,OAAO,EAAE,qCAAqC,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAmEpI;AAED,UAAU,sBAAsB;IAC5B,IAAI,EAAE,IAAI,CAAC;IACX,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,UAAU,kDAAmD,SAAQ,sBAAsB;IACvF,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,YAAY,CAAC;CAC/B;AAED;;;;;GAKG;AACH,wBAAsB,2CAA2C,CAAC,OAAO,EAAE,kDAAkD,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAiCpJ;AAoDD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAGpE;AAiDD;;;;;;;GAOG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAuB5G"}
|
|
@@ -106,14 +106,14 @@ async function clickElementsAndInterceptNavigationRequests(options) {
|
|
|
106
106
|
const onTargetCreated = createTargetCreatedHandler(page, uniqueRequests);
|
|
107
107
|
const onFrameNavigated = createFrameNavigatedHandler(page, uniqueRequests);
|
|
108
108
|
await (0, puppeteer_request_interception_1.addInterceptRequestHandler)(page, onInterceptedRequest);
|
|
109
|
-
browser.on("targetcreated" /* TargetCreated */, onTargetCreated);
|
|
110
|
-
page.on("framenavigated" /* FrameNavigated */, onFrameNavigated);
|
|
109
|
+
browser.on("targetcreated" /* BrowserEmittedEvents.TargetCreated */, onTargetCreated);
|
|
110
|
+
page.on("framenavigated" /* PageEmittedEvents.FrameNavigated */, onFrameNavigated);
|
|
111
111
|
await preventHistoryNavigation(page);
|
|
112
112
|
await clickElements(page, selector, clickOptions);
|
|
113
113
|
await waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis });
|
|
114
114
|
await restoreHistoryNavigationAndSaveCapturedUrls(page, uniqueRequests);
|
|
115
|
-
browser.off("targetcreated" /* TargetCreated */, onTargetCreated);
|
|
116
|
-
page.off("framenavigated" /* FrameNavigated */, onFrameNavigated);
|
|
115
|
+
browser.off("targetcreated" /* BrowserEmittedEvents.TargetCreated */, onTargetCreated);
|
|
116
|
+
page.off("framenavigated" /* PageEmittedEvents.FrameNavigated */, onFrameNavigated);
|
|
117
117
|
await (0, puppeteer_request_interception_1.removeInterceptRequestHandler)(page, onInterceptedRequest);
|
|
118
118
|
const serializedRequests = Array.from(uniqueRequests);
|
|
119
119
|
return serializedRequests.map((r) => JSON.parse(r));
|
|
@@ -194,7 +194,7 @@ function createFrameNavigatedHandler(page, requests) {
|
|
|
194
194
|
async function preventHistoryNavigation(page) {
|
|
195
195
|
/* istanbul ignore next */
|
|
196
196
|
return page.evaluate(() => {
|
|
197
|
-
window.__originalHistory__ = window.history;
|
|
197
|
+
window.__originalHistory__ = window.history;
|
|
198
198
|
delete window.history; // Simple override does not work.
|
|
199
199
|
window.history = {
|
|
200
200
|
stateHistory: [],
|
|
@@ -299,27 +299,26 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
|
|
|
299
299
|
finish();
|
|
300
300
|
}
|
|
301
301
|
function finish() {
|
|
302
|
-
page.off("request" /* Request */, activityHandler);
|
|
303
|
-
page.off("framenavigated" /* FrameNavigated */, activityHandler);
|
|
304
|
-
context.off("targetcreated" /* TargetCreated */, newTabTracker);
|
|
302
|
+
page.off("request" /* PageEmittedEvents.Request */, activityHandler);
|
|
303
|
+
page.off("framenavigated" /* PageEmittedEvents.FrameNavigated */, activityHandler);
|
|
304
|
+
context.off("targetcreated" /* BrowserContextEmittedEvents.TargetCreated */, newTabTracker);
|
|
305
305
|
resolve();
|
|
306
306
|
}
|
|
307
307
|
maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
|
|
308
308
|
activityHandler(); // We call this once manually in case there would be no requests at all.
|
|
309
|
-
page.on("request" /* Request */, activityHandler);
|
|
310
|
-
page.on("framenavigated" /* FrameNavigated */, activityHandler);
|
|
311
|
-
context.on("targetcreated" /* TargetCreated */, newTabTracker);
|
|
309
|
+
page.on("request" /* PageEmittedEvents.Request */, activityHandler);
|
|
310
|
+
page.on("framenavigated" /* PageEmittedEvents.FrameNavigated */, activityHandler);
|
|
311
|
+
context.on("targetcreated" /* BrowserContextEmittedEvents.TargetCreated */, newTabTracker);
|
|
312
312
|
});
|
|
313
313
|
}
|
|
314
314
|
/**
|
|
315
315
|
* @ignore
|
|
316
316
|
*/
|
|
317
317
|
async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {
|
|
318
|
-
/* eslint-disable no-shadow */
|
|
319
318
|
/* istanbul ignore next */
|
|
320
319
|
const state = await page.evaluate(() => {
|
|
321
320
|
const { stateHistory } = window.history;
|
|
322
|
-
window.history = window.__originalHistory__;
|
|
321
|
+
window.history = window.__originalHistory__;
|
|
323
322
|
return stateHistory;
|
|
324
323
|
});
|
|
325
324
|
state.forEach((args) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"click-elements.js","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":";;;;AAAA,8CAa0B;AAC1B,6DAA8B;AAE9B,oDAAoB;AAWpB,6BAA0B;
|
|
1
|
+
{"version":3,"file":"click-elements.js","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":";;;;AAAA,8CAa0B;AAC1B,6DAA8B;AAE9B,oDAAoB;AAWpB,6BAA0B;AAE1B,4FAAoH;AAEpH,MAAM,gBAAgB,GAAG,UAAU,CAAC;AACpC,MAAM,GAAG,GAAG,aAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC,CAAC;AA2H/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACI,KAAK,UAAU,8BAA8B,CAAC,OAA8C;IAC/F,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC;QAC3C,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,QAAQ,EAAE,YAAE,CAAC,MAAM;QACnB,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC;QAC/D,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACvC,sBAAsB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC7C,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,IAAI,EACJ,YAAY,EACZ,QAAQ,EACR,YAAY,EACZ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,mBAAmB,GAAG,CAAC,EACvB,sBAAsB,GAAG,CAAC,GAC7B,GAAG,OAAO,CAAC;IAEZ,MAAM,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;IACzD,MAAM,wBAAwB,GAAG,sBAAsB,GAAG,IAAI,CAAC;IAE/D,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,GAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,8CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,uCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,2CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,MAAM,mBAAmB,GAAG,MAAM,2CAA2C,CAAC;QAC1E,IAAI;QACJ,QAAQ;QACR,qBAAqB;QACrB,wBAAwB;QACxB,YAAY;KACf,CAAC,CAAC;IACH,IAAI,cAAc,GAAG,IAAA,8BAAoB,EAAC,mBAAmB,CAAC,CAAC;IAC/D,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;KACxG;IACD,MAAM,QAAQ,GAAG,IAAA,wBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACnE,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAnED,wEAmEC;AAaD;;;;;GAKG;AACI,KAAK,UAAU,2CAA2C,CAAC,OAA2D;IACzH,MAAM,EACF,IAAI,EACJ,QAAQ,EACR,qBAAqB,EACrB,wBAAwB,EACxB,YAAY,GACf,GAAG,OAAO,CAAC;IAEZ,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAE/B,MAAM,oBAAoB,GAAG,6BAA6B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACjF,MAAM,eAAe,GAAG,0BAA0B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACzE,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAE3E,MAAM,IAAA,2DAA0B,EAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,2DAAqC,eAAe,CAAC,CAAC;IAChE,IAAI,CAAC,EAAE,0DAAmC,gBAAgB,CAAC,CAAC;IAE5D,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAClD,MAAM,eAAe,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAEjF,MAAM,2CAA2C,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAExE,OAAO,CAAC,GAAG,2DAAqC,eAAe,CAAC,CAAC;IACjE,IAAI,CAAC,GAAG,0DAAmC,gBAAgB,CAAC,CAAC;IAC7D,MAAM,IAAA,8DAA6B,EAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAEhE,MAAM,kBAAkB,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACtD,OAAO,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAjCD,kGAiCC;AAED;;GAEG;AACH,SAAS,6BAA6B,CAAC,IAAU,EAAE,QAAqB;IACpE,OAAO,KAAK,UAAU,oBAAoB,CAAC,GAAG;QAC1C,IAAI,CAAC,2BAA2B,CAAC,IAAI,EAAE,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;QACnE,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC;QACtB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;YACxB,GAAG;YACH,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE;YACtB,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE;YACpB,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE;SAC1B,CAAC,CAAC,CAAC;QAEJ,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC,MAAM,EAAE;YAC5B,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,4BAA4B;SAChE;aAAM;YACH,MAAM,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B;SAC3D;IACL,CAAC,CAAC;AACN,CAAC;AAED;;GAEG;AACH,SAAS,2BAA2B,CAAC,IAAU,EAAE,GAAqB;IAClE,OAAO,GAAG,CAAC,mBAAmB,EAAE;WACzB,GAAG,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;AAC5C,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B,CAAC,IAAU,EAAE,QAAqB;IACjE,OAAO,KAAK,UAAU,eAAe,CAAC,MAAM;QACxC,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC;YAAE,OAAO;QAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;QACzB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;QAEtC,iDAAiD;QACjD,sCAAsC;QACtC,IAAI;YACA,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,WAAY,CAAC,KAAK,EAAE,CAAC;SAC9B;QAAC,OAAO,GAAG,EAAE;YACV,GAAG,CAAC,KAAK,CAAC,+DAA+D,EAAE,EAAE,KAAK,EAAG,GAAa,CAAC,KAAK,EAAE,CAAC,CAAC;SAC/G;IACL,CAAC,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,SAAgB,gBAAgB,CAAC,IAAU,EAAE,MAAc;IACvD,OAAO,MAAM,CAAC,IAAI,EAAE,KAAK,MAAM;WACxB,IAAI,CAAC,MAAM,EAAE,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;AAC7C,CAAC;AAHD,4CAGC;AAED;;GAEG;AACH,SAAS,2BAA2B,CAAC,IAAU,EAAE,QAAqB;IAClE,OAAO,SAAS,gBAAgB,CAAC,KAAK;QAClC,IAAI,KAAK,KAAK,IAAI,CAAC,SAAS,EAAE;YAAE,OAAO;QACvC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;QACxB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC;AACN,CAAC;AAaD;;GAEG;AACH,KAAK,UAAU,wBAAwB,CAAC,IAAU;IAC9C,0BAA0B;IAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACrB,MAAgC,CAAC,mBAAmB,GAAG,MAAM,CAAC,OAAO,CAAC;QACvE,OAAQ,MAAgC,CAAC,OAAO,CAAC,CAAC,iCAAiC;QAClF,MAAgC,CAAC,OAAO,GAAG;YACxC,YAAY,EAAE,EAAE;YAChB,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,EAAE;YACT,EAAE,KAAI,CAAC;YACP,IAAI,KAAI,CAAC;YACT,OAAO,KAAI,CAAC;YACZ,SAAS,CAAC,GAAG,IAAe;gBACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;YACD,YAAY,CAAC,GAAG,IAAe;gBAC3B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;SACW,CAAC;IACrB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;;;GAOG;AACI,KAAK,UAAU,aAAa,CAAC,IAAU,EAAE,QAAgB,EAAE,YAA2B;IACzF,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC/C,GAAG,CAAC,KAAK,CAAC,6CAA6C,cAAc,CAAC,MAAM,qBAAqB,CAAC,CAAC;IACnG,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,MAAM,GAAG,gBAAgB,CAAC;IAC9B,IAAI,gBAAgB,GAAG,IAAI,CAAC;IAC5B,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;QACjC,IAAI;YACA,MAAM,IAAI,CAAC,QAAQ,CAAC,kCAAkC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1E,MAAM,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACjC,oBAAoB,EAAE,CAAC;SAC1B;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,CAAC,GAAG,GAAY,CAAC;YACvB,IAAI,gBAAgB,IAAI,CAAC,CAAC,KAAM,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE;gBACpE,GAAG,CAAC,OAAO,CAAC,4BAA4B,QAAQ,+DAA+D;sBACzG,qHAAqH;sBACrH,uHAAuH,CAAC,CAAC;gBAC/H,gBAAgB,GAAG,KAAK,CAAC;aAC5B;YACD,GAAG,CAAC,KAAK,CAAC,+CAA+C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;SAClF;KACJ;IACD,GAAG,CAAC,KAAK,CAAC,wDAAwD,oBAAoB,oBAAoB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;AACvI,CAAC;AAvBD,sCAuBC;AAED,0BAA0B;AAC1B;;GAEG;AACH,SAAS,kCAAkC,CAAC,EAAe,EAAE,MAAc;IACvE,EAAE,CAAC,KAAK,CAAC,UAAU,GAAG,SAAS,CAAC;IAChC,EAAE,CAAC,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,EAAE,CAAC,KAAK,CAAC,QAAQ,GAAG,OAAO,CAAC;IAC5B,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IACjC,EAAE,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC;IACpB,EAAE,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC;IACnB,MAAM,YAAY,GAAG,EAAE,CAAC,qBAAqB,EAAE,CAAC;IAChD,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IACnD,IAAI,CAAC,YAAY,CAAC,KAAK;QAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC;AACrD,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,eAAe,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,wBAAwB,EAA0B;IAC5G,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACjC,IAAI,OAAuB,CAAC;QAC5B,IAAI,UAA0B,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtC,SAAS,aAAa,CAAC,MAAc;YACjC,IAAI,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC;gBAAE,eAAe,EAAE,CAAC;QAC1D,CAAC;QAED,SAAS,eAAe;YACpB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBACtB,YAAY,CAAC,UAAU,CAAC,CAAC;gBACzB,MAAM,EAAE,CAAC;YACb,CAAC,EAAE,qBAAqB,CAAC,CAAC;QAC9B,CAAC;QAED,SAAS,iBAAiB;YACtB,GAAG,CAAC,KAAK,CAAC,oEAAoE,wBAAwB,MAAM;kBACtG,4GAA4G,CAAC,CAAC;YACpH,MAAM,EAAE,CAAC;QACb,CAAC;QAED,SAAS,MAAM;YACX,IAAI,CAAC,GAAG,4CAA4B,eAAe,CAAC,CAAC;YACrD,IAAI,CAAC,GAAG,0DAAmC,eAAe,CAAC,CAAC;YAC5D,OAAO,CAAC,GAAG,kEAA4C,aAAa,CAAC,CAAC;YACtE,OAAO,EAAE,CAAC;QACd,CAAC;QAED,UAAU,GAAG,UAAU,CAAC,iBAAiB,EAAE,wBAAwB,CAAC,CAAC;QACrE,eAAe,EAAE,CAAC,CAAC,wEAAwE;QAC3F,IAAI,CAAC,EAAE,4CAA4B,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,EAAE,0DAAmC,eAAe,CAAC,CAAC;QAC3D,OAAO,CAAC,EAAE,kEAA4C,aAAa,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,2CAA2C,CAAC,IAAU,EAAE,QAAqB;IACxF,0BAA0B;IAC1B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACnC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC,OAAiC,CAAC;QACjE,MAAgC,CAAC,OAAO,GAAI,MAAgC,CAAC,mBAAmB,CAAC;QAClG,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI;YACA,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAW,CAAC;YACjD,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC;YAC/C,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;SACzC;QAAC,OAAO,GAAG,EAAE;YACV,GAAG,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAG,GAAa,CAAC,KAAK,EAAE,CAAC,CAAC;SAC5F;IACL,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -2,14 +2,14 @@ import { BrowserCrawler, BrowserCrawlerHandleRequest, BrowserCrawlerOptions, Bro
|
|
|
2
2
|
import { PuppeteerPlugin } from '@crawlee/browser-pool';
|
|
3
3
|
import { HTTPResponse, LaunchOptions, Page } from 'puppeteer';
|
|
4
4
|
import { PuppeteerLaunchContext } from './puppeteer-launcher';
|
|
5
|
-
import { DirectNavigationOptions } from './utils/puppeteer_utils';
|
|
5
|
+
import { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer_utils';
|
|
6
6
|
export declare type PuppeteerController = ReturnType<PuppeteerPlugin['_createController']>;
|
|
7
|
-
export declare type
|
|
8
|
-
export declare type PuppeteerHook = BrowserHook<
|
|
7
|
+
export declare type PuppeteerCrawlingContext = BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController> & PuppeteerContextUtils;
|
|
8
|
+
export declare type PuppeteerHook = BrowserHook<PuppeteerCrawlingContext, PuppeteerGoToOptions>;
|
|
9
9
|
export declare type PuppeteerRequestHandlerParam = BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController>;
|
|
10
10
|
export declare type PuppeteerRequestHandler = BrowserCrawlerHandleRequest<PuppeteerRequestHandlerParam>;
|
|
11
11
|
export declare type PuppeteerGoToOptions = Parameters<Page['goto']>[1];
|
|
12
|
-
export interface PuppeteerCrawlerOptions extends BrowserCrawlerOptions<
|
|
12
|
+
export interface PuppeteerCrawlerOptions extends BrowserCrawlerOptions<PuppeteerCrawlingContext, {
|
|
13
13
|
browserPlugins: [PuppeteerPlugin];
|
|
14
14
|
}> {
|
|
15
15
|
/**
|
|
@@ -111,16 +111,14 @@ export interface PuppeteerCrawlerOptions extends BrowserCrawlerOptions<Puppeteer
|
|
|
111
111
|
*/
|
|
112
112
|
export declare class PuppeteerCrawler extends BrowserCrawler<{
|
|
113
113
|
browserPlugins: [PuppeteerPlugin];
|
|
114
|
-
}, LaunchOptions,
|
|
114
|
+
}, LaunchOptions, PuppeteerCrawlingContext> {
|
|
115
115
|
protected static optionsShape: {
|
|
116
116
|
browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
117
|
-
launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
118
117
|
handlePageFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
119
|
-
gotoFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
120
|
-
gotoTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
121
118
|
navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
122
119
|
preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
|
|
123
120
|
postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
|
|
121
|
+
launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
124
122
|
sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
125
123
|
persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
126
124
|
useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
@@ -144,6 +142,7 @@ export declare class PuppeteerCrawler extends BrowserCrawler<{
|
|
|
144
142
|
* All `PuppeteerCrawler` parameters are passed via an options object.
|
|
145
143
|
*/
|
|
146
144
|
constructor(options: PuppeteerCrawlerOptions);
|
|
147
|
-
protected
|
|
145
|
+
protected _runRequestHandler(context: PuppeteerCrawlingContext): Promise<void>;
|
|
146
|
+
protected _navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<HTTPResponse | null>;
|
|
148
147
|
}
|
|
149
148
|
//# sourceMappingURL=puppeteer-crawler.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"puppeteer-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,cAAc,EACd,2BAA2B,EAC3B,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,EACd,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"puppeteer-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,cAAc,EACd,2BAA2B,EAC3B,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,EACd,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAsB,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE5E,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAqB,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAE,uBAAuB,EAAgB,qBAAqB,EAA0B,MAAM,yBAAyB,CAAC;AAE/H,oBAAY,mBAAmB,GAAG,UAAU,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC;AACnF,oBAAY,wBAAwB,GAAG,sBAAsB,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,GAAG,qBAAqB,CAAC;AAC/H,oBAAY,aAAa,GAAG,WAAW,CAAC,wBAAwB,EAAE,oBAAoB,CAAC,CAAC;AACxF,oBAAY,4BAA4B,GAAG,sBAAsB,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAC;AAC3G,oBAAY,uBAAuB,GAAG,2BAA2B,CAAC,4BAA4B,CAAC,CAAC;AAChG,oBAAY,oBAAoB,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,uBAAwB,SAAQ,qBAAqB,CAClE,wBAAwB,EACxB;IAAE,cAAc,EAAE,CAAC,eAAe,CAAC,CAAA;CAAE,CACxC;IACG;;OAEG;IACH,aAAa,CAAC,EAAE,sBAAsB,CAAC;IAEvC;;;;;;;;;;;;;OAaG;IACH,kBAAkB,CAAC,EAAE,aAAa,EAAE,CAAC;IAErC;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,aAAa,EAAE,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4DG;AACH,qBAAa,gBAAiB,SAAQ,cAAc,CAAC;IAAE,cAAc,EAAE,CAAC,eAAe,CAAC,CAAA;CAAE,EAAE,aAAa,EAAE,wBAAwB,CAAC;IAChI,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;MAGpC;IAEF;;OAEG;gBACS,OAAO,EAAE,uBAAuB;cAwBnB,kBAAkB,CAAC,OAAO,EAAE,wBAAwB;cAMpD,kBAAkB,CAAC,eAAe,EAAE,wBAAwB,EAAE,WAAW,EAAE,uBAAuB;CAG9H"}
|
|
@@ -5,7 +5,6 @@ const tslib_1 = require("tslib");
|
|
|
5
5
|
const browser_1 = require("@crawlee/browser");
|
|
6
6
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
7
7
|
const puppeteer_launcher_1 = require("./puppeteer-launcher");
|
|
8
|
-
const stealth_1 = require("./stealth");
|
|
9
8
|
const puppeteer_utils_1 = require("./utils/puppeteer_utils");
|
|
10
9
|
/**
|
|
11
10
|
* Provides a simple framework for parallel crawling of web pages
|
|
@@ -75,7 +74,6 @@ class PuppeteerCrawler extends browser_1.BrowserCrawler {
|
|
|
75
74
|
constructor(options) {
|
|
76
75
|
(0, ow_1.default)(options, 'PuppeteerCrawlerOptions', ow_1.default.object.exactShape(PuppeteerCrawler.optionsShape));
|
|
77
76
|
const { launchContext = {}, browserPoolOptions = {}, proxyConfiguration, ...browserCrawlerOptions } = options;
|
|
78
|
-
const { stealth = false } = launchContext;
|
|
79
77
|
if (launchContext.proxyUrl) {
|
|
80
78
|
throw new Error('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.'
|
|
81
79
|
+ 'Use PuppeteerCrawlerOptions.proxyConfiguration');
|
|
@@ -84,22 +82,14 @@ class PuppeteerCrawler extends browser_1.BrowserCrawler {
|
|
|
84
82
|
browserPoolOptions.browserPlugins = [
|
|
85
83
|
puppeteerLauncher.createBrowserPlugin(),
|
|
86
84
|
];
|
|
87
|
-
super({ ...browserCrawlerOptions, proxyConfiguration, browserPoolOptions });
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
(0, stealth_1.applyStealthToBrowser)(browserController.browser, newStealthOptions);
|
|
94
|
-
});
|
|
95
|
-
}
|
|
96
|
-
this.launchContext = launchContext;
|
|
85
|
+
super({ ...browserCrawlerOptions, launchContext, proxyConfiguration, browserPoolOptions });
|
|
86
|
+
}
|
|
87
|
+
async _runRequestHandler(context) {
|
|
88
|
+
(0, puppeteer_utils_1.registerUtilsToContext)(context);
|
|
89
|
+
// eslint-disable-next-line no-underscore-dangle
|
|
90
|
+
await super._runRequestHandler(context);
|
|
97
91
|
}
|
|
98
92
|
async _navigationHandler(crawlingContext, gotoOptions) {
|
|
99
|
-
if (this.gotoFunction) {
|
|
100
|
-
this.log.deprecated('PuppeteerCrawlerOptions.gotoFunction is deprecated. Use "preNavigationHooks" and "postNavigationHooks" instead.');
|
|
101
|
-
return this.gotoFunction(crawlingContext, gotoOptions);
|
|
102
|
-
}
|
|
103
93
|
return (0, puppeteer_utils_1.gotoExtended)(crawlingContext.page, crawlingContext.request, gotoOptions);
|
|
104
94
|
}
|
|
105
95
|
}
|
|
@@ -111,7 +101,6 @@ Object.defineProperty(PuppeteerCrawler, "optionsShape", {
|
|
|
111
101
|
value: {
|
|
112
102
|
...browser_1.BrowserCrawler.optionsShape,
|
|
113
103
|
browserPoolOptions: ow_1.default.optional.object,
|
|
114
|
-
launchContext: ow_1.default.optional.object,
|
|
115
104
|
}
|
|
116
105
|
});
|
|
117
106
|
//# sourceMappingURL=puppeteer-crawler.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"puppeteer-crawler.js","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":";;;;AAAA,8CAM0B;
|
|
1
|
+
{"version":3,"file":"puppeteer-crawler.js","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":";;;;AAAA,8CAM0B;AAE1B,oDAAoB;AAEpB,6DAAiF;AACjF,6DAA+H;AAoD/H;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4DG;AACH,MAAa,gBAAiB,SAAQ,wBAA8F;IAMhI;;OAEG;IACH,YAAY,OAAgC;QACxC,IAAA,YAAE,EAAC,OAAO,EAAE,yBAAyB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE5F,MAAM,EACF,aAAa,GAAG,EAAE,EAClB,kBAAkB,GAAG,EAAwB,EAC7C,kBAAkB,EAClB,GAAG,qBAAqB,EAC3B,GAAG,OAAO,CAAC;QAEZ,IAAI,aAAa,CAAC,QAAQ,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,oFAAoF;kBAC9F,gDAAgD,CAAC,CAAC;SAC3D;QAED,MAAM,iBAAiB,GAAG,IAAI,sCAAiB,CAAC,aAAa,CAAC,CAAC;QAE/D,kBAAkB,CAAC,cAAc,GAAG;YAChC,iBAAiB,CAAC,mBAAmB,EAAE;SAC1C,CAAC;QAEF,KAAK,CAAC,EAAE,GAAG,qBAAqB,EAAE,aAAa,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC/F,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CAAC,OAAiC;QACzE,IAAA,wCAAsB,EAAC,OAAO,CAAC,CAAC;QAChC,gDAAgD;QAChD,MAAM,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CAAC,eAAyC,EAAE,WAAoC;QACvH,OAAO,IAAA,8BAAY,EAAC,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACpF,CAAC;;AAzCL,4CA0CC;AAzCG;;;;WAAyC;QACrC,GAAG,wBAAc,CAAC,YAAY;QAC9B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACzC;GAAC"}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { Browser } from 'puppeteer';
|
|
2
2
|
import { PuppeteerPlugin } from '@crawlee/browser-pool';
|
|
3
3
|
import { BrowserLaunchContext, BrowserLauncher } from '@crawlee/browser';
|
|
4
|
-
import { StealthOptions } from './stealth';
|
|
5
4
|
/**
|
|
6
5
|
* Apify extends the launch options of Puppeteer.
|
|
7
6
|
* You can use any of the Puppeteer compatible
|
|
@@ -35,12 +34,6 @@ export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPl
|
|
|
35
34
|
* Example: `http://bob:pass123@proxy.example.com:1234`.
|
|
36
35
|
*/
|
|
37
36
|
proxyUrl?: string;
|
|
38
|
-
/**
|
|
39
|
-
* The `User-Agent` HTTP header used by the browser.
|
|
40
|
-
* If not provided, the function sets `User-Agent` to a reasonable default
|
|
41
|
-
* to reduce the chance of detection of the crawler.
|
|
42
|
-
*/
|
|
43
|
-
userAgent?: string;
|
|
44
37
|
/**
|
|
45
38
|
* If `true` and `executablePath` is not set,
|
|
46
39
|
* Puppeteer will launch full Google Chrome browser available on the machine
|
|
@@ -65,18 +58,6 @@ export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPl
|
|
|
65
58
|
* @default false
|
|
66
59
|
*/
|
|
67
60
|
useIncognitoPages?: boolean;
|
|
68
|
-
/**
|
|
69
|
-
* This setting hides most of the known properties that identify headless Chrome and makes it nearly undetectable.
|
|
70
|
-
* It is recommended to use it together with the `useChrome` set to `true`.
|
|
71
|
-
* @deprecated
|
|
72
|
-
*/
|
|
73
|
-
stealth?: boolean;
|
|
74
|
-
/**
|
|
75
|
-
* Using this configuration, you can disable some of the hiding tricks.
|
|
76
|
-
* For these settings to take effect `stealth` must be set to true
|
|
77
|
-
* @deprecated
|
|
78
|
-
*/
|
|
79
|
-
stealthOptions?: StealthOptions;
|
|
80
61
|
}
|
|
81
62
|
/**
|
|
82
63
|
* `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
|
|
@@ -85,25 +66,17 @@ export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPl
|
|
|
85
66
|
export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown> {
|
|
86
67
|
protected static optionsShape: {
|
|
87
68
|
launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
88
|
-
userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
89
|
-
stealth: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
90
|
-
stealthOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
91
69
|
proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
92
70
|
useChrome: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
93
71
|
useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
94
72
|
userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
95
73
|
launchOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
74
|
+
userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
96
75
|
};
|
|
97
|
-
stealth?: boolean;
|
|
98
|
-
stealthOptions: StealthOptions;
|
|
99
76
|
/**
|
|
100
77
|
* All `PuppeteerLauncher` parameters are passed via an launchContext object.
|
|
101
78
|
*/
|
|
102
79
|
constructor(launchContext?: PuppeteerLaunchContext);
|
|
103
|
-
launch(): Promise<import("packages/browser-pool/dist/puppeteer/puppeteer-proxy-per-page").Browser>;
|
|
104
|
-
createLaunchOptions(): {
|
|
105
|
-
args?: string[] | undefined;
|
|
106
|
-
} & import("packages/utils/dist").Dictionary<unknown>;
|
|
107
80
|
}
|
|
108
81
|
/**
|
|
109
82
|
* Launches headless Chrome using Puppeteer pre-configured to work within the Apify platform.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"puppeteer-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-launcher.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEzE
|
|
1
|
+
{"version":3,"file":"puppeteer-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-launcher.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEzE;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB,CAAC,eAAe,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC3G;;OAEG;IACH,aAAa,CAAC,EAAE,eAAe,CAAC,eAAe,CAAC,CAAC;IAEjD;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED;;;GAGG;AACH,qBAAa,iBAAkB,SAAQ,eAAe,CAAC,eAAe,EAAE,OAAO,CAAC;IAC5E,iBAA0B,YAAY;;;;;;;;MAGpC;IAEF;;OAEG;gBACS,aAAa,GAAE,sBAA2B;CAezD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAsB,eAAe,CAAC,aAAa,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAI9F"}
|
|
@@ -5,17 +5,6 @@ const tslib_1 = require("tslib");
|
|
|
5
5
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
6
6
|
const browser_pool_1 = require("@crawlee/browser-pool");
|
|
7
7
|
const browser_1 = require("@crawlee/browser");
|
|
8
|
-
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
9
|
-
const stealth_1 = require("./stealth");
|
|
10
|
-
/**
|
|
11
|
-
* The default user agent used by `Actor.launchPuppeteer`.
|
|
12
|
-
* Last updated on 2020-05-22.
|
|
13
|
-
*/
|
|
14
|
-
const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Safari/537.36';
|
|
15
|
-
const LAUNCH_PUPPETEER_DEFAULT_VIEWPORT = {
|
|
16
|
-
width: 1366,
|
|
17
|
-
height: 768,
|
|
18
|
-
};
|
|
19
8
|
/**
|
|
20
9
|
* `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
|
|
21
10
|
* @ignore
|
|
@@ -26,75 +15,13 @@ class PuppeteerLauncher extends browser_1.BrowserLauncher {
|
|
|
26
15
|
*/
|
|
27
16
|
constructor(launchContext = {}) {
|
|
28
17
|
(0, ow_1.default)(launchContext, 'PuppeteerLauncher', ow_1.default.object.exactShape(PuppeteerLauncher.optionsShape));
|
|
29
|
-
const { launcher = browser_1.BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-puppeteer-chrome'),
|
|
18
|
+
const { launcher = browser_1.BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-puppeteer-chrome'), ...browserLauncherOptions } = launchContext;
|
|
30
19
|
super({
|
|
31
20
|
...browserLauncherOptions,
|
|
32
21
|
launcher,
|
|
33
22
|
});
|
|
34
|
-
Object.defineProperty(this, "stealth", {
|
|
35
|
-
enumerable: true,
|
|
36
|
-
configurable: true,
|
|
37
|
-
writable: true,
|
|
38
|
-
value: void 0
|
|
39
|
-
});
|
|
40
|
-
Object.defineProperty(this, "stealthOptions", {
|
|
41
|
-
enumerable: true,
|
|
42
|
-
configurable: true,
|
|
43
|
-
writable: true,
|
|
44
|
-
value: void 0
|
|
45
|
-
});
|
|
46
|
-
this.userAgent = userAgent;
|
|
47
|
-
this.stealth = stealth;
|
|
48
|
-
this.stealthOptions = {
|
|
49
|
-
hideWebDriver: true,
|
|
50
|
-
...stealthOptions,
|
|
51
|
-
};
|
|
52
23
|
this.Plugin = browser_pool_1.PuppeteerPlugin;
|
|
53
24
|
}
|
|
54
|
-
async launch() {
|
|
55
|
-
const browser = await super.launch();
|
|
56
|
-
if (this.stealth) {
|
|
57
|
-
log_1.default.deprecated('Puppeteer "stealth" and "stealthOptions" are deprecated.'
|
|
58
|
-
+ ' You should use fingerprints instead.'
|
|
59
|
-
+ ' Checkout the fingerprints guide: https://sdk.apify.com/docs/guides/avoid-blocking');
|
|
60
|
-
const { hideWebDriver, ...newStealthOptions } = this.stealthOptions;
|
|
61
|
-
(0, stealth_1.applyStealthToBrowser)(browser, newStealthOptions);
|
|
62
|
-
}
|
|
63
|
-
return browser;
|
|
64
|
-
}
|
|
65
|
-
createLaunchOptions() {
|
|
66
|
-
const launchOptions = super.createLaunchOptions();
|
|
67
|
-
launchOptions.args = launchOptions.args || [];
|
|
68
|
-
try {
|
|
69
|
-
// eslint-disable-next-line
|
|
70
|
-
const { isAtHome } = require('apify');
|
|
71
|
-
if (isAtHome())
|
|
72
|
-
launchOptions.args.push('--no-sandbox');
|
|
73
|
-
// eslint-disable-next-line no-empty
|
|
74
|
-
}
|
|
75
|
-
catch { }
|
|
76
|
-
launchOptions.defaultViewport ?? (launchOptions.defaultViewport = LAUNCH_PUPPETEER_DEFAULT_VIEWPORT);
|
|
77
|
-
// When User-Agent is not set and we're using Chromium or headless mode,
|
|
78
|
-
// it is better to use DEFAULT_USER_AGENT to reduce chance of detection
|
|
79
|
-
let { userAgent } = this;
|
|
80
|
-
if (!userAgent && (!launchOptions.executablePath || launchOptions.headless)) {
|
|
81
|
-
userAgent = DEFAULT_USER_AGENT;
|
|
82
|
-
}
|
|
83
|
-
if (userAgent) {
|
|
84
|
-
launchOptions.args.push(`--user-agent=${userAgent}`);
|
|
85
|
-
}
|
|
86
|
-
if (this.stealthOptions?.hideWebDriver) {
|
|
87
|
-
const idx = launchOptions.args.findIndex((arg) => arg.startsWith('--disable-blink-features='));
|
|
88
|
-
if (idx !== -1) {
|
|
89
|
-
const arg = launchOptions.args[idx];
|
|
90
|
-
launchOptions.args[idx] = `${arg},AutomationControlled`;
|
|
91
|
-
}
|
|
92
|
-
else {
|
|
93
|
-
launchOptions.args.push('--disable-blink-features=AutomationControlled');
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
return launchOptions;
|
|
97
|
-
}
|
|
98
25
|
}
|
|
99
26
|
exports.PuppeteerLauncher = PuppeteerLauncher;
|
|
100
27
|
Object.defineProperty(PuppeteerLauncher, "optionsShape", {
|
|
@@ -104,9 +31,6 @@ Object.defineProperty(PuppeteerLauncher, "optionsShape", {
|
|
|
104
31
|
value: {
|
|
105
32
|
...browser_1.BrowserLauncher.optionsShape,
|
|
106
33
|
launcher: ow_1.default.optional.object,
|
|
107
|
-
userAgent: ow_1.default.optional.string,
|
|
108
|
-
stealth: ow_1.default.optional.boolean,
|
|
109
|
-
stealthOptions: ow_1.default.optional.object,
|
|
110
34
|
}
|
|
111
35
|
});
|
|
112
36
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"puppeteer-launcher.js","sourceRoot":"","sources":["../../src/internals/puppeteer-launcher.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AAEpB,wDAAwD;AACxD,8CAAyE;
|
|
1
|
+
{"version":3,"file":"puppeteer-launcher.js","sourceRoot":"","sources":["../../src/internals/puppeteer-launcher.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AAEpB,wDAAwD;AACxD,8CAAyE;AAiEzE;;;GAGG;AACH,MAAa,iBAAkB,SAAQ,yBAAyC;IAM5E;;OAEG;IACH,YAAY,gBAAwC,EAAE;QAClD,IAAA,YAAE,EAAC,aAAa,EAAE,mBAAmB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE7F,MAAM,EACF,QAAQ,GAAG,yBAAe,CAAC,sBAAsB,CAAC,WAAW,EAAE,mCAAmC,CAAC,EACnG,GAAG,sBAAsB,EAC5B,GAAG,aAAa,CAAC;QAElB,KAAK,CAAC;YACF,GAAG,sBAAsB;YACzB,QAAQ;SACX,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,GAAG,8BAAe,CAAC;IAClC,CAAC;;AAvBL,8CAwBC;AAvBG;;;;WAAyC;QACrC,GAAG,yBAAe,CAAC,YAAY;QAC/B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC/B;GAAC;AAsBN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACI,KAAK,UAAU,eAAe,CAAC,aAAsC;IACxE,MAAM,iBAAiB,GAAG,IAAI,iBAAiB,CAAC,aAAa,CAAC,CAAC;IAE/D,OAAO,iBAAiB,CAAC,MAAM,EAAE,CAAC;AACtC,CAAC;AAJD,0CAIC"}
|
|
@@ -21,8 +21,10 @@
|
|
|
21
21
|
import { Page, HTTPResponse, ResponseForRequest } from 'puppeteer';
|
|
22
22
|
import { Request } from '@crawlee/core';
|
|
23
23
|
import { Dictionary } from '@crawlee/utils';
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
24
|
+
import { BatchAddRequestsResult } from '@crawlee/types';
|
|
25
|
+
import { enqueueLinksByClickingElements, EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements';
|
|
26
|
+
import { addInterceptRequestHandler, InterceptHandler, removeInterceptRequestHandler } from './puppeteer_request_interception';
|
|
27
|
+
import { PuppeteerCrawlingContext } from '../puppeteer-crawler';
|
|
26
28
|
export interface DirectNavigationOptions {
|
|
27
29
|
/**
|
|
28
30
|
* Maximum operation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The
|
|
@@ -94,7 +96,7 @@ export declare function injectFile(page: Page, filePath: string, options?: Injec
|
|
|
94
96
|
*
|
|
95
97
|
* **Example usage:**
|
|
96
98
|
* ```javascript
|
|
97
|
-
* await
|
|
99
|
+
* await puppeteerUtils.injectJQuery(page);
|
|
98
100
|
* const title = await page.evaluate(() => {
|
|
99
101
|
* return $('head title').text();
|
|
100
102
|
* });
|
|
@@ -279,6 +281,21 @@ export interface SaveSnapshotOptions {
|
|
|
279
281
|
* @param [options]
|
|
280
282
|
*/
|
|
281
283
|
export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions): Promise<void>;
|
|
284
|
+
export interface PuppeteerContextUtils {
|
|
285
|
+
injectFile(filePath: string, options?: InjectFileOptions): Promise<unknown>;
|
|
286
|
+
injectJQuery(): Promise<unknown>;
|
|
287
|
+
enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>): Promise<BatchAddRequestsResult>;
|
|
288
|
+
blockRequests(options?: BlockRequestsOptions): Promise<void>;
|
|
289
|
+
blockResources(resourceTypes?: string[]): Promise<void>;
|
|
290
|
+
cacheResponses(cache: Dictionary<Partial<ResponseForRequest>>, responseUrlRules: (string | RegExp)[]): Promise<void>;
|
|
291
|
+
compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
|
|
292
|
+
addInterceptRequestHandler(handler: InterceptHandler): Promise<void>;
|
|
293
|
+
removeInterceptRequestHandler(handler: InterceptHandler): Promise<void>;
|
|
294
|
+
infiniteScroll(options?: InfiniteScrollOptions): Promise<void>;
|
|
295
|
+
saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
|
|
296
|
+
}
|
|
297
|
+
/** @internal */
|
|
298
|
+
export declare function registerUtilsToContext(context: PuppeteerCrawlingContext): void;
|
|
282
299
|
export { enqueueLinksByClickingElements, addInterceptRequestHandler, removeInterceptRequestHandler, };
|
|
283
300
|
/** @internal */
|
|
284
301
|
export declare const puppeteerUtils: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"puppeteer_utils.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/puppeteer_utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;
|
|
1
|
+
{"version":3,"file":"puppeteer_utils.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/puppeteer_utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAMH,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,kBAAkB,EAAmC,MAAM,WAAW,CAAC;AAEpG,OAAO,EAAiB,OAAO,EAAc,MAAM,eAAe,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,8BAA8B,EAAE,qCAAqC,EAAE,MAAM,iCAAiC,CAAC;AACxH,OAAO,EAAE,0BAA0B,EAAE,gBAAgB,EAAE,6BAA6B,EAAE,MAAM,kCAAkC,CAAC;AAC/H,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAShE,MAAM,WAAW,uBAAuB;IACpC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,kBAAkB,GAAG,MAAM,GAAG,aAAa,CAAC;IAExD;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAC9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACjC;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,OAAO,CAAC;CACpB;AAED,oBAAY,sBAAsB,GAAG,CAAC,MAAM,EAAE,oBAAoB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAOxF;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAoBhH;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAGzD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAgBjG;AAED;;;;GAIG;AACH,eAAO,MAAM,cAAc,SAAgB,IAAI,4CAQ9C,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA4CrJ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE,UAAgC,GAAG,sBAAsB,CAcrH;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,GAAE,uBAA4B,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAuCxI;AAED,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzD;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAqGnG;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAmC/F;AAED,MAAM,WAAW,qBAAqB;IAClC,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5E,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IACjC,8BAA8B,CAAC,OAAO,EAAE,IAAI,CAAC,qCAAqC,EAAE,MAAM,GAAG,cAAc,CAAC,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC/I,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7D,cAAc,CAAC,aAAa,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,cAAc,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACrH,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,UAAU,GAAG,sBAAsB,CAAC;IAC9E,0BAA0B,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACrE,6BAA6B,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxE,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/D,YAAY,CAAC,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9D;AAED,gBAAgB;AAChB,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,wBAAwB,GAAG,IAAI,CAkB9E;AAED,OAAO,EACH,8BAA8B,EAC9B,0BAA0B,EAC1B,6BAA6B,GAChC,CAAC;AAEF,gBAAgB;AAChB,eAAO,MAAM,cAAc;;;;;2BAvagB,IAAI;;;;;;;;CAob9C,CAAC"}
|