@crawlee/puppeteer 3.13.5 → 4.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +8 -8
- package/index.d.ts.map +1 -1
- package/index.js +6 -10
- package/index.js.map +1 -1
- package/internals/enqueue-links/click-elements.js +33 -40
- package/internals/enqueue-links/click-elements.js.map +1 -1
- package/internals/puppeteer-crawler.d.ts +2 -10
- package/internals/puppeteer-crawler.d.ts.map +1 -1
- package/internals/puppeteer-crawler.js +21 -40
- package/internals/puppeteer-crawler.js.map +1 -1
- package/internals/puppeteer-launcher.d.ts +0 -2
- package/internals/puppeteer-launcher.d.ts.map +1 -1
- package/internals/puppeteer-launcher.js +15 -30
- package/internals/puppeteer-launcher.js.map +1 -1
- package/internals/utils/puppeteer_request_interception.js +12 -25
- package/internals/utils/puppeteer_request_interception.js.map +1 -1
- package/internals/utils/puppeteer_utils.d.ts +5 -5
- package/internals/utils/puppeteer_utils.d.ts.map +1 -1
- package/internals/utils/puppeteer_utils.js +90 -107
- package/internals/utils/puppeteer_utils.js.map +1 -1
- package/package.json +17 -23
- package/tsconfig.build.tsbuildinfo +1 -1
- package/index.mjs +0 -102
package/index.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
export * from '@crawlee/browser';
|
|
2
|
-
export * from './internals/puppeteer-crawler';
|
|
3
|
-
export * from './internals/puppeteer-launcher';
|
|
4
|
-
export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception';
|
|
5
|
-
export type { InterceptHandler } from './internals/utils/puppeteer_request_interception';
|
|
6
|
-
export * as puppeteerUtils from './internals/utils/puppeteer_utils';
|
|
7
|
-
export type { BlockRequestsOptions, CompiledScriptFunction, CompiledScriptParams, DirectNavigationOptions as PuppeteerDirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, SaveSnapshotOptions, } from './internals/utils/puppeteer_utils';
|
|
8
|
-
export * as puppeteerClickElements from './internals/enqueue-links/click-elements';
|
|
9
|
-
export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements';
|
|
2
|
+
export * from './internals/puppeteer-crawler.js';
|
|
3
|
+
export * from './internals/puppeteer-launcher.js';
|
|
4
|
+
export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
|
|
5
|
+
export type { InterceptHandler } from './internals/utils/puppeteer_request_interception.js';
|
|
6
|
+
export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
|
|
7
|
+
export type { BlockRequestsOptions, CompiledScriptFunction, CompiledScriptParams, DirectNavigationOptions as PuppeteerDirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, SaveSnapshotOptions, } from './internals/utils/puppeteer_utils.js';
|
|
8
|
+
export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
|
|
9
|
+
export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements.js';
|
|
10
10
|
//# sourceMappingURL=index.d.ts.map
|
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,kCAAkC,CAAC;AACjD,cAAc,mCAAmC,CAAC;AAElD,OAAO,KAAK,4BAA4B,MAAM,qDAAqD,CAAC;AACpG,YAAY,EAAE,gBAAgB,EAAE,MAAM,qDAAqD,CAAC;AAE5F,OAAO,KAAK,cAAc,MAAM,sCAAsC,CAAC;AACvE,YAAY,EACR,oBAAoB,EACpB,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,IAAI,gCAAgC,EAC3D,qBAAqB,EACrB,iBAAiB,EACjB,mBAAmB,GACtB,MAAM,sCAAsC,CAAC;AAE9C,OAAO,KAAK,sBAAsB,MAAM,6CAA6C,CAAC;AACtF,YAAY,EAAE,qCAAqC,EAAE,MAAM,6CAA6C,CAAC"}
|
package/index.js
CHANGED
|
@@ -1,11 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
tslib_1.__exportStar(require("./internals/puppeteer-launcher"), exports);
|
|
8
|
-
exports.puppeteerRequestInterception = tslib_1.__importStar(require("./internals/utils/puppeteer_request_interception"));
|
|
9
|
-
exports.puppeteerUtils = tslib_1.__importStar(require("./internals/utils/puppeteer_utils"));
|
|
10
|
-
exports.puppeteerClickElements = tslib_1.__importStar(require("./internals/enqueue-links/click-elements"));
|
|
1
|
+
export * from '@crawlee/browser';
|
|
2
|
+
export * from './internals/puppeteer-crawler.js';
|
|
3
|
+
export * from './internals/puppeteer-launcher.js';
|
|
4
|
+
export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
|
|
5
|
+
export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
|
|
6
|
+
export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
|
|
11
7
|
//# sourceMappingURL=index.js.map
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,kCAAkC,CAAC;AACjD,cAAc,mCAAmC,CAAC;AAElD,OAAO,KAAK,4BAA4B,MAAM,qDAAqD,CAAC;AAGpG,OAAO,KAAK,cAAc,MAAM,sCAAsC,CAAC;AAWvE,OAAO,KAAK,sBAAsB,MAAM,6CAA6C,CAAC"}
|
|
@@ -1,17 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
exports.clickElements = clickElements;
|
|
7
|
-
const tslib_1 = require("tslib");
|
|
8
|
-
const node_url_1 = require("node:url");
|
|
9
|
-
const browser_1 = require("@crawlee/browser");
|
|
10
|
-
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
11
|
-
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
12
|
-
const puppeteer_request_interception_1 = require("../utils/puppeteer_request_interception");
|
|
1
|
+
import { URL } from 'node:url';
|
|
2
|
+
import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, createRequests, } from '@crawlee/browser';
|
|
3
|
+
import ow from 'ow';
|
|
4
|
+
import log_ from '@apify/log';
|
|
5
|
+
import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer_request_interception.js';
|
|
13
6
|
const STARTING_Z_INDEX = 2147400000;
|
|
14
|
-
const log =
|
|
7
|
+
const log = log_.child({ prefix: 'Puppeteer Click Elements' });
|
|
15
8
|
/**
|
|
16
9
|
* The function finds elements matching a specific CSS selector in a Puppeteer page,
|
|
17
10
|
* clicks all those elements using a mouse move and a left mouse button click and intercepts
|
|
@@ -53,22 +46,22 @@ const log = log_1.default.child({ prefix: 'Puppeteer Click Elements' });
|
|
|
53
46
|
*
|
|
54
47
|
* @returns Promise that resolves to {@link BatchAddRequestsResult} object.
|
|
55
48
|
*/
|
|
56
|
-
async function enqueueLinksByClickingElements(options) {
|
|
57
|
-
(
|
|
58
|
-
page:
|
|
59
|
-
requestQueue:
|
|
60
|
-
selector:
|
|
61
|
-
userData:
|
|
62
|
-
clickOptions:
|
|
63
|
-
pseudoUrls:
|
|
64
|
-
globs:
|
|
65
|
-
regexps:
|
|
66
|
-
transformRequestFunction:
|
|
67
|
-
waitForPageIdleSecs:
|
|
68
|
-
maxWaitForPageIdleSecs:
|
|
69
|
-
label:
|
|
70
|
-
forefront:
|
|
71
|
-
skipNavigation:
|
|
49
|
+
export async function enqueueLinksByClickingElements(options) {
|
|
50
|
+
ow(options, ow.object.exactShape({
|
|
51
|
+
page: ow.object.hasKeys('goto', 'evaluate'),
|
|
52
|
+
requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
|
|
53
|
+
selector: ow.string,
|
|
54
|
+
userData: ow.optional.object,
|
|
55
|
+
clickOptions: ow.optional.object.hasKeys('clickCount', 'delay'),
|
|
56
|
+
pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
|
|
57
|
+
globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
|
|
58
|
+
regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
|
|
59
|
+
transformRequestFunction: ow.optional.function,
|
|
60
|
+
waitForPageIdleSecs: ow.optional.number,
|
|
61
|
+
maxWaitForPageIdleSecs: ow.optional.number,
|
|
62
|
+
label: ow.optional.string,
|
|
63
|
+
forefront: ow.optional.boolean,
|
|
64
|
+
skipNavigation: ow.optional.boolean,
|
|
72
65
|
}));
|
|
73
66
|
const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, } = options;
|
|
74
67
|
const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
|
|
@@ -76,13 +69,13 @@ async function enqueueLinksByClickingElements(options) {
|
|
|
76
69
|
const urlPatternObjects = [];
|
|
77
70
|
if (pseudoUrls?.length) {
|
|
78
71
|
log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
|
|
79
|
-
urlPatternObjects.push(...
|
|
72
|
+
urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
|
|
80
73
|
}
|
|
81
74
|
if (globs?.length) {
|
|
82
|
-
urlPatternObjects.push(...
|
|
75
|
+
urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
|
|
83
76
|
}
|
|
84
77
|
if (regexps?.length) {
|
|
85
|
-
urlPatternObjects.push(...
|
|
78
|
+
urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
|
|
86
79
|
}
|
|
87
80
|
const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
|
|
88
81
|
page,
|
|
@@ -91,11 +84,11 @@ async function enqueueLinksByClickingElements(options) {
|
|
|
91
84
|
maxWaitForPageIdleMillis,
|
|
92
85
|
clickOptions,
|
|
93
86
|
});
|
|
94
|
-
let requestOptions =
|
|
87
|
+
let requestOptions = createRequestOptions(interceptedRequests, options);
|
|
95
88
|
if (transformRequestFunction) {
|
|
96
89
|
requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r);
|
|
97
90
|
}
|
|
98
|
-
const requests =
|
|
91
|
+
const requests = createRequests(requestOptions, urlPatternObjects);
|
|
99
92
|
const { addedRequests } = await requestQueue.addRequestsBatched(requests, { forefront });
|
|
100
93
|
return { processedRequests: addedRequests, unprocessedRequests: [] };
|
|
101
94
|
}
|
|
@@ -105,14 +98,14 @@ async function enqueueLinksByClickingElements(options) {
|
|
|
105
98
|
* Returns a list of all target URLs.
|
|
106
99
|
* @ignore
|
|
107
100
|
*/
|
|
108
|
-
async function clickElementsAndInterceptNavigationRequests(options) {
|
|
101
|
+
export async function clickElementsAndInterceptNavigationRequests(options) {
|
|
109
102
|
const { page, selector, waitForPageIdleMillis, maxWaitForPageIdleMillis, clickOptions } = options;
|
|
110
103
|
const uniqueRequests = new Set();
|
|
111
104
|
const browser = page.browser();
|
|
112
105
|
const onInterceptedRequest = createInterceptRequestHandler(page, uniqueRequests);
|
|
113
106
|
const onTargetCreated = createTargetCreatedHandler(page, uniqueRequests);
|
|
114
107
|
const onFrameNavigated = createFrameNavigatedHandler(page, uniqueRequests);
|
|
115
|
-
await
|
|
108
|
+
await addInterceptRequestHandler(page, onInterceptedRequest);
|
|
116
109
|
browser.on('targetcreated', onTargetCreated);
|
|
117
110
|
page.on('framenavigated', onFrameNavigated);
|
|
118
111
|
await preventHistoryNavigation(page);
|
|
@@ -121,7 +114,7 @@ async function clickElementsAndInterceptNavigationRequests(options) {
|
|
|
121
114
|
await restoreHistoryNavigationAndSaveCapturedUrls(page, uniqueRequests);
|
|
122
115
|
browser.off('targetcreated', onTargetCreated);
|
|
123
116
|
page.off('framenavigated', onFrameNavigated);
|
|
124
|
-
await
|
|
117
|
+
await removeInterceptRequestHandler(page, onInterceptedRequest);
|
|
125
118
|
const serializedRequests = Array.from(uniqueRequests);
|
|
126
119
|
return serializedRequests.map((r) => JSON.parse(r));
|
|
127
120
|
}
|
|
@@ -178,7 +171,7 @@ function createTargetCreatedHandler(page, requests) {
|
|
|
178
171
|
* We're only interested in pages created by the page we're currently clicking in.
|
|
179
172
|
* There will generally be a lot of other targets being created in the browser.
|
|
180
173
|
*/
|
|
181
|
-
function isTargetRelevant(page, target) {
|
|
174
|
+
export function isTargetRelevant(page, target) {
|
|
182
175
|
return target.type() === 'page' && page.target() === target.opener();
|
|
183
176
|
}
|
|
184
177
|
/**
|
|
@@ -224,7 +217,7 @@ async function preventHistoryNavigation(page) {
|
|
|
224
217
|
* for large element sets, this will take considerable amount of time.
|
|
225
218
|
* @ignore
|
|
226
219
|
*/
|
|
227
|
-
async function clickElements(page, selector, clickOptions) {
|
|
220
|
+
export async function clickElements(page, selector, clickOptions) {
|
|
228
221
|
const elementHandles = await page.$$(selector);
|
|
229
222
|
log.debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
|
|
230
223
|
let clickedElementsCount = 0;
|
|
@@ -328,7 +321,7 @@ async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {
|
|
|
328
321
|
state.forEach((args) => {
|
|
329
322
|
try {
|
|
330
323
|
const stateUrl = args[args.length - 1];
|
|
331
|
-
const url = new
|
|
324
|
+
const url = new URL(stateUrl, page.url()).href;
|
|
332
325
|
requests.add(JSON.stringify({ url }));
|
|
333
326
|
}
|
|
334
327
|
catch (err) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"click-elements.js","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"click-elements.js","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAW/B,OAAO,EACH,6BAA6B,EAC7B,oCAAoC,EACpC,iCAAiC,EACjC,oBAAoB,EACpB,cAAc,GACjB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,IAAI,MAAM,YAAY,CAAC;AAE9B,OAAO,EAAE,0BAA0B,EAAE,6BAA6B,EAAE,MAAM,4CAA4C,CAAC;AAEvH,MAAM,gBAAgB,GAAG,UAAU,CAAC;AACpC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC,CAAC;AAiJ/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAChD,OAA8C;IAE9C,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC;QAC3C,YAAY,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,QAAQ,EAAE,EAAE,CAAC,MAAM;QACnB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,YAAY,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC;QAC/D,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjF,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACvC,sBAAsB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC1C,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KACtC,CAAC,CACL,CAAC;IAEF,MAAM,EACF,IAAI,EACJ,YAAY,EACZ,QAAQ,EACR,YAAY,EACZ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,mBAAmB,GAAG,CAAC,EACvB,sBAAsB,GAAG,CAAC,EAC1B,SAAS,GACZ,GAAG,OAAO,CAAC;IAEZ,MAAM,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;IACzD,MAAM,wBAAwB,GAAG,sBAAsB,GAAG,IAAI,CAAC;IAE/D,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,GAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,oCAAoC,CAAC,UAAU,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,mBAAmB,GAAG,MAAM,2CAA2C,CAAC;QAC1E,IAAI;QACJ,QAAQ;QACR,qBAAqB;QACrB,wBAAwB;QACxB,YAAY;KACf,CAAC,CAAC;IACH,IAAI,cAAc,GAAG,oBAAoB,CAAC,mBAAmB,EAAE,OAAO,CAAC,CAAC;IACxE,IAAI,wBAAwB,EAAE,CAAC;QAC3B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;IACzG,CAAC;IACD,MAAM,QAAQ,GAAG,cAAc,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACnE,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,YAAY,CAAC,kBAAkB,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IAEzF,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC;AACzE,CAAC;AAaD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,2CAA2C,CAC7D,OAA2D;IAE3D,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC;IAElG,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAE/B,MAAM,oBAAoB,GAAG,6BAA6B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACjF,MAAM,eAAe,GAAG,0BAA0B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACzE,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAE3E,MAAM,0BAA0B,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;IAC7C,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;IAE5C,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAClD,MAAM,eAAe,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAEjF,MAAM,2CAA2C,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAExE,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;IAC9C,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;IAC7C,MAAM,6BAA6B,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAEhE,MAAM,kBAAkB,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACtD,OAAO,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED;;GAEG;AACH,SAAS,6BAA6B,CAAC,IAAU,EAAE,QAAqB;IACpE,OAAO,KAAK,UAAU,oBAAoB,CAAC,GAAG;QAC1C,IAAI,CAAC,2BAA2B,CAAC,IAAI,EAAE,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;QACnE,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC;QACtB,QAAQ,CAAC,GAAG,CACR,IAAI,CAAC,SAAS,CAAC;YACX,GAAG;YACH,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE;YACtB,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE;YACpB,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE;SAC1B,CAAC,CACL,CAAC;QAEF,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC,MAAM,EAAE,CAAC;YAC7B,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,4BAA4B;QACjE,CAAC;aAAM,CAAC;YACJ,MAAM,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B;QAC5D,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC;AACN,CAAC;AAED;;GAEG;AACH,SAAS,2BAA2B,CAAC,IAAU,EAAE,GAAqB;IAClE,OAAO,GAAG,CAAC,mBAAmB,EAAE,IAAI,GAAG,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;AACzE,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B,CAAC,IAAU,EAAE,QAAqB;IACjE,OAAO,KAAK,UAAU,eAAe,CAAC,MAAM;QACxC,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC;YAAE,OAAO;QAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;QACzB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;QAEtC,iDAAiD;QACjD,sCAAsC;QACtC,IAAI,CAAC;YACD,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,WAAY,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CAAC,KAAK,CAAC,+DAA+D,EAAE,EAAE,KAAK,EAAG,GAAa,CAAC,KAAK,EAAE,CAAC,CAAC;QAChH,CAAC;IACL,CAAC,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAU,EAAE,MAAc;IACvD,OAAO,MAAM,CAAC,IAAI,EAAE,KAAK,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;AACzE,CAAC;AAED;;GAEG;AACH,SAAS,2BAA2B,CAAC,IAAU,EAAE,QAAqB;IAClE,OAAO,SAAS,gBAAgB,CAAC,KAAK;QAClC,IAAI,KAAK,KAAK,IAAI,CAAC,SAAS,EAAE;YAAE,OAAO;QACvC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;QACxB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC;AACN,CAAC;AAaD;;GAEG;AACH,KAAK,UAAU,wBAAwB,CAAC,IAAU;IAC9C,0BAA0B;IAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACrB,MAAgC,CAAC,mBAAmB,GAAG,MAAM,CAAC,OAAO,CAAC;QACvE,OAAQ,MAAgC,CAAC,OAAO,CAAC,CAAC,iCAAiC;QAClF,MAAgC,CAAC,OAAO,GAAG;YACxC,YAAY,EAAE,EAAE;YAChB,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,EAAE;YACT,EAAE,KAAI,CAAC;YACP,IAAI,KAAI,CAAC;YACT,OAAO,KAAI,CAAC;YACZ,SAAS,CAAC,GAAG,IAAe;gBACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;YACD,YAAY,CAAC,GAAG,IAAe;gBAC3B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;SACW,CAAC;IACrB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAU,EAAE,QAAgB,EAAE,YAA2B;IACzF,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC/C,GAAG,CAAC,KAAK,CAAC,6CAA6C,cAAc,CAAC,MAAM,qBAAqB,CAAC,CAAC;IACnG,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,MAAM,GAAG,gBAAgB,CAAC;IAC9B,IAAI,gBAAgB,GAAG,IAAI,CAAC;IAC5B,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;QAClC,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,QAAQ,CAAC,kCAAkC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1E,MAAM,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACjC,oBAAoB,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,GAAY,CAAC;YACvB,IAAI,gBAAgB,IAAI,CAAC,CAAC,KAAM,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE,CAAC;gBACrE,GAAG,CAAC,OAAO,CACP,4BAA4B,QAAQ,+DAA+D;oBAC/F,qHAAqH;oBACrH,uHAAuH,CAC9H,CAAC;gBACF,gBAAgB,GAAG,KAAK,CAAC;YAC7B,CAAC;YACD,GAAG,CAAC,KAAK,CAAC,+CAA+C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACnF,CAAC;IACL,CAAC;IACD,GAAG,CAAC,KAAK,CACL,wDAAwD,oBAAoB,oBAAoB,cAAc,CAAC,MAAM,EAAE,CAC1H,CAAC;AACN,CAAC;AAED,0BAA0B;AAC1B;;GAEG;AACH,SAAS,kCAAkC,CAAC,EAAW,EAAE,MAAc;IACnE,MAAM,MAAM,GAAG,EAAiB,CAAC;IACjC,MAAM,CAAC,KAAK,CAAC,UAAU,GAAG,SAAS,CAAC;IACpC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;IAC/B,MAAM,CAAC,KAAK,CAAC,QAAQ,GAAG,OAAO,CAAC;IAChC,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC;IACxB,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC;IACvB,MAAM,YAAY,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;IACpD,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IACvD,IAAI,CAAC,YAAY,CAAC,KAAK;QAAE,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC;AACzD,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,eAAe,CAAC,EAC3B,IAAI,EACJ,qBAAqB,EACrB,wBAAwB,GACH;IACrB,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACjC,IAAI,OAAuB,CAAC;QAC5B,IAAI,UAA0B,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtC,SAAS,aAAa,CAAC,MAAc;YACjC,IAAI,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC;gBAAE,eAAe,EAAE,CAAC;QAC1D,CAAC;QAED,SAAS,eAAe;YACpB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBACtB,YAAY,CAAC,UAAU,CAAC,CAAC;gBACzB,MAAM,EAAE,CAAC;YACb,CAAC,EAAE,qBAAqB,CAAC,CAAC;QAC9B,CAAC;QAED,SAAS,iBAAiB;YACtB,GAAG,CAAC,KAAK,CACL,oEAAoE,wBAAwB,MAAM;gBAC9F,4GAA4G,CACnH,CAAC;YACF,MAAM,EAAE,CAAC;QACb,CAAC;QAED,SAAS,MAAM;YACX,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;YACrC,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAC5C,OAAO,EAAE,CAAC;QACd,CAAC;QAED,UAAU,GAAG,UAAU,CAAC,iBAAiB,EAAE,wBAAwB,CAAC,CAAC;QACrE,eAAe,EAAE,CAAC,CAAC,wEAAwE;QAC3F,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;QACpC,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC;QAC3C,OAAO,CAAC,EAAE,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,2CAA2C,CAAC,IAAU,EAAE,QAAqB;IACxF,0BAA0B;IAC1B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACnC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC,OAAiC,CAAC;QACjE,MAAgC,CAAC,OAAO,GAAI,MAAgC,CAAC,mBAAmB,CAAC;QAClG,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAW,CAAC;YACjD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC;YAC/C,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAG,GAAa,CAAC,KAAK,EAAE,CAAC,CAAC;QAC7F,CAAC;IACL,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -4,8 +4,8 @@ import type { PuppeteerController, PuppeteerPlugin } from '@crawlee/browser-pool
|
|
|
4
4
|
import type { Dictionary } from '@crawlee/types';
|
|
5
5
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
6
6
|
import type { HTTPResponse, LaunchOptions, Page } from 'puppeteer';
|
|
7
|
-
import type { PuppeteerLaunchContext } from './puppeteer-launcher';
|
|
8
|
-
import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer_utils';
|
|
7
|
+
import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
|
|
8
|
+
import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer_utils.js';
|
|
9
9
|
export interface PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<PuppeteerCrawler, Page, HTTPResponse, PuppeteerController, UserData>, PuppeteerContextUtils {
|
|
10
10
|
}
|
|
11
11
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -127,8 +127,6 @@ export declare class PuppeteerCrawler extends BrowserCrawler<{
|
|
|
127
127
|
protected static optionsShape: {
|
|
128
128
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
129
129
|
browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
130
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
131
|
-
handlePageFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
132
130
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
133
131
|
navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
134
132
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -157,18 +155,12 @@ export declare class PuppeteerCrawler extends BrowserCrawler<{
|
|
|
157
155
|
requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
158
156
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
159
157
|
requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
160
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
161
|
-
handleRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
162
158
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
163
159
|
requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
164
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
165
|
-
handleRequestTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
166
160
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
167
161
|
errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
168
162
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
169
163
|
failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
170
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
171
|
-
handleFailedRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
172
164
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
173
165
|
maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
174
166
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"puppeteer-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,EACX,qBAAqB,EACrB,sBAAsB,EACtB,aAAa,EACb,YAAY,EACf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAE,aAAa,EAAU,MAAM,kBAAkB,CAAC;AACzE,OAAO,KAAK,EAAsB,mBAAmB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACtG,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEnE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"puppeteer-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,EACX,qBAAqB,EACrB,sBAAsB,EACtB,aAAa,EACb,YAAY,EACf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAE,aAAa,EAAU,MAAM,kBAAkB,CAAC;AACzE,OAAO,KAAK,EAAsB,mBAAmB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACtG,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEnE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AAEtE,OAAO,KAAK,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AAGjG,MAAM,WAAW,wBAAwB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAC9E,SAAQ,sBAAsB,CAAC,gBAAgB,EAAE,IAAI,EAAE,YAAY,EAAE,mBAAmB,EAAE,QAAQ,CAAC,EAC/F,qBAAqB;CAAG;AAChC,MAAM,WAAW,aAAc,SAAQ,WAAW,CAAC,wBAAwB,EAAE,oBAAoB,CAAC;CAAG;AACrG,MAAM,WAAW,uBAAwB,SAAQ,qBAAqB,CAAC,aAAa,CAAC,wBAAwB,CAAC,CAAC;CAAG;AAClH,MAAM,MAAM,oBAAoB,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,uBACb,SAAQ,qBAAqB,CAAC,wBAAwB,EAAE;IAAE,cAAc,EAAE,CAAC,eAAe,CAAC,CAAA;CAAE,CAAC;IAC9F;;OAEG;IACH,aAAa,CAAC,EAAE,sBAAsB,CAAC;IAEvC;;;;;;;;;;;;;;;;OAgBG;IACH,kBAAkB,CAAC,EAAE,aAAa,EAAE,CAAC;IAErC;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,aAAa,EAAE,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AACH,qBAAa,gBAAiB,SAAQ,cAAc,CAChD;IAAE,cAAc,EAAE,CAAC,eAAe,CAAC,CAAA;CAAE,EACrC,aAAa,EACb,wBAAwB,CAC3B;IAUO,OAAO,CAAC,QAAQ,CAAC,OAAO;aACN,MAAM;IAV5B,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAGpC;IAEF;;OAEG;gBAEkB,OAAO,GAAE,uBAA4B,EACpC,MAAM,gBAAkC;cAmCrC,kBAAkB,CAAC,OAAO,EAAE,wBAAwB;cAKpD,kBAAkB,CACvC,eAAe,EAAE,wBAAwB,EACzC,WAAW,EAAE,uBAAuB;CAI3C;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,qBAAqB,CACjC,OAAO,SAAS,wBAAwB,GAAG,wBAAwB,EACnE,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,qDAEzC"}
|
|
@@ -1,12 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
const tslib_1 = require("tslib");
|
|
6
|
-
const browser_1 = require("@crawlee/browser");
|
|
7
|
-
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
8
|
-
const puppeteer_launcher_1 = require("./puppeteer-launcher");
|
|
9
|
-
const puppeteer_utils_1 = require("./utils/puppeteer_utils");
|
|
1
|
+
import { BrowserCrawler, Configuration, Router } from '@crawlee/browser';
|
|
2
|
+
import ow from 'ow';
|
|
3
|
+
import { PuppeteerLauncher } from './puppeteer-launcher.js';
|
|
4
|
+
import { gotoExtended, registerUtilsToContext } from './utils/puppeteer_utils.js';
|
|
10
5
|
/**
|
|
11
6
|
* Provides a simple framework for parallel crawling of web pages
|
|
12
7
|
* using headless Chrome with [Puppeteer](https://github.com/puppeteer/puppeteer).
|
|
@@ -70,12 +65,18 @@ const puppeteer_utils_1 = require("./utils/puppeteer_utils");
|
|
|
70
65
|
* ```
|
|
71
66
|
* @category Crawlers
|
|
72
67
|
*/
|
|
73
|
-
class PuppeteerCrawler extends
|
|
68
|
+
export class PuppeteerCrawler extends BrowserCrawler {
|
|
69
|
+
options;
|
|
70
|
+
config;
|
|
71
|
+
static optionsShape = {
|
|
72
|
+
...BrowserCrawler.optionsShape,
|
|
73
|
+
browserPoolOptions: ow.optional.object,
|
|
74
|
+
};
|
|
74
75
|
/**
|
|
75
76
|
* All `PuppeteerCrawler` parameters are passed via an options object.
|
|
76
77
|
*/
|
|
77
|
-
constructor(options = {}, config =
|
|
78
|
-
(
|
|
78
|
+
constructor(options = {}, config = Configuration.getGlobalConfig()) {
|
|
79
|
+
ow(options, 'PuppeteerCrawlerOptions', ow.object.exactShape(PuppeteerCrawler.optionsShape));
|
|
79
80
|
const { launchContext = {}, headless, proxyConfiguration, ...browserCrawlerOptions } = options;
|
|
80
81
|
const browserPoolOptions = {
|
|
81
82
|
...options.browserPoolOptions,
|
|
@@ -90,43 +91,23 @@ class PuppeteerCrawler extends browser_1.BrowserCrawler {
|
|
|
90
91
|
throw new Error('browserPoolOptions.browserPlugins is disallowed. Use launchContext.launcher instead.');
|
|
91
92
|
}
|
|
92
93
|
if (headless != null) {
|
|
93
|
-
launchContext.launchOptions
|
|
94
|
+
launchContext.launchOptions ??= {};
|
|
94
95
|
launchContext.launchOptions.headless = headless;
|
|
95
96
|
}
|
|
96
|
-
const puppeteerLauncher = new
|
|
97
|
+
const puppeteerLauncher = new PuppeteerLauncher(launchContext, config);
|
|
97
98
|
browserPoolOptions.browserPlugins = [puppeteerLauncher.createBrowserPlugin()];
|
|
98
99
|
super({ ...browserCrawlerOptions, launchContext, proxyConfiguration, browserPoolOptions }, config);
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
configurable: true,
|
|
102
|
-
writable: true,
|
|
103
|
-
value: options
|
|
104
|
-
});
|
|
105
|
-
Object.defineProperty(this, "config", {
|
|
106
|
-
enumerable: true,
|
|
107
|
-
configurable: true,
|
|
108
|
-
writable: true,
|
|
109
|
-
value: config
|
|
110
|
-
});
|
|
100
|
+
this.options = options;
|
|
101
|
+
this.config = config;
|
|
111
102
|
}
|
|
112
103
|
async _runRequestHandler(context) {
|
|
113
|
-
|
|
104
|
+
registerUtilsToContext(context, this.options);
|
|
114
105
|
await super._runRequestHandler(context);
|
|
115
106
|
}
|
|
116
107
|
async _navigationHandler(crawlingContext, gotoOptions) {
|
|
117
|
-
return
|
|
108
|
+
return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
|
|
118
109
|
}
|
|
119
110
|
}
|
|
120
|
-
exports.PuppeteerCrawler = PuppeteerCrawler;
|
|
121
|
-
Object.defineProperty(PuppeteerCrawler, "optionsShape", {
|
|
122
|
-
enumerable: true,
|
|
123
|
-
configurable: true,
|
|
124
|
-
writable: true,
|
|
125
|
-
value: {
|
|
126
|
-
...browser_1.BrowserCrawler.optionsShape,
|
|
127
|
-
browserPoolOptions: ow_1.default.optional.object,
|
|
128
|
-
}
|
|
129
|
-
});
|
|
130
111
|
/**
|
|
131
112
|
* Creates new {@link Router} instance that works based on request labels.
|
|
132
113
|
* This instance can then serve as a `requestHandler` of your {@link PuppeteerCrawler}.
|
|
@@ -151,7 +132,7 @@ Object.defineProperty(PuppeteerCrawler, "optionsShape", {
|
|
|
151
132
|
* await crawler.run();
|
|
152
133
|
* ```
|
|
153
134
|
*/
|
|
154
|
-
function createPuppeteerRouter(routes) {
|
|
155
|
-
return
|
|
135
|
+
export function createPuppeteerRouter(routes) {
|
|
136
|
+
return Router.create(routes);
|
|
156
137
|
}
|
|
157
138
|
//# sourceMappingURL=puppeteer-crawler.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"puppeteer-crawler.js","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"puppeteer-crawler.js","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAGzE,OAAO,EAAE,MAAM,IAAI,CAAC;AAIpB,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAE5D,OAAO,EAAE,YAAY,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AAqDlF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AACH,MAAM,OAAO,gBAAiB,SAAQ,cAIrC;IAUwB;IACC;IAVZ,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,cAAc,CAAC,YAAY;QAC9B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KACzC,CAAC;IAEF;;OAEG;IACH,YACqB,UAAmC,EAAE,EACpC,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,OAAO,EAAE,yBAAyB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE5F,MAAM,EAAE,aAAa,GAAG,EAAE,EAAE,QAAQ,EAAE,kBAAkB,EAAE,GAAG,qBAAqB,EAAE,GAAG,OAAO,CAAC;QAE/F,MAAM,kBAAkB,GAAG;YACvB,GAAG,OAAO,CAAC,kBAAkB;SACV,CAAC;QAExB,IAAI,aAAa,CAAC,QAAQ,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACX,oFAAoF;gBAChF,gDAAgD,CACvD,CAAC;QACN,CAAC;QAED,2EAA2E;QAC3E,uFAAuF;QACvF,IAAI,kBAAkB,CAAC,cAAc,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,sFAAsF,CAAC,CAAC;QAC5G,CAAC;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,aAAa,CAAC,aAAa,KAAK,EAAmB,CAAC;YACpD,aAAa,CAAC,aAAa,CAAC,QAAQ,GAAG,QAAmB,CAAC;QAC/D,CAAC;QAED,MAAM,iBAAiB,GAAG,IAAI,iBAAiB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QAEvE,kBAAkB,CAAC,cAAc,GAAG,CAAC,iBAAiB,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAE9E,KAAK,CAAC,EAAE,GAAG,qBAAqB,EAAE,aAAa,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,EAAE,MAAM,CAAC,CAAC;QAjClF,YAAO,GAAP,OAAO,CAA8B;QACpC,WAAM,GAAN,MAAM,CAAkC;IAiC9D,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CAAC,OAAiC;QACzE,sBAAsB,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9C,MAAM,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CACvC,eAAyC,EACzC,WAAoC;QAEpC,OAAO,YAAY,CAAC,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACpF,CAAC;;AAGL;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,qBAAqB,CAGnC,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
|
@@ -78,8 +78,6 @@ export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin,
|
|
|
78
78
|
useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
79
79
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
80
80
|
browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
81
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
82
|
-
experimentalContainers: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
83
81
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
84
82
|
userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
85
83
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"puppeteer-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-launcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB,CAAC,eAAe,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC3G;;OAEG;IACH,aAAa,CAAC,EAAE,eAAe,CAAC,eAAe,CAAC,CAAC;IAEjD;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED;;;GAGG;AACH,qBAAa,iBAAkB,SAAQ,eAAe,CAAC,eAAe,EAAE,OAAO,CAAC;aAWtD,MAAM;IAV5B,iBAA0B,YAAY
|
|
1
|
+
{"version":3,"file":"puppeteer-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-launcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB,CAAC,eAAe,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC3G;;OAEG;IACH,aAAa,CAAC,EAAE,eAAe,CAAC,eAAe,CAAC,CAAC;IAEjD;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED;;;GAGG;AACH,qBAAa,iBAAkB,SAAQ,eAAe,CAAC,eAAe,EAAE,OAAO,CAAC;aAWtD,MAAM;IAV5B,iBAA0B,YAAY;;;;;;;;;MAGpC;IAEF;;OAEG;gBAEC,aAAa,GAAE,sBAA2B,EACxB,MAAM,gBAAkC;cAoB3C,yBAAyB,IAAI,OAAO;CAI1D;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,wBAAsB,eAAe,CACjC,aAAa,CAAC,EAAE,sBAAsB,EACtC,MAAM,gBAAkC,GACzC,OAAO,CAAC,OAAO,CAAC,CAIlB"}
|
|
@@ -1,49 +1,34 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
exports.launchPuppeteer = launchPuppeteer;
|
|
5
|
-
const tslib_1 = require("tslib");
|
|
6
|
-
const browser_1 = require("@crawlee/browser");
|
|
7
|
-
const browser_pool_1 = require("@crawlee/browser-pool");
|
|
8
|
-
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
1
|
+
import { BrowserLauncher, Configuration } from '@crawlee/browser';
|
|
2
|
+
import { PuppeteerPlugin } from '@crawlee/browser-pool';
|
|
3
|
+
import ow from 'ow';
|
|
9
4
|
/**
|
|
10
5
|
* `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
|
|
11
6
|
* @ignore
|
|
12
7
|
*/
|
|
13
|
-
class PuppeteerLauncher extends
|
|
8
|
+
export class PuppeteerLauncher extends BrowserLauncher {
|
|
9
|
+
config;
|
|
10
|
+
static optionsShape = {
|
|
11
|
+
...BrowserLauncher.optionsShape,
|
|
12
|
+
launcher: ow.optional.object,
|
|
13
|
+
};
|
|
14
14
|
/**
|
|
15
15
|
* All `PuppeteerLauncher` parameters are passed via an launchContext object.
|
|
16
16
|
*/
|
|
17
|
-
constructor(launchContext = {}, config =
|
|
18
|
-
(
|
|
19
|
-
const { launcher =
|
|
17
|
+
constructor(launchContext = {}, config = Configuration.getGlobalConfig()) {
|
|
18
|
+
ow(launchContext, 'PuppeteerLauncher', ow.object.exactShape(PuppeteerLauncher.optionsShape));
|
|
19
|
+
const { launcher = BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-puppeteer-chrome'), ...browserLauncherOptions } = launchContext;
|
|
20
20
|
super({
|
|
21
21
|
...browserLauncherOptions,
|
|
22
22
|
launcher,
|
|
23
23
|
}, config);
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
configurable: true,
|
|
27
|
-
writable: true,
|
|
28
|
-
value: config
|
|
29
|
-
});
|
|
30
|
-
this.Plugin = browser_pool_1.PuppeteerPlugin;
|
|
24
|
+
this.config = config;
|
|
25
|
+
this.Plugin = PuppeteerPlugin;
|
|
31
26
|
}
|
|
32
27
|
_getDefaultHeadlessOption() {
|
|
33
28
|
const headless = super._getDefaultHeadlessOption();
|
|
34
29
|
return headless ? 'new' : headless;
|
|
35
30
|
}
|
|
36
31
|
}
|
|
37
|
-
exports.PuppeteerLauncher = PuppeteerLauncher;
|
|
38
|
-
Object.defineProperty(PuppeteerLauncher, "optionsShape", {
|
|
39
|
-
enumerable: true,
|
|
40
|
-
configurable: true,
|
|
41
|
-
writable: true,
|
|
42
|
-
value: {
|
|
43
|
-
...browser_1.BrowserLauncher.optionsShape,
|
|
44
|
-
launcher: ow_1.default.optional.object,
|
|
45
|
-
}
|
|
46
|
-
});
|
|
47
32
|
/**
|
|
48
33
|
* Launches headless Chrome using Puppeteer pre-configured to work within the Apify platform.
|
|
49
34
|
* The function has the same argument and the return value as `puppeteer.launch()`.
|
|
@@ -76,7 +61,7 @@ Object.defineProperty(PuppeteerLauncher, "optionsShape", {
|
|
|
76
61
|
* @returns
|
|
77
62
|
* Promise that resolves to Puppeteer's `Browser` instance.
|
|
78
63
|
*/
|
|
79
|
-
async function launchPuppeteer(launchContext, config =
|
|
64
|
+
export async function launchPuppeteer(launchContext, config = Configuration.getGlobalConfig()) {
|
|
80
65
|
const puppeteerLauncher = new PuppeteerLauncher(launchContext, config);
|
|
81
66
|
return puppeteerLauncher.launch();
|
|
82
67
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"puppeteer-launcher.js","sourceRoot":"","sources":["../../src/internals/puppeteer-launcher.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"puppeteer-launcher.js","sourceRoot":"","sources":["../../src/internals/puppeteer-launcher.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,MAAM,IAAI,CAAC;AAkEpB;;;GAGG;AACH,MAAM,OAAO,iBAAkB,SAAQ,eAAyC;IAWtD;IAVZ,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,eAAe,CAAC,YAAY;QAC/B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAC/B,CAAC;IAEF;;OAEG;IACH,YACI,gBAAwC,EAAE,EACxB,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,aAAa,EAAE,mBAAmB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE7F,MAAM,EACF,QAAQ,GAAG,eAAe,CAAC,sBAAsB,CAAC,WAAW,EAAE,mCAAmC,CAAC,EACnG,GAAG,sBAAsB,EAC5B,GAAG,aAAa,CAAC;QAElB,KAAK,CACD;YACI,GAAG,sBAAsB;YACzB,QAAQ;SACX,EACD,MAAM,CACT,CAAC;QAfgB,WAAM,GAAN,MAAM,CAAkC;QAiB1D,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC;IAClC,CAAC;IAEkB,yBAAyB;QACxC,MAAM,QAAQ,GAAG,KAAK,CAAC,yBAAyB,EAAE,CAAC;QACnD,OAAO,QAAQ,CAAC,CAAC,CAAE,KAAa,CAAC,CAAC,CAAC,QAAQ,CAAC;IAChD,CAAC;;AAGL;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACjC,aAAsC,EACtC,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE;IAExC,MAAM,iBAAiB,GAAG,IAAI,iBAAiB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IAEvE,OAAO,iBAAiB,CAAC,MAAM,EAAE,CAAC;AACtC,CAAC"}
|
|
@@ -1,11 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
exports.removeInterceptRequestHandler = removeInterceptRequestHandler;
|
|
5
|
-
const tslib_1 = require("tslib");
|
|
6
|
-
const node_events_1 = require("node:events");
|
|
7
|
-
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
8
|
-
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
1
|
+
import { EventEmitter } from 'node:events';
|
|
2
|
+
import ow from 'ow';
|
|
3
|
+
import log from '@apify/log';
|
|
9
4
|
// We use weak maps here so that the content gets discarded after page gets closed.
|
|
10
5
|
const pageInterceptRequestHandlersMap = new WeakMap(); // Maps page to an array of request interception handlers.
|
|
11
6
|
const pageInterceptRequestMasterHandlerMap = new WeakMap(); // Maps page to master request interception handler.
|
|
@@ -14,16 +9,8 @@ const pageInterceptedRequestsMap = new WeakMap(); // Maps page to a set of its p
|
|
|
14
9
|
* Enables observation of changes of internal state to be able to queue other actions based on it.
|
|
15
10
|
* @ignore
|
|
16
11
|
*/
|
|
17
|
-
class ObservableSet extends
|
|
18
|
-
|
|
19
|
-
super(...arguments);
|
|
20
|
-
Object.defineProperty(this, "set", {
|
|
21
|
-
enumerable: true,
|
|
22
|
-
configurable: true,
|
|
23
|
-
writable: true,
|
|
24
|
-
value: new Set()
|
|
25
|
-
});
|
|
26
|
-
}
|
|
12
|
+
class ObservableSet extends EventEmitter {
|
|
13
|
+
set = new Set();
|
|
27
14
|
add(value) {
|
|
28
15
|
this.set.add(value);
|
|
29
16
|
this.emit('add', value);
|
|
@@ -149,9 +136,9 @@ async function handleRequest(request, interceptRequestHandlers) {
|
|
|
149
136
|
* @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
|
|
150
137
|
* @param handler Request interception handler.
|
|
151
138
|
*/
|
|
152
|
-
async function addInterceptRequestHandler(page, handler) {
|
|
153
|
-
(
|
|
154
|
-
(
|
|
139
|
+
export async function addInterceptRequestHandler(page, handler) {
|
|
140
|
+
ow(page, ow.object.hasKeys('goto', 'evaluate'));
|
|
141
|
+
ow(handler, ow.function);
|
|
155
142
|
if (!pageInterceptRequestHandlersMap.has(page)) {
|
|
156
143
|
pageInterceptRequestHandlersMap.set(page, []);
|
|
157
144
|
}
|
|
@@ -186,9 +173,9 @@ async function addInterceptRequestHandler(page, handler) {
|
|
|
186
173
|
* @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
|
|
187
174
|
* @param handler Request interception handler.
|
|
188
175
|
*/
|
|
189
|
-
async function removeInterceptRequestHandler(page, handler) {
|
|
190
|
-
(
|
|
191
|
-
(
|
|
176
|
+
export async function removeInterceptRequestHandler(page, handler) {
|
|
177
|
+
ow(page, ow.object.hasKeys('goto', 'evaluate'));
|
|
178
|
+
ow(handler, ow.function);
|
|
192
179
|
const handlersArray = pageInterceptRequestHandlersMap.get(page).filter((item) => item !== handler);
|
|
193
180
|
pageInterceptRequestHandlersMap.set(page, handlersArray);
|
|
194
181
|
if (handlersArray.length === 0) {
|
|
@@ -207,7 +194,7 @@ async function removeInterceptRequestHandler(page, handler) {
|
|
|
207
194
|
interceptedRequestsInProgress.removeListener('delete', onDelete);
|
|
208
195
|
}
|
|
209
196
|
catch (error) {
|
|
210
|
-
|
|
197
|
+
log.debug('Error while disabling request interception', { error });
|
|
211
198
|
}
|
|
212
199
|
}
|
|
213
200
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"puppeteer_request_interception.js","sourceRoot":"","sources":["../../../src/internals/utils/puppeteer_request_interception.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"puppeteer_request_interception.js","sourceRoot":"","sources":["../../../src/internals/utils/puppeteer_request_interception.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,GAAG,MAAM,YAAY,CAAC;AAI7B,mFAAmF;AACnF,MAAM,+BAA+B,GAAsC,IAAI,OAAO,EAAE,CAAC,CAAC,0DAA0D;AACpJ,MAAM,oCAAoC,GAAG,IAAI,OAAO,EAAE,CAAC,CAAC,oDAAoD;AAChH,MAAM,0BAA0B,GAAG,IAAI,OAAO,EAAE,CAAC,CAAC,0DAA0D;AAE5G;;;GAGG;AACH,MAAM,aAAiB,SAAQ,YAAY;IACvC,GAAG,GAAG,IAAI,GAAG,EAAK,CAAC;IAEnB,GAAG,CAAC,KAAQ;QACR,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC,GAAG,CAAC;IACpB,CAAC;IAED,MAAM,CAAC,KAAQ;QACX,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC3B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,IAAI,IAAI;QACJ,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;IACzB,CAAC;CACJ;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,OAA+B;IACtD,MAAM,YAAY,GAAuB,EAAE,CAAC;IAC5C,wCAAwC;IACxC,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/C,GAAG,GAAG,GAAG;aACJ,WAAW,EAAE;aACb,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aACxD,IAAI,CAAC,GAAG,CAAC,CAAC;QAEf,YAAY,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IAC9B,CAAC;IAED,OAAO,YAAY,CAAC;AACxB,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,aAAa,CAAC,OAAyB,EAAE,wBAA6C;IACjG,kGAAkG;IAClG,uFAAuF;IACvF,IAAI,CAAC,wBAAwB,EAAE,MAAM;QAAE,OAAO,SAAS,CAAC;IAExD,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,oBAAoB,GAAG;QACzB,OAAO,EAAE,iBAAiB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;KAChD,CAAC;IAEF,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxD,OAAO,CAAC,QAAQ,GAAG,KAAK,EAAE,SAAS,GAAG,EAAE,EAAE,EAAE;QACxC,YAAY,GAAG,IAAI,CAAC;QACpB,MAAM,OAAO,GAAG,iBAAiB,CAAC,EAAE,GAAG,oBAAoB,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,oBAAoB,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAChE,CAAC,CAAC;IAEF,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACnC,OAAO,CAAC,KAAK,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,EAAE;QAC9B,UAAU,GAAG,IAAI,CAAC;QAClB,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IACxC,CAAC,CAAC;IACF,OAAO,CAAC,OAAO,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,EAAE;QAChC,YAAY,GAAG,IAAI,CAAC;QACpB,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IAC1C,CAAC,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,wBAAwB,EAAE,CAAC;QAC7C,YAAY,GAAG,KAAK,CAAC;QAErB,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACvB,8CAA8C;QAC9C,IAAI,CAAC,UAAU,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,sFAAsF,CAAC,CAAC;QAC5G,CAAC;QAED,sEAAsE;QACtE,IAAI,UAAU,IAAI,YAAY;YAAE,OAAO,SAAS,CAAC;IACrD,CAAC;IAED,OAAO,gBAAgB,CAAC,oBAAoB,CAAC,CAAC;AAClD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAU,EAAE,OAAyB;IAClF,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IAChD,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;IAEzB,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7C,+BAA+B,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,0BAA0B,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,aAAa,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,aAAa,GAAG,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;IACjE,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAE5B,wFAAwF;IACxF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAExC,4FAA4F;QAC5F,yBAAyB;QACzB,MAAM,aAAa,GAAG,KAAK,EAAE,OAAoB,EAAE,EAAE;YACjD,MAAM,mBAAmB,GAAG,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACjE,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACjC,MAAM,iBAAiB,GAAG,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACpE,IAAI,CAAC;gBACD,MAAM,aAAa,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;YACpD,CAAC;oBAAS,CAAC;gBACP,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACxC,CAAC;QACL,CAAC,CAAC;QAEF,oCAAoC,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAC9D,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IACtC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,IAAU,EAAE,OAAyB;IACrF,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IAChD,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;IAEzB,MAAM,aAAa,GAAG,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;IAEpG,+BAA+B,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAEzD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,6BAA6B,GAAG,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3E,6EAA6E;QAC7E,gEAAgE;QAChE,mEAAmE;QACnE,IAAI,6BAA6B,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC3C,MAAM,0BAA0B,CAAC,IAAI,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACJ,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;gBACxB,IAAI,6BAA6B,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;oBAC3C,IAAI,CAAC;wBACD,MAAM,0BAA0B,CAAC,IAAI,CAAC,CAAC;wBACvC,6BAA6B,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;oBACrE,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACb,GAAG,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;oBACvE,CAAC;gBACL,CAAC;YACL,CAAC,CAAC;YACF,6BAA6B,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;AACL,CAAC;AAED,KAAK,UAAU,0BAA0B,CAAC,IAAU;IAChD,MAAM,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,cAAc,GAAG,oCAAoC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACtE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AACxC,CAAC"}
|