@crawlee/browser-pool 3.0.0-alpha.9 → 3.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/abstract-classes/browser-controller.d.ts +153 -0
- package/abstract-classes/browser-controller.d.ts.map +1 -0
- package/abstract-classes/browser-controller.js +230 -0
- package/abstract-classes/browser-controller.js.map +1 -0
- package/abstract-classes/browser-plugin.d.ts +102 -0
- package/abstract-classes/browser-plugin.d.ts.map +1 -0
- package/abstract-classes/browser-plugin.js +154 -0
- package/abstract-classes/browser-plugin.js.map +1 -0
- package/abstract-classes/utils.d.ts +2 -0
- package/abstract-classes/utils.d.ts.map +1 -0
- package/abstract-classes/utils.js +8 -0
- package/abstract-classes/utils.js.map +1 -0
- package/anonymize-proxy.d.ts +4 -0
- package/anonymize-proxy.d.ts.map +1 -0
- package/anonymize-proxy.js +34 -0
- package/anonymize-proxy.js.map +1 -0
- package/browser-pool.d.ts +406 -0
- package/browser-pool.d.ts.map +1 -0
- package/browser-pool.js +583 -0
- package/browser-pool.js.map +1 -0
- package/events.d.ts +11 -0
- package/events.d.ts.map +1 -0
- package/events.js +17 -0
- package/events.js.map +1 -0
- package/fingerprinting/hooks.d.ts +17 -0
- package/fingerprinting/hooks.d.ts.map +1 -0
- package/fingerprinting/hooks.js +87 -0
- package/fingerprinting/hooks.js.map +1 -0
- package/fingerprinting/types.d.ts +33 -0
- package/fingerprinting/types.d.ts.map +1 -0
- package/fingerprinting/types.js +21 -0
- package/fingerprinting/types.js.map +1 -0
- package/fingerprinting/utils.d.ts +4 -0
- package/fingerprinting/utils.d.ts.map +1 -0
- package/fingerprinting/utils.js +47 -0
- package/fingerprinting/utils.js.map +1 -0
- package/index.d.ts +37 -0
- package/index.d.ts.map +1 -0
- package/index.js.map +1 -0
- package/index.mjs +11 -0
- package/launch-context.d.ts +68 -0
- package/launch-context.d.ts.map +1 -0
- package/launch-context.js +94 -0
- package/launch-context.js.map +1 -0
- package/logger.d.ts +2 -0
- package/logger.d.ts.map +1 -0
- package/logger.js +9 -0
- package/logger.js.map +1 -0
- package/package.json +6 -9
- package/playwright/browser.d.ts +27 -0
- package/playwright/browser.d.ts.map +1 -0
- package/playwright/browser.js +66 -0
- package/playwright/browser.js.map +1 -0
- package/playwright/playwright-controller.d.ts +12 -0
- package/playwright/playwright-controller.d.ts.map +1 -0
- package/playwright/playwright-controller.js +71 -0
- package/playwright/playwright-controller.js.map +1 -0
- package/playwright/playwright-plugin.d.ts +13 -0
- package/playwright/playwright-plugin.d.ts.map +1 -0
- package/playwright/playwright-plugin.js +89 -0
- package/playwright/playwright-plugin.js.map +1 -0
- package/proxy-server.d.ts +2 -0
- package/proxy-server.d.ts.map +1 -0
- package/proxy-server.js +18 -0
- package/proxy-server.js.map +1 -0
- package/puppeteer/puppeteer-controller.d.ts +11 -0
- package/puppeteer/puppeteer-controller.d.ts.map +1 -0
- package/puppeteer/puppeteer-controller.js +112 -0
- package/puppeteer/puppeteer-controller.js.map +1 -0
- package/puppeteer/puppeteer-plugin.d.ts +12 -0
- package/puppeteer/puppeteer-plugin.d.ts.map +1 -0
- package/puppeteer/puppeteer-plugin.js +151 -0
- package/puppeteer/puppeteer-plugin.js.map +1 -0
- package/puppeteer/puppeteer-proxy-per-page.d.ts +27 -0
- package/tsconfig.build.tsbuildinfo +1 -0
- package/utils.d.ts +6 -0
- package/utils.d.ts.map +1 -0
- package/utils.js +6 -0
- package/utils.js.map +1 -0
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PlaywrightController = void 0;
|
|
4
|
+
const timeout_1 = require("@apify/timeout");
|
|
5
|
+
const browser_controller_1 = require("../abstract-classes/browser-controller");
|
|
6
|
+
const anonymize_proxy_1 = require("../anonymize-proxy");
|
|
7
|
+
class PlaywrightController extends browser_controller_1.BrowserController {
|
|
8
|
+
normalizeProxyOptions(proxyUrl, pageOptions) {
|
|
9
|
+
if (!proxyUrl) {
|
|
10
|
+
return {};
|
|
11
|
+
}
|
|
12
|
+
const url = new URL(proxyUrl);
|
|
13
|
+
const username = decodeURIComponent(url.username);
|
|
14
|
+
const password = decodeURIComponent(url.password);
|
|
15
|
+
return {
|
|
16
|
+
proxy: {
|
|
17
|
+
server: url.origin,
|
|
18
|
+
username,
|
|
19
|
+
password,
|
|
20
|
+
bypass: pageOptions?.proxy?.bypass,
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
async _newPage(contextOptions) {
|
|
25
|
+
if (contextOptions !== undefined && !this.launchContext.useIncognitoPages) {
|
|
26
|
+
throw new Error('A new page can be created with provided context only when using incognito pages.');
|
|
27
|
+
}
|
|
28
|
+
// eslint-disable-next-line @typescript-eslint/no-empty-function
|
|
29
|
+
let close = async () => { };
|
|
30
|
+
if (contextOptions?.proxy) {
|
|
31
|
+
const [anonymizedProxyUrl, closeProxy] = await (0, anonymize_proxy_1.anonymizeProxySugar)(contextOptions.proxy.server, contextOptions.proxy.username, contextOptions.proxy.password);
|
|
32
|
+
if (anonymizedProxyUrl) {
|
|
33
|
+
contextOptions.proxy = {
|
|
34
|
+
server: anonymizedProxyUrl,
|
|
35
|
+
bypass: contextOptions.proxy.bypass,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
close = closeProxy;
|
|
39
|
+
}
|
|
40
|
+
try {
|
|
41
|
+
const page = await this.browser.newPage(contextOptions);
|
|
42
|
+
(0, timeout_1.tryCancel)();
|
|
43
|
+
page.once('close', async () => {
|
|
44
|
+
this.activePages--;
|
|
45
|
+
await close();
|
|
46
|
+
});
|
|
47
|
+
return page;
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
await close();
|
|
51
|
+
throw error;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
async _close() {
|
|
55
|
+
await this.browser.close();
|
|
56
|
+
}
|
|
57
|
+
async _kill() {
|
|
58
|
+
// TODO: We need to be absolutely sure the browser dies.
|
|
59
|
+
await this.browser.close(); // Playwright does not have the browser child process attached to normal browser server
|
|
60
|
+
}
|
|
61
|
+
_getCookies(page) {
|
|
62
|
+
const context = page.context();
|
|
63
|
+
return context.cookies();
|
|
64
|
+
}
|
|
65
|
+
_setCookies(page, cookies) {
|
|
66
|
+
const context = page.context();
|
|
67
|
+
return context.addCookies(cookies);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
exports.PlaywrightController = PlaywrightController;
|
|
71
|
+
//# sourceMappingURL=playwright-controller.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright-controller.js","sourceRoot":"","sources":["../../src/playwright/playwright-controller.ts"],"names":[],"mappings":";;;AACA,4CAA2C;AAC3C,+EAAmF;AACnF,wDAAyD;AAEzD,MAAa,oBAAqB,SAAQ,sCAA6E;IACnH,qBAAqB,CAAC,QAA4B,EAAE,WAAgB;QAChE,IAAI,CAAC,QAAQ,EAAE;YACX,OAAO,EAAE,CAAC;SACb;QAED,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAElD,OAAO;YACH,KAAK,EAAE;gBACH,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ;gBACR,QAAQ;gBACR,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM;aACrC;SACJ,CAAC;IACN,CAAC;IAES,KAAK,CAAC,QAAQ,CAAC,cAAkD;QACvE,IAAI,cAAc,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,iBAAiB,EAAE;YACvE,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;SACvG;QAED,gEAAgE;QAChE,IAAI,KAAK,GAAG,KAAK,IAAI,EAAE,GAAE,CAAC,CAAC;QAC3B,IAAI,cAAc,EAAE,KAAK,EAAE;YACvB,MAAM,CAAC,kBAAkB,EAAE,UAAU,CAAC,GAAG,MAAM,IAAA,qCAAmB,EAC9D,cAAc,CAAC,KAAK,CAAC,MAAM,EAC3B,cAAc,CAAC,KAAK,CAAC,QAAQ,EAC7B,cAAc,CAAC,KAAK,CAAC,QAAQ,CAChC,CAAC;YAEF,IAAI,kBAAkB,EAAE;gBACpB,cAAc,CAAC,KAAK,GAAG;oBACnB,MAAM,EAAE,kBAAkB;oBAC1B,MAAM,EAAE,cAAc,CAAC,KAAK,CAAC,MAAM;iBACtC,CAAC;aACL;YAED,KAAK,GAAG,UAAU,CAAC;SACtB;QAED,IAAI;YACA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YACxD,IAAA,mBAAS,GAAE,CAAC;YAEZ,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;gBAC1B,IAAI,CAAC,WAAW,EAAE,CAAC;gBAEnB,MAAM,KAAK,EAAE,CAAC;YAClB,CAAC,CAAC,CAAC;YAEH,OAAO,IAAI,CAAC;SACf;QAAC,OAAO,KAAK,EAAE;YACZ,MAAM,KAAK,EAAE,CAAC;YAEd,MAAM,KAAK,CAAC;SACf;IACL,CAAC;IAES,KAAK,CAAC,MAAM;QAClB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAES,KAAK,CAAC,KAAK;QACjB,wDAAwD;QACxD,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,uFAAuF;IACvH,CAAC;IAES,WAAW,CAAC,IAAU;QAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAC/B,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;IAES,WAAW,CAAC,IAAU,EAAE,OAAiB;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAC/B,OAAO,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;CACJ;AAhFD,oDAgFC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// @ts-ignore optional peer dependency
|
|
2
|
+
import type { Browser as PlaywrightBrowser, BrowserType } from 'playwright';
|
|
3
|
+
import { BrowserController } from '../abstract-classes/browser-controller';
|
|
4
|
+
import { BrowserPlugin } from '../abstract-classes/browser-plugin';
|
|
5
|
+
import { LaunchContext } from '../launch-context';
|
|
6
|
+
export declare class PlaywrightPlugin extends BrowserPlugin<BrowserType, Parameters<BrowserType['launch']>[0], PlaywrightBrowser> {
|
|
7
|
+
private _browserVersion?;
|
|
8
|
+
protected _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
|
|
9
|
+
protected _createController(): BrowserController<BrowserType, Parameters<BrowserType['launch']>[0], PlaywrightBrowser>;
|
|
10
|
+
protected _addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;
|
|
11
|
+
protected _isChromiumBasedBrowser(): boolean;
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=playwright-plugin.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright-plugin.d.ts","sourceRoot":"","sources":["../../src/playwright/playwright-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,IAAI,iBAAiB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAG5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,wCAAwC,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,MAAM,oCAAoC,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAKlD,qBAAa,gBAAiB,SAAQ,aAAa,CAAC,WAAW,EAAE,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,iBAAiB,CAAC;IACrH,OAAO,CAAC,eAAe,CAAC,CAAS;cAEjB,OAAO,CAAC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA+D9F,SAAS,CAAC,iBAAiB,IAAI,iBAAiB,CAAC,WAAW,EAAE,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,iBAAiB,CAAC;cAItG,wBAAwB,CAAC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBlG,SAAS,CAAC,uBAAuB,IAAI,OAAO;CAI/C"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PlaywrightPlugin = void 0;
|
|
4
|
+
const browser_1 = require("./browser");
|
|
5
|
+
const playwright_controller_1 = require("./playwright-controller");
|
|
6
|
+
const browser_plugin_1 = require("../abstract-classes/browser-plugin");
|
|
7
|
+
const logger_1 = require("../logger");
|
|
8
|
+
const proxy_server_1 = require("../proxy-server");
|
|
9
|
+
const anonymize_proxy_1 = require("../anonymize-proxy");
|
|
10
|
+
class PlaywrightPlugin extends browser_plugin_1.BrowserPlugin {
|
|
11
|
+
constructor() {
|
|
12
|
+
super(...arguments);
|
|
13
|
+
Object.defineProperty(this, "_browserVersion", {
|
|
14
|
+
enumerable: true,
|
|
15
|
+
configurable: true,
|
|
16
|
+
writable: true,
|
|
17
|
+
value: void 0
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
async _launch(launchContext) {
|
|
21
|
+
const { launchOptions, useIncognitoPages, userDataDir, proxyUrl, } = launchContext;
|
|
22
|
+
let browser;
|
|
23
|
+
// Required for the `proxy` context option to work.
|
|
24
|
+
launchOptions.proxy = {
|
|
25
|
+
server: await (0, proxy_server_1.getLocalProxyAddress)(),
|
|
26
|
+
...launchOptions.proxy,
|
|
27
|
+
};
|
|
28
|
+
const [anonymizedProxyUrl, close] = await (0, anonymize_proxy_1.anonymizeProxySugar)(proxyUrl);
|
|
29
|
+
if (anonymizedProxyUrl) {
|
|
30
|
+
launchOptions.proxy = {
|
|
31
|
+
server: anonymizedProxyUrl,
|
|
32
|
+
bypass: launchOptions.proxy.bypass,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
try {
|
|
36
|
+
if (useIncognitoPages) {
|
|
37
|
+
browser = await this.library.launch(launchOptions);
|
|
38
|
+
if (anonymizedProxyUrl) {
|
|
39
|
+
browser.on('disconnected', async () => {
|
|
40
|
+
await close();
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
const browserContext = await this.library.launchPersistentContext(userDataDir, launchOptions);
|
|
46
|
+
if (anonymizedProxyUrl) {
|
|
47
|
+
browserContext.on('close', async () => {
|
|
48
|
+
await close();
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
if (!this._browserVersion) {
|
|
52
|
+
// Launches unused browser just to get the browser version.
|
|
53
|
+
const inactiveBrowser = await this.library.launch(launchOptions);
|
|
54
|
+
this._browserVersion = inactiveBrowser.version();
|
|
55
|
+
inactiveBrowser.close().catch((error) => {
|
|
56
|
+
logger_1.log.exception(error, 'Failed to close browser.');
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
browser = new browser_1.Browser({ browserContext, version: this._browserVersion });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
await close();
|
|
64
|
+
throw error;
|
|
65
|
+
}
|
|
66
|
+
return browser;
|
|
67
|
+
}
|
|
68
|
+
_createController() {
|
|
69
|
+
return new playwright_controller_1.PlaywrightController(this);
|
|
70
|
+
}
|
|
71
|
+
async _addProxyToLaunchOptions(launchContext) {
|
|
72
|
+
launchContext.launchOptions ?? (launchContext.launchOptions = {});
|
|
73
|
+
const { launchOptions, proxyUrl } = launchContext;
|
|
74
|
+
if (proxyUrl) {
|
|
75
|
+
const url = new URL(proxyUrl);
|
|
76
|
+
launchOptions.proxy = {
|
|
77
|
+
server: url.origin,
|
|
78
|
+
username: decodeURIComponent(url.username),
|
|
79
|
+
password: decodeURIComponent(url.password),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
_isChromiumBasedBrowser() {
|
|
84
|
+
const name = this.library.name();
|
|
85
|
+
return name === 'chromium';
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
exports.PlaywrightPlugin = PlaywrightPlugin;
|
|
89
|
+
//# sourceMappingURL=playwright-plugin.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright-plugin.js","sourceRoot":"","sources":["../../src/playwright/playwright-plugin.ts"],"names":[],"mappings":";;;AACA,uCAA8E;AAC9E,mEAA+D;AAE/D,uEAAmE;AAEnE,sCAAgC;AAChC,kDAAuD;AACvD,wDAAyD;AAEzD,MAAa,gBAAiB,SAAQ,8BAAmF;IAAzH;;QACI;;;;;WAAiC;IAyFrC,CAAC;IAvFa,KAAK,CAAC,OAAO,CAAC,aAAyC;QAC7D,MAAM,EACF,aAAa,EACb,iBAAiB,EACjB,WAAW,EACX,QAAQ,GACX,GAAG,aAAa,CAAC;QAElB,IAAI,OAA0B,CAAC;QAE/B,mDAAmD;QACnD,aAAc,CAAC,KAAK,GAAG;YACnB,MAAM,EAAE,MAAM,IAAA,mCAAoB,GAAE;YACpC,GAAG,aAAc,CAAC,KAAK;SAC1B,CAAC;QAEF,MAAM,CAAC,kBAAkB,EAAE,KAAK,CAAC,GAAG,MAAM,IAAA,qCAAmB,EAAC,QAAQ,CAAC,CAAC;QACxE,IAAI,kBAAkB,EAAE;YACpB,aAAc,CAAC,KAAK,GAAG;gBACnB,MAAM,EAAE,kBAAkB;gBAC1B,MAAM,EAAE,aAAc,CAAC,KAAK,CAAC,MAAM;aACtC,CAAC;SACL;QAED,IAAI;YACA,IAAI,iBAAiB,EAAE;gBACnB,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;gBAEnD,IAAI,kBAAkB,EAAE;oBACpB,OAAO,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;wBAClC,MAAM,KAAK,EAAE,CAAC;oBAClB,CAAC,CAAC,CAAC;iBACN;aACJ;iBAAM;gBACH,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,uBAAuB,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;gBAE9F,IAAI,kBAAkB,EAAE;oBACpB,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;wBAClC,MAAM,KAAK,EAAE,CAAC;oBAClB,CAAC,CAAC,CAAC;iBACN;gBAED,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;oBACvB,2DAA2D;oBAC3D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBACjE,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC,OAAO,EAAE,CAAC;oBAEjD,eAAe,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;wBACpC,YAAG,CAAC,SAAS,CAAC,KAAK,EAAE,0BAA0B,CAAC,CAAC;oBACrD,CAAC,CAAC,CAAC;iBACN;gBAED,OAAO,GAAG,IAAI,iBAAsC,CAAC,EAAE,cAAc,EAAE,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;aAC3G;SACJ;QAAC,OAAO,KAAK,EAAE;YACZ,MAAM,KAAK,EAAE,CAAC;YAEd,MAAM,KAAK,CAAC;SACf;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAES,iBAAiB;QACvB,OAAO,IAAI,4CAAoB,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IAES,KAAK,CAAC,wBAAwB,CAAC,aAAyC;QAC9E,aAAa,CAAC,aAAa,KAA3B,aAAa,CAAC,aAAa,GAAK,EAAE,EAAC;QAEnC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,GAAG,aAAa,CAAC;QAElD,IAAI,QAAQ,EAAE;YACV,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;YAE9B,aAAa,CAAC,KAAK,GAAG;gBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAC1C,QAAQ,EAAE,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC;aAC7C,CAAC;SACL;IACL,CAAC;IAES,uBAAuB;QAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACjC,OAAO,IAAI,KAAK,UAAU,CAAC;IAC/B,CAAC;CACJ;AA1FD,4CA0FC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"proxy-server.d.ts","sourceRoot":"","sources":["../src/proxy-server.ts"],"names":[],"mappings":"AAaA,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC,CAI5D"}
|
package/proxy-server.js
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getLocalProxyAddress = void 0;
|
|
4
|
+
const proxy_chain_1 = require("proxy-chain");
|
|
5
|
+
const server = new proxy_chain_1.Server({
|
|
6
|
+
port: 0,
|
|
7
|
+
});
|
|
8
|
+
server.server.unref();
|
|
9
|
+
const listenPromise = server.listen();
|
|
10
|
+
// eslint-disable-next-line @typescript-eslint/no-empty-function
|
|
11
|
+
listenPromise.catch(() => { });
|
|
12
|
+
// https://github.com/microsoft/playwright/blob/2e4722d460b5142267e0e506ca7ea9a259556b5f/packages/playwright-core/src/server/browserContext.ts#L423-L427
|
|
13
|
+
async function getLocalProxyAddress() {
|
|
14
|
+
await listenPromise;
|
|
15
|
+
return `http://127.0.0.1:${server.port}`;
|
|
16
|
+
}
|
|
17
|
+
exports.getLocalProxyAddress = getLocalProxyAddress;
|
|
18
|
+
//# sourceMappingURL=proxy-server.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"proxy-server.js","sourceRoot":"","sources":["../src/proxy-server.ts"],"names":[],"mappings":";;;AAAA,6CAAyD;AAEzD,MAAM,MAAM,GAAG,IAAI,oBAAgB,CAAC;IAChC,IAAI,EAAE,CAAC;CACV,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;AAEtB,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;AACtC,gEAAgE;AAChE,aAAa,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AAE9B,wJAAwJ;AACjJ,KAAK,UAAU,oBAAoB;IACtC,MAAM,aAAa,CAAC;IAEpB,OAAO,oBAAoB,MAAM,CAAC,IAAI,EAAE,CAAC;AAC7C,CAAC;AAJD,oDAIC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type Puppeteer from './puppeteer-proxy-per-page';
|
|
2
|
+
import { BrowserController, Cookie } from '../abstract-classes/browser-controller';
|
|
3
|
+
export declare class PuppeteerController extends BrowserController<typeof Puppeteer> {
|
|
4
|
+
normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
|
|
5
|
+
protected _newPage(contextOptions?: Puppeteer.ContextOptions): Promise<Puppeteer.Page>;
|
|
6
|
+
protected _close(): Promise<void>;
|
|
7
|
+
protected _kill(): Promise<void>;
|
|
8
|
+
protected _getCookies(page: Puppeteer.Page): Promise<Cookie[]>;
|
|
9
|
+
protected _setCookies(page: Puppeteer.Page, cookies: Cookie[]): Promise<void>;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=puppeteer-controller.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"puppeteer-controller.d.ts","sourceRoot":"","sources":["../../src/puppeteer/puppeteer-controller.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM,wCAAwC,CAAC;AAMnF,qBAAa,mBAAoB,SAAQ,iBAAiB,CAAC,OAAO,SAAS,CAAC;IACxE,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAAE,WAAW,EAAE,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;cAiB9E,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC;cAuE5E,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;cAIvB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAuBtC,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAI9D,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAGhF"}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PuppeteerController = void 0;
|
|
4
|
+
const timeout_1 = require("@apify/timeout");
|
|
5
|
+
const browser_controller_1 = require("../abstract-classes/browser-controller");
|
|
6
|
+
const logger_1 = require("../logger");
|
|
7
|
+
const anonymize_proxy_1 = require("../anonymize-proxy");
|
|
8
|
+
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
|
|
9
|
+
class PuppeteerController extends browser_controller_1.BrowserController {
|
|
10
|
+
normalizeProxyOptions(proxyUrl, pageOptions) {
|
|
11
|
+
if (!proxyUrl) {
|
|
12
|
+
return {};
|
|
13
|
+
}
|
|
14
|
+
const url = new URL(proxyUrl);
|
|
15
|
+
const username = decodeURIComponent(url.username);
|
|
16
|
+
const password = decodeURIComponent(url.password);
|
|
17
|
+
return {
|
|
18
|
+
proxyServer: url.origin,
|
|
19
|
+
proxyUsername: username,
|
|
20
|
+
proxyPassword: password,
|
|
21
|
+
proxyBypassList: pageOptions?.proxyBypassList,
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
async _newPage(contextOptions) {
|
|
25
|
+
if (contextOptions !== undefined) {
|
|
26
|
+
if (!this.launchContext.useIncognitoPages) {
|
|
27
|
+
throw new Error('A new page can be created with provided context only when using incognito pages.');
|
|
28
|
+
}
|
|
29
|
+
// eslint-disable-next-line @typescript-eslint/no-empty-function
|
|
30
|
+
let close = async () => { };
|
|
31
|
+
if (contextOptions.proxyServer) {
|
|
32
|
+
const [anonymizedProxyUrl, closeProxy] = await (0, anonymize_proxy_1.anonymizeProxySugar)(contextOptions.proxyServer, contextOptions.proxyUsername, contextOptions.proxyPassword);
|
|
33
|
+
if (anonymizedProxyUrl) {
|
|
34
|
+
contextOptions.proxyServer = anonymizedProxyUrl;
|
|
35
|
+
delete contextOptions.proxyUsername;
|
|
36
|
+
delete contextOptions.proxyPassword;
|
|
37
|
+
}
|
|
38
|
+
close = closeProxy;
|
|
39
|
+
}
|
|
40
|
+
try {
|
|
41
|
+
const context = await this.browser.createIncognitoBrowserContext(contextOptions);
|
|
42
|
+
(0, timeout_1.tryCancel)();
|
|
43
|
+
const page = await context.newPage();
|
|
44
|
+
(0, timeout_1.tryCancel)();
|
|
45
|
+
/*
|
|
46
|
+
// DO NOT USE YET! DOING SO DISABLES CACHE WHICH IS 50% PERFORMANCE HIT!
|
|
47
|
+
if (contextOptions.proxyUsername || contextOptions.proxyPassword) {
|
|
48
|
+
await page.authenticate({
|
|
49
|
+
username: contextOptions.proxyUsername ?? '',
|
|
50
|
+
password: contextOptions.proxyPassword ?? '',
|
|
51
|
+
});
|
|
52
|
+
tryCancel();
|
|
53
|
+
}
|
|
54
|
+
*/
|
|
55
|
+
page.once('close', async () => {
|
|
56
|
+
this.activePages--;
|
|
57
|
+
try {
|
|
58
|
+
await context.close();
|
|
59
|
+
}
|
|
60
|
+
catch (error) {
|
|
61
|
+
logger_1.log.exception(error, 'Failed to close context.');
|
|
62
|
+
}
|
|
63
|
+
finally {
|
|
64
|
+
await close();
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
return page;
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
await close();
|
|
71
|
+
throw error;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
const page = await this.browser.newPage();
|
|
75
|
+
(0, timeout_1.tryCancel)();
|
|
76
|
+
page.once('close', () => {
|
|
77
|
+
this.activePages--;
|
|
78
|
+
});
|
|
79
|
+
return page;
|
|
80
|
+
}
|
|
81
|
+
async _close() {
|
|
82
|
+
await this.browser.close();
|
|
83
|
+
}
|
|
84
|
+
async _kill() {
|
|
85
|
+
const browserProcess = this.browser.process();
|
|
86
|
+
if (!browserProcess) {
|
|
87
|
+
// TODO: LOG browser was connected using the `puppeteer.connect` method no browser to kill.
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
const timeout = setTimeout(() => {
|
|
91
|
+
// This is here because users reported that it happened
|
|
92
|
+
// that error `TypeError: Cannot read property 'kill' of null` was thrown.
|
|
93
|
+
// Likely Chrome process wasn't started due to some error ...
|
|
94
|
+
browserProcess?.kill('SIGKILL');
|
|
95
|
+
}, PROCESS_KILL_TIMEOUT_MILLIS);
|
|
96
|
+
try {
|
|
97
|
+
await this.browser.close();
|
|
98
|
+
clearTimeout(timeout);
|
|
99
|
+
}
|
|
100
|
+
catch (e) {
|
|
101
|
+
// TODO: LOG Browser was already killed.
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
_getCookies(page) {
|
|
105
|
+
return page.cookies();
|
|
106
|
+
}
|
|
107
|
+
_setCookies(page, cookies) {
|
|
108
|
+
return page.setCookie(...cookies);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
exports.PuppeteerController = PuppeteerController;
|
|
112
|
+
//# sourceMappingURL=puppeteer-controller.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"puppeteer-controller.js","sourceRoot":"","sources":["../../src/puppeteer/puppeteer-controller.ts"],"names":[],"mappings":";;;AAAA,4CAA2C;AAE3C,+EAAmF;AACnF,sCAAgC;AAChC,wDAAyD;AAEzD,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAEzC,MAAa,mBAAoB,SAAQ,sCAAmC;IACxE,qBAAqB,CAAC,QAA4B,EAAE,WAAgB;QAChE,IAAI,CAAC,QAAQ,EAAE;YACX,OAAO,EAAE,CAAC;SACb;QAED,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAElD,OAAO;YACH,WAAW,EAAE,GAAG,CAAC,MAAM;YACvB,aAAa,EAAE,QAAQ;YACvB,aAAa,EAAE,QAAQ;YACvB,eAAe,EAAE,WAAW,EAAE,eAAe;SAChD,CAAC;IACN,CAAC;IAES,KAAK,CAAC,QAAQ,CAAC,cAAyC;QAC9D,IAAI,cAAc,KAAK,SAAS,EAAE;YAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,iBAAiB,EAAE;gBACvC,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;aACvG;YAED,gEAAgE;YAChE,IAAI,KAAK,GAAG,KAAK,IAAI,EAAE,GAAE,CAAC,CAAC;YAC3B,IAAI,cAAc,CAAC,WAAW,EAAE;gBAC5B,MAAM,CAAC,kBAAkB,EAAE,UAAU,CAAC,GAAG,MAAM,IAAA,qCAAmB,EAC9D,cAAc,CAAC,WAAW,EAC1B,cAAc,CAAC,aAAa,EAC5B,cAAc,CAAC,aAAa,CAC/B,CAAC;gBAEF,IAAI,kBAAkB,EAAE;oBACpB,cAAc,CAAC,WAAW,GAAG,kBAAkB,CAAC;oBAChD,OAAO,cAAc,CAAC,aAAa,CAAC;oBACpC,OAAO,cAAc,CAAC,aAAa,CAAC;iBACvC;gBAED,KAAK,GAAG,UAAU,CAAC;aACtB;YAED,IAAI;gBACA,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,6BAA6B,CAAC,cAAc,CAAC,CAAC;gBACjF,IAAA,mBAAS,GAAE,CAAC;gBACZ,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;gBACrC,IAAA,mBAAS,GAAE,CAAC;gBAEZ;;;;;;;;;kBASE;gBAEF,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;oBAC1B,IAAI,CAAC,WAAW,EAAE,CAAC;oBAEnB,IAAI;wBACA,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;qBACzB;oBAAC,OAAO,KAAU,EAAE;wBACjB,YAAG,CAAC,SAAS,CAAC,KAAK,EAAE,0BAA0B,CAAC,CAAC;qBACpD;4BAAS;wBACN,MAAM,KAAK,EAAE,CAAC;qBACjB;gBACL,CAAC,CAAC,CAAC;gBAEH,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,KAAK,EAAE;gBACZ,MAAM,KAAK,EAAE,CAAC;gBAEd,MAAM,KAAK,CAAC;aACf;SACJ;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC1C,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE;YACpB,IAAI,CAAC,WAAW,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC;IAChB,CAAC;IAES,KAAK,CAAC,MAAM;QAClB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAES,KAAK,CAAC,KAAK;QACjB,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAE9C,IAAI,CAAC,cAAc,EAAE;YACjB,2FAA2F;YAC3F,OAAO;SACV;QAED,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,uDAAuD;YACvD,0EAA0E;YAC1E,6DAA6D;YAC7D,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACpC,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAEhC,IAAI;YACA,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,YAAY,CAAC,OAAO,CAAC,CAAC;SACzB;QAAC,OAAO,CAAC,EAAE;YACR,wCAAwC;SAC3C;IACL,CAAC;IAES,WAAW,CAAC,IAAoB;QACtC,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAES,WAAW,CAAC,IAAoB,EAAE,OAAiB;QACzD,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,CAAC;IACtC,CAAC;CACJ;AA3HD,kDA2HC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type Puppeteer from './puppeteer-proxy-per-page';
|
|
2
|
+
import type { Browser } from './puppeteer-proxy-per-page';
|
|
3
|
+
import { BrowserController } from '../abstract-classes/browser-controller';
|
|
4
|
+
import { BrowserPlugin } from '../abstract-classes/browser-plugin';
|
|
5
|
+
import { LaunchContext } from '../launch-context';
|
|
6
|
+
export declare class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer> {
|
|
7
|
+
protected _launch(launchContext: LaunchContext<typeof Puppeteer>): Promise<Browser>;
|
|
8
|
+
protected _createController(): BrowserController<typeof Puppeteer>;
|
|
9
|
+
protected _addProxyToLaunchOptions(launchContext: LaunchContext<typeof Puppeteer>): Promise<void>;
|
|
10
|
+
protected _isChromiumBasedBrowser(launchContext: LaunchContext<typeof Puppeteer>): boolean;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=puppeteer-plugin.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"puppeteer-plugin.d.ts","sourceRoot":"","sources":["../../src/puppeteer/puppeteer-plugin.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,4BAA4B,CAAC;AACxD,OAAO,KAAK,EAAE,OAAO,EAA0B,MAAM,4BAA4B,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wCAAwC,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,MAAM,oCAAoC,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAQlD,qBAAa,eAAgB,SAAQ,aAAa,CAAC,OAAO,SAAS,CAAC;cAChD,OAAO,CAAC,aAAa,EAAE,aAAa,CAAC,OAAO,SAAS,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC;IA2HzF,SAAS,CAAC,iBAAiB,IAAI,iBAAiB,CAAC,OAAO,SAAS,CAAC;cAIlD,wBAAwB,CACpC,aAAa,EAAE,aAAa,CAAC,OAAO,SAAS,CAAC,GAC/C,OAAO,CAAC,IAAI,CAAC;IA8BhB,SAAS,CAAC,uBAAuB,CAAC,aAAa,EAAE,aAAa,CAAC,OAAO,SAAS,CAAC,GAAG,OAAO;CAM7F"}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PuppeteerPlugin = void 0;
|
|
4
|
+
const browser_plugin_1 = require("../abstract-classes/browser-plugin");
|
|
5
|
+
const logger_1 = require("../logger");
|
|
6
|
+
const utils_1 = require("../utils");
|
|
7
|
+
const puppeteer_controller_1 = require("./puppeteer-controller");
|
|
8
|
+
const anonymize_proxy_1 = require("../anonymize-proxy");
|
|
9
|
+
const PROXY_SERVER_ARG = '--proxy-server=';
|
|
10
|
+
class PuppeteerPlugin extends browser_plugin_1.BrowserPlugin {
|
|
11
|
+
async _launch(launchContext) {
|
|
12
|
+
const { launchOptions, userDataDir, useIncognitoPages, proxyUrl, } = launchContext;
|
|
13
|
+
launchOptions.userDataDir = launchOptions.userDataDir ?? userDataDir;
|
|
14
|
+
if (launchOptions.headless === false) {
|
|
15
|
+
if (Array.isArray(launchOptions.args)) {
|
|
16
|
+
launchOptions.args.push('--disable-site-isolation-trials');
|
|
17
|
+
}
|
|
18
|
+
else {
|
|
19
|
+
launchOptions.args = ['--disable-site-isolation-trials'];
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
let browser;
|
|
23
|
+
{
|
|
24
|
+
const [anonymizedProxyUrl, close] = await (0, anonymize_proxy_1.anonymizeProxySugar)(proxyUrl);
|
|
25
|
+
if (proxyUrl) {
|
|
26
|
+
const proxyArg = `${PROXY_SERVER_ARG}${anonymizedProxyUrl ?? proxyUrl}`;
|
|
27
|
+
if (Array.isArray(launchOptions.args)) {
|
|
28
|
+
launchOptions.args.push(proxyArg);
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
launchOptions.args = [proxyArg];
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
try {
|
|
35
|
+
browser = await this.library.launch(launchOptions);
|
|
36
|
+
if (anonymizedProxyUrl) {
|
|
37
|
+
browser.on('disconnected', async () => {
|
|
38
|
+
await close();
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
await close();
|
|
44
|
+
throw error;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
browser.on('targetcreated', async (target) => {
|
|
48
|
+
try {
|
|
49
|
+
const page = await target.page();
|
|
50
|
+
if (page) {
|
|
51
|
+
page.on('error', (error) => {
|
|
52
|
+
logger_1.log.exception(error, 'Page crashed.');
|
|
53
|
+
page.close().catch(utils_1.noop);
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
logger_1.log.exception(error, 'Failed to retrieve page from target.');
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
const newPage = browser.newPage.bind(browser);
|
|
62
|
+
browser = new Proxy(browser, {
|
|
63
|
+
get: (target, property) => {
|
|
64
|
+
if (property === 'newPage') {
|
|
65
|
+
return (async (...args) => {
|
|
66
|
+
let page;
|
|
67
|
+
if (useIncognitoPages) {
|
|
68
|
+
const [anonymizedProxyUrl, close] = await (0, anonymize_proxy_1.anonymizeProxySugar)(proxyUrl);
|
|
69
|
+
try {
|
|
70
|
+
const context = await browser.createIncognitoBrowserContext({
|
|
71
|
+
proxyServer: anonymizedProxyUrl ?? proxyUrl,
|
|
72
|
+
});
|
|
73
|
+
page = await context.newPage(...args);
|
|
74
|
+
if (anonymizedProxyUrl) {
|
|
75
|
+
page.on('close', async () => {
|
|
76
|
+
await close();
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
await close();
|
|
82
|
+
throw error;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
page = await newPage(...args);
|
|
87
|
+
}
|
|
88
|
+
/*
|
|
89
|
+
// DO NOT USE YET! DOING SO DISABLES CACHE WHICH IS 50% PERFORMANCE HIT!
|
|
90
|
+
if (useIncognitoPages) {
|
|
91
|
+
const context = await browser.createIncognitoBrowserContext({
|
|
92
|
+
proxyServer: proxyUrl,
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
page = await context.newPage(...args);
|
|
96
|
+
} else {
|
|
97
|
+
page = await newPage(...args);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (proxyCredentials) {
|
|
101
|
+
await page.authenticate(proxyCredentials as Credentials);
|
|
102
|
+
}
|
|
103
|
+
*/
|
|
104
|
+
return page;
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
return target[property];
|
|
108
|
+
},
|
|
109
|
+
});
|
|
110
|
+
return browser;
|
|
111
|
+
}
|
|
112
|
+
_createController() {
|
|
113
|
+
return new puppeteer_controller_1.PuppeteerController(this);
|
|
114
|
+
}
|
|
115
|
+
async _addProxyToLaunchOptions(launchContext) {
|
|
116
|
+
launchContext;
|
|
117
|
+
/*
|
|
118
|
+
// DO NOT USE YET! DOING SO DISABLES CACHE WHICH IS 50% PERFORMANCE HIT!
|
|
119
|
+
launchContext.launchOptions ??= {};
|
|
120
|
+
|
|
121
|
+
const { launchOptions, proxyUrl } = launchContext;
|
|
122
|
+
|
|
123
|
+
if (proxyUrl) {
|
|
124
|
+
const url = new URL(proxyUrl);
|
|
125
|
+
|
|
126
|
+
if (url.username || url.password) {
|
|
127
|
+
launchContext.proxyCredentials = {
|
|
128
|
+
username: decodeURIComponent(url.username),
|
|
129
|
+
password: decodeURIComponent(url.password),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const proxyArg = `${PROXY_SERVER_ARG}${url.origin}`;
|
|
134
|
+
|
|
135
|
+
if (Array.isArray(launchOptions.args)) {
|
|
136
|
+
launchOptions.args.push(proxyArg);
|
|
137
|
+
} else {
|
|
138
|
+
launchOptions.args = [proxyArg];
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
*/
|
|
142
|
+
}
|
|
143
|
+
_isChromiumBasedBrowser(launchContext) {
|
|
144
|
+
const { launchOptions } = launchContext;
|
|
145
|
+
// @ts-expect-error cannot find .product on this.library
|
|
146
|
+
const browserName = launchOptions.product || this.library.product;
|
|
147
|
+
return browserName === 'chrome';
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
exports.PuppeteerPlugin = PuppeteerPlugin;
|
|
151
|
+
//# sourceMappingURL=puppeteer-plugin.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"puppeteer-plugin.js","sourceRoot":"","sources":["../../src/puppeteer/puppeteer-plugin.ts"],"names":[],"mappings":";;;AAIA,uEAAmE;AAEnE,sCAAgC;AAChC,oCAAgC;AAChC,iEAA6D;AAC7D,wDAAyD;AAEzD,MAAM,gBAAgB,GAAG,iBAAiB,CAAC;AAE3C,MAAa,eAAgB,SAAQ,8BAA+B;IACtD,KAAK,CAAC,OAAO,CAAC,aAA8C;QAClE,MAAM,EACF,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,QAAQ,GACX,GAAG,aAAa,CAAC;QAElB,aAAc,CAAC,WAAW,GAAG,aAAc,CAAC,WAAW,IAAI,WAAW,CAAC;QAEvE,IAAI,aAAc,CAAC,QAAQ,KAAK,KAAK,EAAE;YACnC,IAAI,KAAK,CAAC,OAAO,CAAC,aAAc,CAAC,IAAI,CAAC,EAAE;gBACpC,aAAc,CAAC,IAAI,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;aAC/D;iBAAM;gBACH,aAAc,CAAC,IAAI,GAAG,CAAC,iCAAiC,CAAC,CAAC;aAC7D;SACJ;QAED,IAAI,OAA0B,CAAC;QAE/B;YACI,MAAM,CAAC,kBAAkB,EAAE,KAAK,CAAC,GAAG,MAAM,IAAA,qCAAmB,EAAC,QAAQ,CAAC,CAAC;YAExE,IAAI,QAAQ,EAAE;gBACV,MAAM,QAAQ,GAAG,GAAG,gBAAgB,GAAG,kBAAkB,IAAI,QAAQ,EAAE,CAAC;gBAExE,IAAI,KAAK,CAAC,OAAO,CAAC,aAAc,CAAC,IAAI,CAAC,EAAE;oBACpC,aAAc,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;iBACtC;qBAAM;oBACH,aAAc,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;iBACpC;aACJ;YAED,IAAI;gBACA,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;gBAEnD,IAAI,kBAAkB,EAAE;oBACpB,OAAO,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;wBAClC,MAAM,KAAK,EAAE,CAAC;oBAClB,CAAC,CAAC,CAAC;iBACN;aACJ;YAAC,OAAO,KAAK,EAAE;gBACZ,MAAM,KAAK,EAAE,CAAC;gBAEd,MAAM,KAAK,CAAC;aACf;SACJ;QAED,OAAO,CAAC,EAAE,CAAC,eAAe,EAAE,KAAK,EAAE,MAAc,EAAE,EAAE;YACjD,IAAI;gBACA,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAEjC,IAAI,IAAI,EAAE;oBACN,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;wBACvB,YAAG,CAAC,SAAS,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC;wBACtC,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,YAAI,CAAC,CAAC;oBAC7B,CAAC,CAAC,CAAC;iBACN;aACJ;YAAC,OAAO,KAAU,EAAE;gBACjB,YAAG,CAAC,SAAS,CAAC,KAAK,EAAE,sCAAsC,CAAC,CAAC;aAChE;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAE9C,OAAO,GAAG,IAAI,KAAK,CAAC,OAAO,EAAE;YACzB,GAAG,EAAE,CAAC,MAAM,EAAE,QAA8B,EAAE,EAAE;gBAC5C,IAAI,QAAQ,KAAK,SAAS,EAAE;oBACxB,OAAO,CAAC,KAAK,EAAE,GAAG,IAA2C,EAAE,EAAE;wBAC7D,IAAI,IAAoB,CAAC;wBAEzB,IAAI,iBAAiB,EAAE;4BACnB,MAAM,CAAC,kBAAkB,EAAE,KAAK,CAAC,GAAG,MAAM,IAAA,qCAAmB,EAAC,QAAQ,CAAC,CAAC;4BAExE,IAAI;gCACA,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,6BAA6B,CAAC;oCACxD,WAAW,EAAE,kBAAkB,IAAI,QAAQ;iCAC9C,CAAC,CAAC;gCAEH,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;gCAEtC,IAAI,kBAAkB,EAAE;oCACpB,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;wCACxB,MAAM,KAAK,EAAE,CAAC;oCAClB,CAAC,CAAC,CAAC;iCACN;6BACJ;4BAAC,OAAO,KAAK,EAAE;gCACZ,MAAM,KAAK,EAAE,CAAC;gCAEd,MAAM,KAAK,CAAC;6BACf;yBACJ;6BAAM;4BACH,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;yBACjC;wBAED;;;;;;;;;;;;;;;0BAeE;wBAEF,OAAO,IAAI,CAAC;oBAChB,CAAC,CAAC,CAAC;iBACN;gBAED,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC5B,CAAC;SACJ,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC;IACnB,CAAC;IAES,iBAAiB;QACvB,OAAO,IAAI,0CAAmB,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC;IAES,KAAK,CAAC,wBAAwB,CACpC,aAA8C;QAE9C,aAAwB,CAAC;QAEzB;;;;;;;;;;;;;;;;;;;;;;;;UAwBE;IACN,CAAC;IAES,uBAAuB,CAAC,aAA8C;QAC5E,MAAM,EAAE,aAAa,EAAE,GAAG,aAAoB,CAAC;QAC/C,wDAAwD;QACxD,MAAM,WAAW,GAAG,aAAa,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,OAAQ,CAAC;QACnE,OAAO,WAAW,KAAK,QAAQ,CAAC;IACpC,CAAC;CACJ;AAtKD,0CAsKC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
BrowserContextOptions,
|
|
3
|
+
Browser as PuppeteerBrowser,
|
|
4
|
+
Page,
|
|
5
|
+
LaunchOptions,
|
|
6
|
+
BrowserLaunchArgumentOptions,
|
|
7
|
+
BrowserConnectOptions,
|
|
8
|
+
Product,
|
|
9
|
+
// @ts-ignore optional peer dependency
|
|
10
|
+
} from 'puppeteer';
|
|
11
|
+
|
|
12
|
+
// @ts-ignore optional peer dependency
|
|
13
|
+
export * from 'puppeteer';
|
|
14
|
+
|
|
15
|
+
export interface ContextOptions extends BrowserContextOptions {
|
|
16
|
+
proxyUsername?: string;
|
|
17
|
+
proxyPassword?: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export declare class Browser extends PuppeteerBrowser {
|
|
21
|
+
newPage: (contextOptions?: ContextOptions) => Promise<Page>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export declare function launch(options?: LaunchOptions & BrowserLaunchArgumentOptions & BrowserConnectOptions & {
|
|
25
|
+
product?: Product;
|
|
26
|
+
extraPrefsFirefox?: Record<string, unknown>;
|
|
27
|
+
}): Promise<Browser>;
|