@crawlee/puppeteer 3.0.0-alpha.5 → 3.0.0-alpha.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.d.ts CHANGED
@@ -1,8 +1,6 @@
1
1
  export * from '@crawlee/browser';
2
2
  export * from './internals/puppeteer-crawler';
3
3
  export * from './internals/puppeteer-launcher';
4
- export * as puppeteerStealth from './internals/stealth';
5
- export type { StealthOptions } from './internals/stealth';
6
4
  export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception';
7
5
  export type { InterceptHandler } from './internals/utils/puppeteer_request_interception';
8
6
  export * as puppeteerUtils from './internals/utils/puppeteer_utils';
package/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAE/C,OAAO,KAAK,gBAAgB,MAAM,qBAAqB,CAAC;AACxD,YAAY,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAE1D,OAAO,KAAK,4BAA4B,MAAM,kDAAkD,CAAC;AACjG,YAAY,EAAE,gBAAgB,EAAE,MAAM,kDAAkD,CAAC;AAEzF,OAAO,KAAK,cAAc,MAAM,mCAAmC,CAAC;AACpE,YAAY,EACR,oBAAoB,EACpB,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,IAAI,gCAAgC,EAC3D,qBAAqB,EACrB,iBAAiB,EACjB,mBAAmB,GACtB,MAAM,mCAAmC,CAAC;AAE3C,OAAO,KAAK,sBAAsB,MAAM,0CAA0C,CAAC;AACnF,YAAY,EAAE,qCAAqC,EAAE,MAAM,0CAA0C,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAE/C,OAAO,KAAK,4BAA4B,MAAM,kDAAkD,CAAC;AACjG,YAAY,EAAE,gBAAgB,EAAE,MAAM,kDAAkD,CAAC;AAEzF,OAAO,KAAK,cAAc,MAAM,mCAAmC,CAAC;AACpE,YAAY,EACR,oBAAoB,EACpB,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,IAAI,gCAAgC,EAC3D,qBAAqB,EACrB,iBAAiB,EACjB,mBAAmB,GACtB,MAAM,mCAAmC,CAAC;AAE3C,OAAO,KAAK,sBAAsB,MAAM,0CAA0C,CAAC;AACnF,YAAY,EAAE,qCAAqC,EAAE,MAAM,0CAA0C,CAAC"}
package/index.js CHANGED
@@ -1,11 +1,10 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.puppeteerClickElements = exports.puppeteerUtils = exports.puppeteerRequestInterception = exports.puppeteerStealth = void 0;
3
+ exports.puppeteerClickElements = exports.puppeteerUtils = exports.puppeteerRequestInterception = void 0;
4
4
  const tslib_1 = require("tslib");
5
5
  tslib_1.__exportStar(require("@crawlee/browser"), exports);
6
6
  tslib_1.__exportStar(require("./internals/puppeteer-crawler"), exports);
7
7
  tslib_1.__exportStar(require("./internals/puppeteer-launcher"), exports);
8
- exports.puppeteerStealth = tslib_1.__importStar(require("./internals/stealth"));
9
8
  exports.puppeteerRequestInterception = tslib_1.__importStar(require("./internals/utils/puppeteer_request_interception"));
10
9
  exports.puppeteerUtils = tslib_1.__importStar(require("./internals/utils/puppeteer_utils"));
11
10
  exports.puppeteerClickElements = tslib_1.__importStar(require("./internals/enqueue-links/click-elements"));
package/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,2DAAiC;AACjC,wEAA8C;AAC9C,yEAA+C;AAE/C,gFAAwD;AAGxD,yHAAiG;AAGjG,4FAAoE;AAWpE,2GAAmF"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,2DAAiC;AACjC,wEAA8C;AAC9C,yEAA+C;AAE/C,yHAAiG;AAGjG,4FAAoE;AAWpE,2GAAmF"}
package/index.mjs CHANGED
@@ -64,7 +64,6 @@ export const mergeCookies = mod.mergeCookies;
64
64
  export const openDataset = mod.openDataset;
65
65
  export const puppeteerClickElements = mod.puppeteerClickElements;
66
66
  export const puppeteerRequestInterception = mod.puppeteerRequestInterception;
67
- export const puppeteerStealth = mod.puppeteerStealth;
68
67
  export const puppeteerUtils = mod.puppeteerUtils;
69
68
  export const pushData = mod.pushData;
70
69
  export const resolveBaseUrl = mod.resolveBaseUrl;
@@ -114,13 +114,13 @@ export declare class PuppeteerCrawler extends BrowserCrawler<{
114
114
  }, LaunchOptions, PuppeteerCrawlContext> {
115
115
  protected static optionsShape: {
116
116
  browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
117
- launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
118
117
  handlePageFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
119
118
  gotoFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
120
119
  gotoTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
121
120
  navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
122
121
  preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
123
122
  postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
123
+ launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
124
124
  sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
125
125
  persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
126
126
  useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
@@ -1 +1 @@
1
- {"version":3,"file":"puppeteer-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,cAAc,EACd,2BAA2B,EAC3B,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,EACd,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EAAsB,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE5E,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAqB,MAAM,sBAAsB,CAAC;AAEjF,OAAO,EAAE,uBAAuB,EAAgB,MAAM,yBAAyB,CAAC;AAEhF,oBAAY,mBAAmB,GAAG,UAAU,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC;AAEnF,oBAAY,qBAAqB,GAAG,sBAAsB,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAA;AAEnG,oBAAY,aAAa,GAAG,WAAW,CAAC,qBAAqB,EAAE,oBAAoB,CAAC,CAAC;AAErF,oBAAY,4BAA4B,GAAG,sBAAsB,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAA;AAE1G,oBAAY,uBAAuB,GAAG,2BAA2B,CAAC,4BAA4B,CAAC,CAAC;AAEhG,oBAAY,oBAAoB,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,uBAAwB,SAAQ,qBAAqB,CAClE,qBAAqB,EACrB,oBAAoB,EACpB;IAAE,cAAc,EAAE,CAAC,eAAe,CAAC,CAAA;CAAE,CACxC;IACG;;OAEG;IACH,aAAa,CAAC,EAAE,sBAAsB,CAAC;IAEvC;;;;;;;;;;;;;OAaG;IACH,kBAAkB,CAAC,EAAE,aAAa,EAAE,CAAC;IAErC;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,aAAa,EAAE,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4DG;AACH,qBAAa,gBAAiB,SAAQ,cAAc,CAAC;IAAE,cAAc,EAAE,CAAC,eAAe,CAAC,CAAA;CAAE,EAAE,aAAa,EAAE,qBAAqB,CAAC;IAC7H,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;MAIpC;IAEF;;OAEG;gBACS,OAAO,EAAE,uBAAuB;cAqCnB,kBAAkB,CAAC,eAAe,EAAE,qBAAqB,EAAE,WAAW,EAAE,uBAAuB;CAQ3H"}
1
+ {"version":3,"file":"puppeteer-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,cAAc,EACd,2BAA2B,EAC3B,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,EACd,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EAAsB,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE5E,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAqB,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAE,uBAAuB,EAAgB,MAAM,yBAAyB,CAAC;AAEhF,oBAAY,mBAAmB,GAAG,UAAU,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC;AAEnF,oBAAY,qBAAqB,GAAG,sBAAsB,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAA;AAEnG,oBAAY,aAAa,GAAG,WAAW,CAAC,qBAAqB,EAAE,oBAAoB,CAAC,CAAC;AAErF,oBAAY,4BAA4B,GAAG,sBAAsB,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAA;AAE1G,oBAAY,uBAAuB,GAAG,2BAA2B,CAAC,4BAA4B,CAAC,CAAC;AAEhG,oBAAY,oBAAoB,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,uBAAwB,SAAQ,qBAAqB,CAClE,qBAAqB,EACrB,oBAAoB,EACpB;IAAE,cAAc,EAAE,CAAC,eAAe,CAAC,CAAA;CAAE,CACxC;IACG;;OAEG;IACH,aAAa,CAAC,EAAE,sBAAsB,CAAC;IAEvC;;;;;;;;;;;;;OAaG;IACH,kBAAkB,CAAC,EAAE,aAAa,EAAE,CAAC;IAErC;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,aAAa,EAAE,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4DG;AACH,qBAAa,gBAAiB,SAAQ,cAAc,CAAC;IAAE,cAAc,EAAE,CAAC,eAAe,CAAC,CAAA;CAAE,EAAE,aAAa,EAAE,qBAAqB,CAAC;IAC7H,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;MAGpC;IAEF;;OAEG;gBACS,OAAO,EAAE,uBAAuB;cAwBnB,kBAAkB,CAAC,eAAe,EAAE,qBAAqB,EAAE,WAAW,EAAE,uBAAuB;CAS3H"}
@@ -5,7 +5,6 @@ const tslib_1 = require("tslib");
5
5
  const browser_1 = require("@crawlee/browser");
6
6
  const ow_1 = tslib_1.__importDefault(require("ow"));
7
7
  const puppeteer_launcher_1 = require("./puppeteer-launcher");
8
- const stealth_1 = require("./stealth");
9
8
  const puppeteer_utils_1 = require("./utils/puppeteer_utils");
10
9
  /**
11
10
  * Provides a simple framework for parallel crawling of web pages
@@ -75,7 +74,6 @@ class PuppeteerCrawler extends browser_1.BrowserCrawler {
75
74
  constructor(options) {
76
75
  (0, ow_1.default)(options, 'PuppeteerCrawlerOptions', ow_1.default.object.exactShape(PuppeteerCrawler.optionsShape));
77
76
  const { launchContext = {}, browserPoolOptions = {}, proxyConfiguration, ...browserCrawlerOptions } = options;
78
- const { stealth = false } = launchContext;
79
77
  if (launchContext.proxyUrl) {
80
78
  throw new Error('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.'
81
79
  + 'Use PuppeteerCrawlerOptions.proxyConfiguration');
@@ -84,18 +82,10 @@ class PuppeteerCrawler extends browser_1.BrowserCrawler {
84
82
  browserPoolOptions.browserPlugins = [
85
83
  puppeteerLauncher.createBrowserPlugin(),
86
84
  ];
87
- super({ ...browserCrawlerOptions, proxyConfiguration, browserPoolOptions });
88
- if (stealth) {
89
- this.browserPool.postLaunchHooks.push(async (_pageId, browserController) => {
90
- // TODO: We can do this better now. It is not necessary to override the page.
91
- // we can modify the page in the postPageCreateHook
92
- const { hideWebDriver, ...newStealthOptions } = puppeteerLauncher.stealthOptions;
93
- (0, stealth_1.applyStealthToBrowser)(browserController.browser, newStealthOptions);
94
- });
95
- }
96
- this.launchContext = launchContext;
85
+ super({ ...browserCrawlerOptions, launchContext, proxyConfiguration, browserPoolOptions });
97
86
  }
98
87
  async _navigationHandler(crawlingContext, gotoOptions) {
88
+ // TODO remove deprecated options in v3
99
89
  if (this.gotoFunction) {
100
90
  this.log.deprecated('PuppeteerCrawlerOptions.gotoFunction is deprecated. Use "preNavigationHooks" and "postNavigationHooks" instead.');
101
91
  return this.gotoFunction(crawlingContext, gotoOptions);
@@ -111,7 +101,6 @@ Object.defineProperty(PuppeteerCrawler, "optionsShape", {
111
101
  value: {
112
102
  ...browser_1.BrowserCrawler.optionsShape,
113
103
  browserPoolOptions: ow_1.default.optional.object,
114
- launchContext: ow_1.default.optional.object,
115
104
  }
116
105
  });
117
106
  //# sourceMappingURL=puppeteer-crawler.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"puppeteer-crawler.js","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":";;;;AAAA,8CAM0B;AAK1B,oDAAoB;AAEpB,6DAAiF;AACjF,uCAAkD;AAClD,6DAAgF;AA0DhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4DG;AACH,MAAa,gBAAiB,SAAQ,wBAA2F;IAO7H;;OAEG;IACH,YAAY,OAAgC;QACxC,IAAA,YAAE,EAAC,OAAO,EAAE,yBAAyB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE5F,MAAM,EACF,aAAa,GAAG,EAAE,EAClB,kBAAkB,GAAG,EAAwB,EAC7C,kBAAkB,EAClB,GAAG,qBAAqB,EAC3B,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,OAAO,GAAG,KAAK,EAAE,GAAG,aAAa,CAAC;QAE1C,IAAI,aAAa,CAAC,QAAQ,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,oFAAoF;kBAC9F,gDAAgD,CAAC,CAAC;SAC3D;QAED,MAAM,iBAAiB,GAAG,IAAI,sCAAiB,CAAC,aAAa,CAAC,CAAC;QAE/D,kBAAkB,CAAC,cAAc,GAAG;YAChC,iBAAiB,CAAC,mBAAmB,EAAE;SAC1C,CAAC;QAEF,KAAK,CAAC,EAAE,GAAG,qBAAqB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAE5E,IAAI,OAAO,EAAE;YACT,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,iBAAiB,EAAE,EAAE;gBACvE,6EAA6E;gBAC7E,oDAAoD;gBACpD,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,EAAE,GAAG,iBAAiB,CAAC,cAAe,CAAC;gBAClF,IAAA,+BAAqB,EAAC,iBAAiB,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;YACxE,CAAC,CAAC,CAAC;SACN;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CAAC,eAAsC,EAAE,WAAoC;QACpH,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,iHAAiH,CAAC,CAAC;YAEvI,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,WAAyB,CAAC,CAAC;SACxE;QACD,OAAO,IAAA,8BAAY,EAAC,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACpF,CAAC;;AAtDL,4CAuDC;AAtDG;;;;WAAyC;QACrC,GAAG,wBAAc,CAAC,YAAY;QAC9B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACpC;GAAC"}
1
+ {"version":3,"file":"puppeteer-crawler.js","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":";;;;AAAA,8CAM0B;AAK1B,oDAAoB;AAEpB,6DAAiF;AACjF,6DAAgF;AA0DhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4DG;AACH,MAAa,gBAAiB,SAAQ,wBAA2F;IAM7H;;OAEG;IACH,YAAY,OAAgC;QACxC,IAAA,YAAE,EAAC,OAAO,EAAE,yBAAyB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE5F,MAAM,EACF,aAAa,GAAG,EAAE,EAClB,kBAAkB,GAAG,EAAwB,EAC7C,kBAAkB,EAClB,GAAG,qBAAqB,EAC3B,GAAG,OAAO,CAAC;QAEZ,IAAI,aAAa,CAAC,QAAQ,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,oFAAoF;kBAC9F,gDAAgD,CAAC,CAAC;SAC3D;QAED,MAAM,iBAAiB,GAAG,IAAI,sCAAiB,CAAC,aAAa,CAAC,CAAC;QAE/D,kBAAkB,CAAC,cAAc,GAAG;YAChC,iBAAiB,CAAC,mBAAmB,EAAE;SAC1C,CAAC;QAEF,KAAK,CAAC,EAAE,GAAG,qBAAqB,EAAE,aAAa,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC/F,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CAAC,eAAsC,EAAE,WAAoC;QACpH,uCAAuC;QACvC,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,iHAAiH,CAAC,CAAC;YAEvI,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,WAAyB,CAAC,CAAC;SACxE;QACD,OAAO,IAAA,8BAAY,EAAC,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACpF,CAAC;;AAzCL,4CA0CC;AAzCG;;;;WAAyC;QACrC,GAAG,wBAAc,CAAC,YAAY;QAC9B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACzC;GAAC"}
@@ -1,7 +1,6 @@
1
1
  import { Browser } from 'puppeteer';
2
2
  import { PuppeteerPlugin } from '@crawlee/browser-pool';
3
3
  import { BrowserLaunchContext, BrowserLauncher } from '@crawlee/browser';
4
- import { StealthOptions } from './stealth';
5
4
  /**
6
5
  * Apify extends the launch options of Puppeteer.
7
6
  * You can use any of the Puppeteer compatible
@@ -35,12 +34,6 @@ export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPl
35
34
  * Example: `http://bob:pass123@proxy.example.com:1234`.
36
35
  */
37
36
  proxyUrl?: string;
38
- /**
39
- * The `User-Agent` HTTP header used by the browser.
40
- * If not provided, the function sets `User-Agent` to a reasonable default
41
- * to reduce the chance of detection of the crawler.
42
- */
43
- userAgent?: string;
44
37
  /**
45
38
  * If `true` and `executablePath` is not set,
46
39
  * Puppeteer will launch full Google Chrome browser available on the machine
@@ -65,18 +58,6 @@ export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPl
65
58
  * @default false
66
59
  */
67
60
  useIncognitoPages?: boolean;
68
- /**
69
- * This setting hides most of the known properties that identify headless Chrome and makes it nearly undetectable.
70
- * It is recommended to use it together with the `useChrome` set to `true`.
71
- * @deprecated
72
- */
73
- stealth?: boolean;
74
- /**
75
- * Using this configuration, you can disable some of the hiding tricks.
76
- * For these settings to take effect `stealth` must be set to true
77
- * @deprecated
78
- */
79
- stealthOptions?: StealthOptions;
80
61
  }
81
62
  /**
82
63
  * `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
@@ -85,25 +66,17 @@ export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPl
85
66
  export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown> {
86
67
  protected static optionsShape: {
87
68
  launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
88
- userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
89
- stealth: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
90
- stealthOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
91
69
  proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
92
70
  useChrome: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
93
71
  useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
94
72
  userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
95
73
  launchOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
74
+ userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
96
75
  };
97
- stealth?: boolean;
98
- stealthOptions: StealthOptions;
99
76
  /**
100
77
  * All `PuppeteerLauncher` parameters are passed via an launchContext object.
101
78
  */
102
79
  constructor(launchContext?: PuppeteerLaunchContext);
103
- launch(): Promise<import("packages/browser-pool/dist/puppeteer/puppeteer-proxy-per-page").Browser>;
104
- createLaunchOptions(): {
105
- args?: string[] | undefined;
106
- } & import("packages/utils/dist").Dictionary<unknown>;
107
80
  }
108
81
  /**
109
82
  * Launches headless Chrome using Puppeteer pre-configured to work within the Apify platform.
@@ -1 +1 @@
1
- {"version":3,"file":"puppeteer-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-launcher.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEzE,OAAO,EAAyB,cAAc,EAAE,MAAM,WAAW,CAAC;AAalE;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB,CAAC,eAAe,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC3G;;OAEG;IACH,aAAa,CAAC,EAAE,eAAe,CAAC,eAAe,CAAC,CAAC;IAEjD;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;OAIG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;CACnC;AAED;;;GAGG;AACH,qBAAa,iBAAkB,SAAQ,eAAe,CAAC,eAAe,EAAE,OAAO,CAAC;IAC5E,iBAA0B,YAAY;;;;;;;;;;MAMpC;IAEF,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,EAAE,cAAc,CAAC;IAE/B;;OAEG;gBACS,aAAa,GAAE,sBAA2B;IA0BvC,MAAM;IAiBZ,mBAAmB;;;CAoC/B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAsB,eAAe,CAAC,aAAa,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAI9F"}
1
+ {"version":3,"file":"puppeteer-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-launcher.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEzE;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB,CAAC,eAAe,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC3G;;OAEG;IACH,aAAa,CAAC,EAAE,eAAe,CAAC,eAAe,CAAC,CAAC;IAEjD;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED;;;GAGG;AACH,qBAAa,iBAAkB,SAAQ,eAAe,CAAC,eAAe,EAAE,OAAO,CAAC;IAC5E,iBAA0B,YAAY;;;;;;;;MAGpC;IAEF;;OAEG;gBACS,aAAa,GAAE,sBAA2B;CAezD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAsB,eAAe,CAAC,aAAa,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAI9F"}
@@ -5,17 +5,6 @@ const tslib_1 = require("tslib");
5
5
  const ow_1 = tslib_1.__importDefault(require("ow"));
6
6
  const browser_pool_1 = require("@crawlee/browser-pool");
7
7
  const browser_1 = require("@crawlee/browser");
8
- const log_1 = tslib_1.__importDefault(require("@apify/log"));
9
- const stealth_1 = require("./stealth");
10
- /**
11
- * The default user agent used by `Actor.launchPuppeteer`.
12
- * Last updated on 2020-05-22.
13
- */
14
- const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Safari/537.36';
15
- const LAUNCH_PUPPETEER_DEFAULT_VIEWPORT = {
16
- width: 1366,
17
- height: 768,
18
- };
19
8
  /**
20
9
  * `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
21
10
  * @ignore
@@ -26,75 +15,13 @@ class PuppeteerLauncher extends browser_1.BrowserLauncher {
26
15
  */
27
16
  constructor(launchContext = {}) {
28
17
  (0, ow_1.default)(launchContext, 'PuppeteerLauncher', ow_1.default.object.exactShape(PuppeteerLauncher.optionsShape));
29
- const { launcher = browser_1.BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-puppeteer-chrome'), userAgent, stealth = false, stealthOptions = {}, ...browserLauncherOptions } = launchContext;
18
+ const { launcher = browser_1.BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-puppeteer-chrome'), ...browserLauncherOptions } = launchContext;
30
19
  super({
31
20
  ...browserLauncherOptions,
32
21
  launcher,
33
22
  });
34
- Object.defineProperty(this, "stealth", {
35
- enumerable: true,
36
- configurable: true,
37
- writable: true,
38
- value: void 0
39
- });
40
- Object.defineProperty(this, "stealthOptions", {
41
- enumerable: true,
42
- configurable: true,
43
- writable: true,
44
- value: void 0
45
- });
46
- this.userAgent = userAgent;
47
- this.stealth = stealth;
48
- this.stealthOptions = {
49
- hideWebDriver: true,
50
- ...stealthOptions,
51
- };
52
23
  this.Plugin = browser_pool_1.PuppeteerPlugin;
53
24
  }
54
- async launch() {
55
- const browser = await super.launch();
56
- if (this.stealth) {
57
- log_1.default.deprecated('Puppeteer "stealth" and "stealthOptions" are deprecated.'
58
- + ' You should use fingerprints instead.'
59
- + ' Checkout the fingerprints guide: https://sdk.apify.com/docs/guides/avoid-blocking');
60
- const { hideWebDriver, ...newStealthOptions } = this.stealthOptions;
61
- (0, stealth_1.applyStealthToBrowser)(browser, newStealthOptions);
62
- }
63
- return browser;
64
- }
65
- createLaunchOptions() {
66
- const launchOptions = super.createLaunchOptions();
67
- launchOptions.args = launchOptions.args || [];
68
- try {
69
- // eslint-disable-next-line
70
- const { isAtHome } = require('apify');
71
- if (isAtHome())
72
- launchOptions.args.push('--no-sandbox');
73
- // eslint-disable-next-line no-empty
74
- }
75
- catch { }
76
- launchOptions.defaultViewport ?? (launchOptions.defaultViewport = LAUNCH_PUPPETEER_DEFAULT_VIEWPORT);
77
- // When User-Agent is not set and we're using Chromium or headless mode,
78
- // it is better to use DEFAULT_USER_AGENT to reduce chance of detection
79
- let { userAgent } = this;
80
- if (!userAgent && (!launchOptions.executablePath || launchOptions.headless)) {
81
- userAgent = DEFAULT_USER_AGENT;
82
- }
83
- if (userAgent) {
84
- launchOptions.args.push(`--user-agent=${userAgent}`);
85
- }
86
- if (this.stealthOptions?.hideWebDriver) {
87
- const idx = launchOptions.args.findIndex((arg) => arg.startsWith('--disable-blink-features='));
88
- if (idx !== -1) {
89
- const arg = launchOptions.args[idx];
90
- launchOptions.args[idx] = `${arg},AutomationControlled`;
91
- }
92
- else {
93
- launchOptions.args.push('--disable-blink-features=AutomationControlled');
94
- }
95
- }
96
- return launchOptions;
97
- }
98
25
  }
99
26
  exports.PuppeteerLauncher = PuppeteerLauncher;
100
27
  Object.defineProperty(PuppeteerLauncher, "optionsShape", {
@@ -104,9 +31,6 @@ Object.defineProperty(PuppeteerLauncher, "optionsShape", {
104
31
  value: {
105
32
  ...browser_1.BrowserLauncher.optionsShape,
106
33
  launcher: ow_1.default.optional.object,
107
- userAgent: ow_1.default.optional.string,
108
- stealth: ow_1.default.optional.boolean,
109
- stealthOptions: ow_1.default.optional.object,
110
34
  }
111
35
  });
112
36
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"puppeteer-launcher.js","sourceRoot":"","sources":["../../src/internals/puppeteer-launcher.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AAEpB,wDAAwD;AACxD,8CAAyE;AACzE,6DAA6B;AAC7B,uCAAkE;AAElE;;;GAGG;AACH,MAAM,kBAAkB,GAAG,0HAA0H,CAAC;AAEtJ,MAAM,iCAAiC,GAAG;IACtC,KAAK,EAAE,IAAI;IACX,MAAM,EAAE,GAAG;CACd,CAAC;AAsFF;;;GAGG;AACH,MAAa,iBAAkB,SAAQ,yBAAyC;IAY5E;;OAEG;IACH,YAAY,gBAAwC,EAAE;QAClD,IAAA,YAAE,EAAC,aAAa,EAAE,mBAAmB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE7F,MAAM,EACF,QAAQ,GAAG,yBAAe,CAAC,sBAAsB,CAAC,WAAW,EAAE,mCAAmC,CAAC,EACnG,SAAS,EACT,OAAO,GAAG,KAAK,EACf,cAAc,GAAG,EAAE,EACnB,GAAG,sBAAsB,EAC5B,GAAG,aAAa,CAAC;QAElB,KAAK,CAAC;YACF,GAAG,sBAAsB;YACzB,QAAQ;SACX,CAAC,CAAC;QApBP;;;;;WAAkB;QAClB;;;;;WAA+B;QAqB3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG;YAClB,aAAa,EAAE,IAAI;YACnB,GAAG,cAAc;SACpB,CAAC;QAEF,IAAI,CAAC,MAAM,GAAG,8BAAe,CAAC;IAClC,CAAC;IAEQ,KAAK,CAAC,MAAM;QACjB,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;QAErC,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,aAAG,CAAC,UAAU,CACV,0DAA0D;kBACxD,uCAAuC;kBACvC,oFAAoF,CACzF,CAAC;YACF,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,EAAE,GAAG,IAAI,CAAC,cAAe,CAAC;YAErE,IAAA,+BAAqB,EAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;SACrD;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAEQ,mBAAmB;QACxB,MAAM,aAAa,GAAG,KAAK,CAAC,mBAAmB,EAAE,CAAC;QAClD,aAAa,CAAC,IAAI,GAAG,aAAa,CAAC,IAAI,IAAI,EAAE,CAAC;QAE9C,IAAI;YACA,2BAA2B;YAC3B,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;YACtC,IAAI,QAAQ,EAAE;gBAAE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC5D,oCAAoC;SACnC;QAAC,MAAM,GAAE;QAEV,aAAa,CAAC,eAAe,KAA7B,aAAa,CAAC,eAAe,GAAK,iCAAiC,EAAC;QAEpE,wEAAwE;QACxE,uEAAuE;QACvE,IAAI,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC,aAAa,CAAC,cAAc,IAAI,aAAa,CAAC,QAAQ,CAAC,EAAE;YACzE,SAAS,GAAG,kBAAkB,CAAC;SAClC;QAED,IAAI,SAAS,EAAE;YACX,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,SAAS,EAAE,CAAC,CAAC;SACxD;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,aAAa,EAAE;YACpC,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC,CAAC;YAC/F,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE;gBACZ,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,uBAAuB,CAAC;aAC3D;iBAAM;gBACH,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;aAC5E;SACJ;QAED,OAAO,aAAa,CAAC;IACzB,CAAC;;AA7FL,8CA8FC;AA7FG;;;;WAAyC;QACrC,GAAG,yBAAe,CAAC,YAAY;QAC/B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC7B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC5B,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACrC;GAAC;AAyFN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACI,KAAK,UAAU,eAAe,CAAC,aAAsC;IACxE,MAAM,iBAAiB,GAAG,IAAI,iBAAiB,CAAC,aAAa,CAAC,CAAC;IAE/D,OAAO,iBAAiB,CAAC,MAAM,EAAE,CAAC;AACtC,CAAC;AAJD,0CAIC"}
1
+ {"version":3,"file":"puppeteer-launcher.js","sourceRoot":"","sources":["../../src/internals/puppeteer-launcher.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AAEpB,wDAAwD;AACxD,8CAAyE;AAiEzE;;;GAGG;AACH,MAAa,iBAAkB,SAAQ,yBAAyC;IAM5E;;OAEG;IACH,YAAY,gBAAwC,EAAE;QAClD,IAAA,YAAE,EAAC,aAAa,EAAE,mBAAmB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE7F,MAAM,EACF,QAAQ,GAAG,yBAAe,CAAC,sBAAsB,CAAC,WAAW,EAAE,mCAAmC,CAAC,EACnG,GAAG,sBAAsB,EAC5B,GAAG,aAAa,CAAC;QAElB,KAAK,CAAC;YACF,GAAG,sBAAsB;YACzB,QAAQ;SACX,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,GAAG,8BAAe,CAAC;IAClC,CAAC;;AAvBL,8CAwBC;AAvBG;;;;WAAyC;QACrC,GAAG,yBAAe,CAAC,YAAY;QAC/B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC/B;GAAC;AAsBN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACI,KAAK,UAAU,eAAe,CAAC,aAAsC;IACxE,MAAM,iBAAiB,GAAG,IAAI,iBAAiB,CAAC,aAAa,CAAC,CAAC;IAE/D,OAAO,iBAAiB,CAAC,MAAM,EAAE,CAAC;AACtC,CAAC;AAJD,0CAIC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/puppeteer",
3
- "version": "3.0.0-alpha.5",
3
+ "version": "3.0.0-alpha.8",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=16.0.0"
@@ -50,7 +50,7 @@
50
50
  },
51
51
  "dependencies": {
52
52
  "@apify/log": "^1.1.10",
53
- "@crawlee/browser": "^3.0.0-alpha.5"
53
+ "@crawlee/browser": "^3.0.0-alpha.8"
54
54
  },
55
55
  "peerDependencies": {
56
56
  "puppeteer": ">= 9.x <= 13.x"