apify 3.0.0-beta.9 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -12
- package/actor.d.ts +136 -138
- package/actor.d.ts.map +1 -1
- package/actor.js +162 -164
- package/actor.js.map +1 -1
- package/configuration.d.ts +168 -0
- package/configuration.d.ts.map +1 -0
- package/configuration.js +184 -0
- package/configuration.js.map +1 -0
- package/index.d.ts +3 -0
- package/index.d.ts.map +1 -1
- package/index.js +6 -0
- package/index.js.map +1 -1
- package/index.mjs +4 -0
- package/key_value_store.d.ts +17 -0
- package/key_value_store.d.ts.map +1 -0
- package/key_value_store.js +26 -0
- package/key_value_store.js.map +1 -0
- package/package.json +22 -17
- package/platform_event_manager.d.ts +3 -0
- package/platform_event_manager.d.ts.map +1 -1
- package/platform_event_manager.js +11 -4
- package/platform_event_manager.js.map +1 -1
- package/proxy_configuration.d.ts +9 -6
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +13 -7
- package/proxy_configuration.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/utils.js +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proxy_configuration.d.ts","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AAEA,OAAO,
|
|
1
|
+
{"version":3,"file":"proxy_configuration.d.ts","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,yBAAyB,IAAI,6BAA6B,EAC1D,SAAS,IAAI,aAAa,EAC7B,MAAM,eAAe,CAAC;AACvB,OAAO,EACH,kBAAkB,IAAI,sBAAsB,EAC/C,MAAM,eAAe,CAAC;AAGvB,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAQhD,MAAM,WAAW,yBAA0B,SAAQ,6BAA6B;IAC5E;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,WAAW,SAAU,SAAQ,aAAa;IAC5C;;;OAGG;IACH,MAAM,EAAE,MAAM,EAAE,CAAC;IAEjB;;;;;;;;;OASG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,qBAAa,kBAAmB,SAAQ,sBAAsB;IAWL,QAAQ,CAAC,MAAM;IAVpE,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,WAAW,CAAC,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,IAAI,CAAC,CAAS;IACtB,OAAO,CAAC,cAAc,CAAC,CAAU;IAEjC;;OAEG;gBACS,OAAO,GAAE,yBAA8B,EAAW,MAAM,gBAAkC;IA6CtG;;;;;;;OAOG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAOjC;;;;;;;;;;;;;;;;OAgBG;IACY,YAAY,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAkB5E;;;;;;;;;;;;OAYG;IACY,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAenE;;OAEG;IACH,SAAS,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM;IAsBlD;;OAEG;cACa,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC;IAuBpD;;;;OAIG;cACa,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAa7C;;OAEG;cACa,YAAY,IAAI,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,OAAO,CAAA;KAAE,GAAG,SAAS,CAAC;IAqB/H;;;OAGG;IACH,SAAS,CAAC,+BAA+B,CAAC,YAAY,EAAE,MAAM;IAI9D;;;OAGG;IACH,SAAS,CAAC,kCAAkC;CAK/C"}
|
package/proxy_configuration.js
CHANGED
|
@@ -7,6 +7,7 @@ const consts_1 = require("@apify/consts");
|
|
|
7
7
|
const core_1 = require("@crawlee/core");
|
|
8
8
|
const got_scraping_1 = require("got-scraping");
|
|
9
9
|
const actor_1 = require("./actor");
|
|
10
|
+
const configuration_1 = require("./configuration");
|
|
10
11
|
// https://docs.apify.com/proxy/datacenter-proxy#username-parameters
|
|
11
12
|
const MAX_SESSION_ID_LENGTH = 50;
|
|
12
13
|
const CHECK_ACCESS_REQUEST_TIMEOUT_MILLIS = 4000;
|
|
@@ -37,7 +38,7 @@ const COUNTRY_CODE_REGEX = /^[A-Z]{2}$/;
|
|
|
37
38
|
* const crawler = new CheerioCrawler({
|
|
38
39
|
* // ...
|
|
39
40
|
* proxyConfiguration,
|
|
40
|
-
*
|
|
41
|
+
* requestHandler({ proxyInfo }) {
|
|
41
42
|
* const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
|
|
42
43
|
* }
|
|
43
44
|
* })
|
|
@@ -49,9 +50,15 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
|
|
|
49
50
|
/**
|
|
50
51
|
* @internal
|
|
51
52
|
*/
|
|
52
|
-
constructor(options = {}, config =
|
|
53
|
+
constructor(options = {}, config = configuration_1.Configuration.getGlobalConfig()) {
|
|
53
54
|
const { proxyUrls, newUrlFunction, ...rest } = options;
|
|
54
|
-
super({ proxyUrls, newUrlFunction
|
|
55
|
+
super({ proxyUrls, newUrlFunction, ['validateRequired']: false });
|
|
56
|
+
Object.defineProperty(this, "config", {
|
|
57
|
+
enumerable: true,
|
|
58
|
+
configurable: true,
|
|
59
|
+
writable: true,
|
|
60
|
+
value: config
|
|
61
|
+
});
|
|
55
62
|
Object.defineProperty(this, "groups", {
|
|
56
63
|
enumerable: true,
|
|
57
64
|
configurable: true,
|
|
@@ -134,7 +141,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
|
|
|
134
141
|
/**
|
|
135
142
|
* This function creates a new {@link ProxyInfo} info object.
|
|
136
143
|
* It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
|
|
137
|
-
* the currently used proxy via the
|
|
144
|
+
* the currently used proxy via the requestHandler parameter `proxyInfo`.
|
|
138
145
|
* Use it if you want to work with a rich representation of a proxy URL.
|
|
139
146
|
* If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
|
|
140
147
|
* @param [sessionId]
|
|
@@ -216,8 +223,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
|
|
|
216
223
|
* Checks if Apify Token is provided in env and gets the password via API and sets it to env
|
|
217
224
|
*/
|
|
218
225
|
async _setPasswordIfToken() {
|
|
219
|
-
const
|
|
220
|
-
const token = process.env[consts_1.ENV_VARS.TOKEN] ?? storageClientOptions.token;
|
|
226
|
+
const token = this.config.get('token');
|
|
221
227
|
if (token) {
|
|
222
228
|
const { proxy } = await actor_1.Actor.apifyClient.user().get();
|
|
223
229
|
const { password } = proxy;
|
|
@@ -258,7 +264,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
|
|
|
258
264
|
* Apify Proxy can be down for a second or a minute, but this should not crash processes.
|
|
259
265
|
*/
|
|
260
266
|
async _fetchStatus() {
|
|
261
|
-
const proxyStatusUrl =
|
|
267
|
+
const proxyStatusUrl = this.config.get('proxyStatusUrl', 'http://proxy.apify.com');
|
|
262
268
|
const requestOpts = {
|
|
263
269
|
url: `${proxyStatusUrl}/?format=json`,
|
|
264
270
|
proxyUrl: await this.newUrl(),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proxy_configuration.js","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,0CAAkE;
|
|
1
|
+
{"version":3,"file":"proxy_configuration.js","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,0CAAkE;AAKlE,wCAEuB;AACvB,+CAA2C;AAC3C,mCAAgC;AAChC,mDAAgD;AAEhD,oEAAoE;AACpE,MAAM,qBAAqB,GAAG,EAAE,CAAC;AACjC,MAAM,mCAAmC,GAAG,IAAK,CAAC;AAClD,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,kBAAkB,GAAG,YAAY,CAAC;AAoGxC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAa,kBAAmB,SAAQ,yBAAsB;IAQ1D;;OAEG;IACH,YAAY,UAAqC,EAAE,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;QAClG,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QACvD,KAAK,CAAC,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,kBAA4B,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;;;;;mBAFlB;;QAV9D;;;;;WAAyB;QACzB;;;;;WAA6B;QAC7B;;;;;WAA0B;QAC1B;;;;;WAAyB;QACzB;;;;;WAAsB;QACtB;;;;;WAAiC;QAQ7B,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC1B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,gCAAuB,CAAC,CAAC;YAC5E,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,gCAAuB,CAAC,CAAC;YACtF,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;YAC3D,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;YACjE,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC/B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,MAAM,GAAG,EAAE,EACX,gBAAgB,GAAG,EAAE,EACrB,WAAW,EACX,iBAAiB,EACjB,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,GACzC,GAAG,OAAO,CAAC;QAEZ,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAC9D,MAAM,gBAAgB,GAAG,WAAW,IAAI,iBAAiB,CAAC;QAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAErC,aAAa;QACb,IAAI,CAAC,CAAC,SAAS,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,CAAC,EAAE;YAC/E,IAAI,CAAC,kCAAkC,EAAE,CAAC;SAC7C;QACD,IAAI,SAAS,IAAI,cAAc;YAAE,IAAI,CAAC,gCAAgC,EAAE,CAAC;QAEzE,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,gBAAgB,CAAC;QACpC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,QAAS,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,cAAc,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC;QAE9D,IAAI,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,EAAE;YACjE,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,uHAAuH;kBACrH,kFAAkF,CACvF,CAAC;SACL;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;SAC7B;IACL,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACM,KAAK,CAAC,YAAY,CAAC,SAA2B;QACnD,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAC9D,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC,OAAO,CAAC,gCAAuB,CAAC,CAAC,CAAC;QACpG,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAEzC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAuB,CAAC;QAE5H,OAAO;YACH,SAAS;YACT,GAAG;YACH,MAAM;YACN,WAAW;YACX,QAAQ,EAAE,QAAQ,IAAI,EAAE;YACxB,QAAQ;YACR,IAAI,EAAE,IAAK;SACd,CAAC;IACN,CAAC;IAED;;;;;;;;;;;;OAYG;IACM,KAAK,CAAC,MAAM,CAAC,SAA2B;QAC7C,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAC9D,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC,OAAO,CAAC,gCAAuB,CAAC,CAAC,CAAC;QACpG,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,OAAO,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAE,CAAC;SAC/C;QACD,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;SAC3C;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QAE1C,OAAO,UAAU,QAAQ,IAAI,QAAQ,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;IAChE,CAAC;IAED;;OAEG;IACO,YAAY,CAAC,SAAkB;QACrC,IAAI,QAAQ,CAAC;QACb,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC;QACrC,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE;YACzB,KAAK,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;SAC5C;QACD,IAAI,SAAS,EAAE;YACX,KAAK,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;SACtC;QACD,IAAI,WAAW,EAAE;YACb,KAAK,CAAC,IAAI,CAAC,WAAW,WAAW,EAAE,CAAC,CAAC;SACxC;QAED,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE3B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,QAAQ,GAAG,MAAM,CAAC;QAE1C,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAEvC,IAAI,KAAK,EAAE;YACP,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,aAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,CAAC;YACvD,MAAM,EAAE,QAAQ,EAAE,GAAG,KAAM,CAAC;YAE5B,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACf,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE;oBAC5B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,kDAAkD;0BACjE,qFAAqF,CAAC,CAAC;iBAC5F;aACJ;iBAAM;gBACH,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;aAC5B;SACJ;QAED,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,wEAAwE,iBAAQ,CAAC,cAAc,0BAA0B;kBACnI,mBAAmB,iBAAQ,CAAC,KAAK,sEAAsE,CAAC,CAAC;SAClH;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,YAAY;QACxB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QACzC,IAAI,MAAM,EAAE;YACR,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,gBAAgB,EAAE,GAAG,MAAM,CAAC;YAChE,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;YAEzC,IAAI,CAAC,SAAS;gBAAE,IAAI,CAAC,+BAA+B,CAAC,eAAe,CAAC,CAAC;SACzE;aAAM;YACH,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,iFAAiF;kBAC5F,0HAA0H,CAAC,CAAC;SACrI;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,YAAY;QACxB,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gBAAgB,EAAE,wBAAwB,CAAC,CAAC;QACnF,MAAM,WAAW,GAAG;YAChB,GAAG,EAAE,GAAG,cAAc,eAAe;YACrC,QAAQ,EAAE,MAAM,IAAI,CAAC,MAAM,EAAE;YAC7B,OAAO,EAAE,EAAE,OAAO,EAAE,mCAAmC,EAAE;YACzD,YAAY,EAAE,MAAM;SACd,CAAC;QAEX,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,yBAAyB,EAAE,OAAO,EAAE,EAAE;YACnE,IAAI;gBACA,MAAM,QAAQ,GAAG,MAAM,IAAA,0BAAW,EAA6E,WAAW,CAAC,CAAC;gBAC5H,OAAO,QAAQ,CAAC,IAAI,CAAC;aACxB;YAAC,MAAM;gBACJ,0BAA0B;aAC7B;SACJ;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;OAGG;IACO,+BAA+B,CAAC,YAAoB;QAC1D,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC;IAClC,CAAC;IAED;;;OAGG;IACO,kCAAkC;QACxC,MAAM,IAAI,KAAK,CAAC,iDAAiD;cAC3D,yFAAyF;cACzF,0GAA0G,CAAC,CAAC;IACtH,CAAC;CACJ;AApPD,gDAoPC"}
|