apify-client 2.3.0 → 2.4.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
2.4.0 / 2022/05/11
|
|
2
|
+
- Add exponential backoff to `batchAddRequests`
|
|
3
|
+
- Add option `minDelayBetweenUnprocessedRequestsRetriesMillis` to `batchAddRequests`
|
|
4
|
+
2.3.1 / 2022/04/28
|
|
5
|
+
===================
|
|
6
|
+
- Fix: Retries in batch requests insert endpoint
|
|
7
|
+
|
|
1
8
|
2.3.0 / 2022/04/20
|
|
2
9
|
===================
|
|
3
10
|
- Add batch delete requests method for request queue (for now experimental)
|
|
@@ -115,6 +115,7 @@ export interface RequestQueueClientBatchAddRequestWithRetriesOptions {
|
|
|
115
115
|
forefront?: boolean;
|
|
116
116
|
maxUnprocessedRequestsRetries?: number;
|
|
117
117
|
maxParallel?: number;
|
|
118
|
+
minDelayBetweenUnprocessedRequestsRetriesMillis?: number;
|
|
118
119
|
}
|
|
119
120
|
export interface RequestQueueClientRequestSchema {
|
|
120
121
|
id: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue.d.ts","sourceRoot":"","sources":["../../src/resource_clients/request_queue.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"request_queue.d.ts","sourceRoot":"","sources":["../../src/resource_clients/request_queue.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAezD;;GAEG;AACH,qBAAa,kBAAmB,SAAQ,cAAc;IAClD,OAAO,CAAC,SAAS,CAAC,CAAS;IAE3B,OAAO,CAAC,aAAa,CAAC,CAAS;gBAEnB,OAAO,EAAE,2BAA2B,EAAE,WAAW,GAAE,uBAA4B;IAU3F;;OAEG;IACG,GAAG,IAAI,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC;IAI9C;;OAEG;IACG,MAAM,CAAC,SAAS,EAAE,+BAA+B,GAAG,OAAO,CAAC,YAAY,CAAC;IAM/E;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAI7B;;OAEG;IACG,QAAQ,CAAC,OAAO,GAAE,iCAAsC,GAAG,OAAO,CAAC,gCAAgC,CAAC;IAkB1G;;OAEG;IACG,UAAU,CACZ,OAAO,EAAE,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,EACpD,OAAO,GAAE,mCAAwC,GAClD,OAAO,CAAC,kCAAkC,CAAC;IAuB9C;;;;;;OAMG;cACa,iBAAiB,CAC7B,QAAQ,EAAE,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,EAAE,EACvD,OAAO,GAAE,mCAAwC,GAClD,OAAO,CAAC,8CAA8C,CAAC;cAsB1C,4BAA4B,CACxC,QAAQ,EAAE,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,EAAE,EACvD,OAAO,GAAE,mDAAwD,GAClE,OAAO,CAAC,8CAA8C,CAAC;IA0D1D;;;;;;OAMG;IACG,gBAAgB,CAClB,QAAQ,EAAE,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,EAAE,EACvD,OAAO,GAAE,mDAAwD,GAClE,OAAO,CAAC,8CAA8C,CAAC;IA8C1D;;;;;;OAMG;IACG,mBAAmB,CACrB,QAAQ,EAAE,iCAAiC,EAAE,GAC9C,OAAO,CAAC,8CAA8C,CAAC;IAmB1D;;OAEG;IACG,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,kCAAkC,GAAG,SAAS,CAAC;IAkBrF;;OAEG;IACG,aAAa,CACf,OAAO,EAAE,+BAA+B,EACxC,OAAO,GAAE,mCAAwC,GAClD,OAAO,CAAC,kCAAkC,CAAC;IAuBxC,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAYjD;AAED,MAAM,WAAW,uBAAuB;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,KAAK,EAAE,iBAAiB,CAAC;CAC5B;AAED,MAAM,WAAW,iBAAiB;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,+BAA+B;IAC5C,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iCAAiC;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gCAAgC;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,IAAI,CAAC;IACtB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,KAAK,EAAE,0BAA0B,EAAE,CAAC;CACvC;AAED,MAAM,WAAW,0BAA0B;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;CAC9B;AAED,MAAM,WAAW,mCAAmC;IAChD,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,mDAAmD;IAChE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,6BAA6B,CAAC,EAAE,MAAM,CAAC;IACvC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+CAA+C,CAAC,EAAE,MAAM,CAAC;CAC5D;AAED,MAAM,WAAW,+BAA+B;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kCAAkC;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,iBAAiB,EAAE,OAAO,CAAC;CAC9B;AAED,UAAU,gBAAgB;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,iBAAiB,EAAE,OAAO,CAAC;CAC9B;AAED,UAAU,kBAAkB;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,kBAAkB,CAAC;CAC/B;AAED,MAAM,WAAW,8CAA8C;IAC3D,iBAAiB,EAAE,gBAAgB,EAAE,CAAC;IACtC,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;CAC7C;AAED,oBAAY,iCAAiC,GAAG,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC,+BAA+B,EAAE,WAAW,CAAC,CAAA;AAEhJ,oBAAY,kCAAkC,GAAG,IAAI,CAAC,0BAA0B,EAAE,YAAY,CAAC,CAAA;AAE/F,oBAAY,kBAAkB,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,GAAG,SAAS,GAAG,OAAO,CAAA"}
|
|
@@ -10,6 +10,7 @@ const utils_1 = require("../utils");
|
|
|
10
10
|
const MAX_REQUESTS_PER_BATCH_OPERATION = 25;
|
|
11
11
|
// The number of 50 parallel requests seemed optimal, if it was higher it did not seem to bring any extra value.
|
|
12
12
|
const DEFAULT_PARALLEL_BATCH_ADD_REQUESTS = 50;
|
|
13
|
+
const DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS = 500;
|
|
13
14
|
/**
|
|
14
15
|
* @hideconstructor
|
|
15
16
|
*/
|
|
@@ -120,7 +121,7 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
120
121
|
return (0, utils_1.cast)((0, utils_1.parseDateFields)((0, utils_1.pluckData)(data)));
|
|
121
122
|
}
|
|
122
123
|
async _batchAddRequestsWithRetries(requests, options = {}) {
|
|
123
|
-
const { maxUnprocessedRequestsRetries =
|
|
124
|
+
const { forefront, maxUnprocessedRequestsRetries = 0, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
|
|
124
125
|
// Keep track of the requests that remain to be processed (in parameter format)
|
|
125
126
|
let remainingRequests = requests;
|
|
126
127
|
// Keep track of the requests that have been processed (in api format)
|
|
@@ -128,7 +129,7 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
128
129
|
// The requests we have not been able to process in the last call
|
|
129
130
|
// ie. those we have not been able to process at all
|
|
130
131
|
let unprocessedRequests = [];
|
|
131
|
-
for (let
|
|
132
|
+
for (let i = 0; i < 1 + maxUnprocessedRequestsRetries; i++) {
|
|
132
133
|
try {
|
|
133
134
|
const response = await this._batchAddRequests(remainingRequests, {
|
|
134
135
|
forefront,
|
|
@@ -138,7 +139,7 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
138
139
|
// Consider request with unprocessed requests as rate limited.
|
|
139
140
|
// NOTE: This is important for SDK, the rate limit errors are read by AutoScalePool and used to potentially downscale.
|
|
140
141
|
if (unprocessedRequests.length !== 0) {
|
|
141
|
-
this.httpClient.stats.addRateLimitError(
|
|
142
|
+
this.httpClient.stats.addRateLimitError(i + 1);
|
|
142
143
|
}
|
|
143
144
|
// Get unique keys of all requests processed so far
|
|
144
145
|
const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
|
|
@@ -159,9 +160,9 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
159
160
|
.map(({ method, uniqueKey, url }) => ({ method, uniqueKey, url }));
|
|
160
161
|
break;
|
|
161
162
|
}
|
|
162
|
-
//
|
|
163
|
-
|
|
164
|
-
await new Promise((resolve) => setTimeout(resolve,
|
|
163
|
+
// Exponential backoff
|
|
164
|
+
const delayMillis = Math.floor((1 + Math.random()) * (2 ** i) * minDelayBetweenUnprocessedRequestsRetriesMillis);
|
|
165
|
+
await new Promise((resolve) => setTimeout(resolve, delayMillis));
|
|
165
166
|
}
|
|
166
167
|
const result = { processedRequests, unprocessedRequests };
|
|
167
168
|
return (0, utils_1.cast)((0, utils_1.parseDateFields)(result));
|
|
@@ -174,26 +175,31 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
174
175
|
* @experimental
|
|
175
176
|
*/
|
|
176
177
|
async batchAddRequests(requests, options = {}) {
|
|
177
|
-
const { forefront, maxUnprocessedRequestsRetries =
|
|
178
|
+
const { forefront, maxUnprocessedRequestsRetries = 0, maxParallel = DEFAULT_PARALLEL_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
|
|
178
179
|
(0, ow_1.default)(requests, ow_1.default.array.ofType(ow_1.default.object.partialShape({
|
|
179
180
|
id: ow_1.default.undefined,
|
|
180
181
|
})).minLength(1));
|
|
181
182
|
(0, ow_1.default)(forefront, ow_1.default.optional.boolean);
|
|
182
183
|
(0, ow_1.default)(maxUnprocessedRequestsRetries, ow_1.default.optional.number);
|
|
183
184
|
(0, ow_1.default)(maxParallel, ow_1.default.optional.number);
|
|
184
|
-
|
|
185
|
+
(0, ow_1.default)(minDelayBetweenUnprocessedRequestsRetriesMillis, ow_1.default.optional.number);
|
|
186
|
+
const executingRequests = new Set();
|
|
185
187
|
const individualResults = [];
|
|
186
|
-
//
|
|
187
|
-
for (let i = 0; i < requests.length; i +=
|
|
188
|
-
const requestsInBatch = requests.slice(i, i +
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
188
|
+
// Keep a pool of up to `maxParallel` requests running at once
|
|
189
|
+
for (let i = 0; i < requests.length; i += MAX_REQUESTS_PER_BATCH_OPERATION) {
|
|
190
|
+
const requestsInBatch = requests.slice(i, i + MAX_REQUESTS_PER_BATCH_OPERATION);
|
|
191
|
+
const requestPromise = this._batchAddRequestsWithRetries(requestsInBatch, options);
|
|
192
|
+
executingRequests.add(requestPromise);
|
|
193
|
+
requestPromise.then((batchAddResult) => {
|
|
194
|
+
executingRequests.delete(requestPromise);
|
|
195
|
+
individualResults.push(batchAddResult);
|
|
196
|
+
});
|
|
197
|
+
if (executingRequests.size >= maxParallel) {
|
|
198
|
+
await Promise.race(executingRequests);
|
|
193
199
|
}
|
|
194
200
|
}
|
|
195
201
|
// Get results from remaining operations
|
|
196
|
-
|
|
202
|
+
await Promise.all(executingRequests);
|
|
197
203
|
// Combine individual results together
|
|
198
204
|
const result = {
|
|
199
205
|
processedRequests: [],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue.js","sourceRoot":"","sources":["../../src/resource_clients/request_queue.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,oDAAoB;AAIpB,6DAAyD;AAEzD,oCAKkB;AAElB,0FAA0F;AAC1F,MAAM,gCAAgC,GAAG,EAAE,CAAC;AAC5C,gHAAgH;AAChH,MAAM,mCAAmC,GAAG,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"request_queue.js","sourceRoot":"","sources":["../../src/resource_clients/request_queue.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,oDAAoB;AAIpB,6DAAyD;AAEzD,oCAKkB;AAElB,0FAA0F;AAC1F,MAAM,gCAAgC,GAAG,EAAE,CAAC;AAC5C,gHAAgH;AAChH,MAAM,mCAAmC,GAAG,EAAE,CAAC;AAC/C,MAAM,6DAA6D,GAAG,GAAG,CAAC;AAE1E;;GAEG;AACH,MAAa,kBAAmB,SAAQ,gCAAc;IAKlD,YAAY,OAAoC,EAAE,cAAuC,EAAE;QACvF,KAAK,CAAC;YACF,YAAY,EAAE,gBAAgB;YAC9B,GAAG,OAAO;SACb,CAAC,CAAC;QARP;;;;;WAA2B;QAE3B;;;;;WAA+B;QAQ3B,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,SAAS,CAAC;QACvC,IAAI,CAAC,aAAa,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7F,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG;QACL,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,SAA0C;QACnD,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAEzB,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,UAA6C,EAAE;QAC1D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC5B,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACxC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YACtB,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;gBACjB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC;SACL,CAAC,CAAC;QAEH,OAAO,IAAA,YAAI,EAAC,IAAA,uBAAe,EAAC,IAAA,iBAAS,EAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CACZ,OAAoD,EACpD,UAA+C,EAAE;QAEjD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAC/B,EAAE,EAAE,YAAE,CAAC,SAAS;SACnB,CAAC,CAAC,CAAC;QAEJ,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACxC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;YAC1B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;gBACjB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC;SACL,CAAC,CAAC;QAEH,OAAO,IAAA,YAAI,EAAC,IAAA,uBAAe,EAAC,IAAA,iBAAS,EAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED;;;;;;OAMG;IACO,KAAK,CAAC,iBAAiB,CAC7B,QAAuD,EACvD,UAA+C,EAAE;QAEjD,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAChD,EAAE,EAAE,YAAE,CAAC,SAAS;SACnB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,gCAAgC,CAAC,CAAC,CAAC;QAC9D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACxC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC;YAChC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;gBACjB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC;SACL,CAAC,CAAC;QAEH,OAAO,IAAA,YAAI,EAAC,IAAA,uBAAe,EAAC,IAAA,iBAAS,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IAES,KAAK,CAAC,4BAA4B,CACxC,QAAuD,EACvD,UAA+D,EAAE;QAEjE,MAAM,EACF,SAAS,EACT,6BAA6B,GAAG,CAAC,EACjC,+CAA+C,GAAG,6DAA6D,GAClH,GAAG,OAAO,CAAC;QACZ,+EAA+E;QAC/E,IAAI,iBAAiB,GAAG,QAAQ,CAAC;QACjC,sEAAsE;QACtE,MAAM,iBAAiB,GAAuB,EAAE,CAAC;QACjD,iEAAiE;QACjE,oDAAoD;QACpD,IAAI,mBAAmB,GAAyB,EAAE,CAAC;QACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,6BAA6B,EAAE,CAAC,EAAE,EAAE;YACxD,IAAI;gBACA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,EAAE;oBAC7D,SAAS;iBACZ,CAAC,CAAC;gBACH,iBAAiB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;gBACtD,mBAAmB,GAAG,QAAQ,CAAC,mBAAmB,CAAC;gBAEnD,8DAA8D;gBAC9D,sHAAsH;gBACtH,IAAI,mBAAmB,CAAC,MAAM,KAAK,CAAC,EAAE;oBAClC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;iBAClD;gBAED,mDAAmD;gBACnD,MAAM,2BAA2B,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC;gBACxF,6DAA6D;gBAC7D,iBAAiB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,2BAA2B,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;gBAEzG,2CAA2C;gBAC3C,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;oBAChC,MAAM;iBACT;aACJ;YAAC,OAAO,GAAG,EAAE;gBACV,aAAG,CAAC,SAAS,CAAC,GAAY,EAAE,6BAA6B,CAAC,CAAC;gBAC3D,0GAA0G;gBAC1G,wEAAwE;gBACxE,MAAM,2BAA2B,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC;gBACxF,mBAAmB,GAAG,QAAQ;qBACzB,MAAM,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,2BAA2B,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;qBAC3E,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;gBAEvE,MAAM;aACT;YAED,sBAAsB;YACtB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,+CAA+C,CAAC,CAAC;YACjH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;SACpE;QAED,MAAM,MAAM,GAAG,EAAE,iBAAiB,EAAE,mBAAmB,EAA2B,CAAC;QAEnF,OAAO,IAAA,YAAI,EAAC,IAAA,uBAAe,EAAC,MAAM,CAAC,CAAC,CAAC;IACzC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,gBAAgB,CAClB,QAAuD,EACvD,UAA+D,EAAE;QAEjE,MAAM,EACF,SAAS,EACT,6BAA6B,GAAG,CAAC,EACjC,WAAW,GAAG,mCAAmC,EACjD,+CAA+C,GAAG,6DAA6D,GAClH,GAAG,OAAO,CAAC;QACZ,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAChD,EAAE,EAAE,YAAE,CAAC,SAAS;SACnB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACnC,IAAA,YAAE,EAAC,6BAA6B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtD,IAAA,YAAE,EAAC,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACpC,IAAA,YAAE,EAAC,+CAA+C,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAExE,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAE,CAAC;QACpC,MAAM,iBAAiB,GAAqD,EAAE,CAAC;QAE/E,8DAA8D;QAC9D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,gCAAgC,EAAE;YACxE,MAAM,eAAe,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gCAAgC,CAAC,CAAC;YAChF,MAAM,cAAc,GAAG,IAAI,CAAC,4BAA4B,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;YACnF,iBAAiB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACtC,cAAc,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,EAAE;gBACnC,iBAAiB,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;gBACzC,iBAAiB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC3C,CAAC,CAAC,CAAC;YACH,IAAI,iBAAiB,CAAC,IAAI,IAAI,WAAW,EAAE;gBACvC,MAAM,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;aACzC;SACJ;QACD,wCAAwC;QACxC,MAAM,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAErC,sCAAsC;QACtC,MAAM,MAAM,GAAmD;YAC3D,iBAAiB,EAAE,EAAE;YACrB,mBAAmB,EAAE,EAAE;SAC1B,CAAC;QACF,iBAAiB,CAAC,OAAO,CAAC,CAAC,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,EAAE,EAAE;YACrE,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,CAAC;YACpD,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,GAAG,mBAAmB,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,mBAAmB,CACrB,QAA6C;QAE7C,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAC/B,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,YAAE,CAAC,MAAM,EAAE,CAAC,EACzC,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,YAAE,CAAC,MAAM,EAAE,CAAC,CACnD,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,gCAAgC,CAAC,CAAC,CAAC;QAE7D,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACxC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC;YAChC,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;gBACjB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC;SACL,CAAC,CAAC;QAEH,OAAO,IAAA,YAAI,EAAC,IAAA,uBAAe,EAAC,IAAA,iBAAS,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,EAAU;QACvB,IAAA,YAAE,EAAC,EAAE,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAClB,MAAM,WAAW,GAAuB;YACpC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YAChC,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE;SACzB,CAAC;QACF,IAAI;YACA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACzD,OAAO,IAAA,YAAI,EAAC,IAAA,uBAAe,EAAC,IAAA,iBAAS,EAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SAC1D;QAAC,OAAO,GAAG,EAAE;YACV,IAAA,4BAAoB,EAAC,GAAoB,CAAC,CAAC;SAC9C;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CACf,OAAwC,EACxC,UAA+C,EAAE;QAEjD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAC/B,EAAE,EAAE,YAAE,CAAC,MAAM;SAChB,CAAC,CAAC,CAAC;QAEJ,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACxC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,OAAO,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;gBACjB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC;SACL,CAAC,CAAC;QAEH,OAAO,IAAA,YAAI,EAAC,IAAA,uBAAe,EAAC,IAAA,iBAAS,EAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,EAAU;QAC1B,IAAA,YAAE,EAAC,EAAE,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAElB,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACvB,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YAChC,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;gBACjB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC;SACL,CAAC,CAAC;IACP,CAAC;CACJ;AAvUD,gDAuUC"}
|
package/package.json
CHANGED
|
@@ -1,109 +1,109 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
2
|
+
"name": "apify-client",
|
|
3
|
+
"version": "2.4.0-beta.0",
|
|
4
|
+
"description": "Apify API client for JavaScript",
|
|
5
|
+
"main": "dist/index.js",
|
|
6
|
+
"module": "dist/index.mjs",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
"./package.json": "./package.json",
|
|
10
|
+
".": {
|
|
11
|
+
"import": "./dist/index.mjs",
|
|
12
|
+
"require": "./dist/index.js"
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"keywords": [
|
|
16
|
+
"apify",
|
|
17
|
+
"api",
|
|
18
|
+
"apifier",
|
|
19
|
+
"crawler",
|
|
20
|
+
"scraper"
|
|
21
|
+
],
|
|
22
|
+
"author": {
|
|
23
|
+
"name": "Apify",
|
|
24
|
+
"email": "support@apify.com",
|
|
25
|
+
"url": "https://apify.com"
|
|
26
|
+
},
|
|
27
|
+
"contributors": [
|
|
28
|
+
"Jan Curn <jan@apify.com>",
|
|
29
|
+
"Marek Trunkát <marek@apify.com>",
|
|
30
|
+
"Ondra Urban <ondra@apify.com>",
|
|
31
|
+
"Jakub Drobník <jakub.drobnik@apify.com>"
|
|
32
|
+
],
|
|
33
|
+
"license": "Apache-2.0",
|
|
34
|
+
"repository": {
|
|
35
|
+
"type": "git",
|
|
36
|
+
"url": "git+https://github.com/apifytech/apify-client-js"
|
|
37
|
+
},
|
|
38
|
+
"bugs": {
|
|
39
|
+
"url": "https://github.com/apifytech/apify-client-js/issues"
|
|
40
|
+
},
|
|
41
|
+
"homepage": "https://apify.com/docs/sdk/apify-client-js/latest",
|
|
42
|
+
"files": [
|
|
43
|
+
"dist",
|
|
44
|
+
"!dist/*.tsbuildinfo",
|
|
45
|
+
"!dist/bundle.js"
|
|
46
|
+
],
|
|
47
|
+
"scripts": {
|
|
48
|
+
"build": "run-s clean build:node build:browser",
|
|
49
|
+
"postbuild": "gen-esm-wrapper dist/index.js dist/index.mjs",
|
|
50
|
+
"prepublishOnly": "(test $CI || (echo \"Publishing is reserved to CI!\"; exit 1))",
|
|
51
|
+
"clean": "rimraf dist",
|
|
52
|
+
"test": "npm run build && jest",
|
|
53
|
+
"lint": "eslint src test --ext js,jsx,mjs,ts",
|
|
54
|
+
"lint:fix": "eslint src test --ext js,jsx,mjs,ts --fix",
|
|
55
|
+
"build:node": "tsc",
|
|
56
|
+
"build:browser": "webpack",
|
|
57
|
+
"build-docs": "npm run build-toc && node docs/build_docs.js",
|
|
58
|
+
"build-toc": "markdown-toc docs/README.hbs -i"
|
|
59
|
+
},
|
|
60
|
+
"dependencies": {
|
|
61
|
+
"@apify/consts": "^1.4.0",
|
|
62
|
+
"@apify/log": "^1.1.1",
|
|
63
|
+
"agentkeepalive": "^4.1.4",
|
|
64
|
+
"async-retry": "^1.3.1",
|
|
65
|
+
"axios": "^0.21.1",
|
|
66
|
+
"content-type": "^1.0.4",
|
|
67
|
+
"ow": "^0.27.0"
|
|
68
|
+
},
|
|
69
|
+
"devDependencies": {
|
|
70
|
+
"@apify/eslint-config-ts": "^0.1.4",
|
|
71
|
+
"@apify/tsconfig": "^0.1.0",
|
|
72
|
+
"@babel/cli": "^7.14.8",
|
|
73
|
+
"@babel/core": "^7.15.0",
|
|
74
|
+
"@babel/preset-env": "^7.15.0",
|
|
75
|
+
"@babel/register": "^7.15.3",
|
|
76
|
+
"@types/async-retry": "^1.4.3",
|
|
77
|
+
"@types/content-type": "^1.1.5",
|
|
78
|
+
"@types/express": "^4.17.13",
|
|
79
|
+
"@types/fs-extra": "^9.0.12",
|
|
80
|
+
"@types/jest": "^27.0.1",
|
|
81
|
+
"@types/node": "^16.6.1",
|
|
82
|
+
"@typescript-eslint/eslint-plugin": "^4.29.1",
|
|
83
|
+
"@typescript-eslint/parser": "^4.29.1",
|
|
84
|
+
"apify": "^2.0.2",
|
|
85
|
+
"babel-loader": "^8.2.2",
|
|
86
|
+
"body-parser": "^1.19.0",
|
|
87
|
+
"compression": "^1.7.4",
|
|
88
|
+
"eslint": "^7.32.0",
|
|
89
|
+
"express": "^4.17.1",
|
|
90
|
+
"fs-extra": "^10.0.0",
|
|
91
|
+
"gen-esm-wrapper": "^1.1.2",
|
|
92
|
+
"jest": "^27.0.6",
|
|
93
|
+
"jest-circus": "^27.0.6",
|
|
94
|
+
"jsdoc-to-markdown": "^7.0.1",
|
|
95
|
+
"markdown-toc": "^1.2.0",
|
|
96
|
+
"morgan": "^1.10.0",
|
|
97
|
+
"npm-run-all": "^4.1.5",
|
|
98
|
+
"process": "^0.11.10",
|
|
99
|
+
"puppeteer": "^10.2.0",
|
|
100
|
+
"rimraf": "^3.0.2",
|
|
101
|
+
"terser-webpack-plugin": "^5.1.4",
|
|
102
|
+
"ts-jest": "^27.0.4",
|
|
103
|
+
"ts-loader": "^9.2.5",
|
|
104
|
+
"ts-node": "^10.2.0",
|
|
105
|
+
"typescript": "^4.3.5",
|
|
106
|
+
"webpack": "^5.50.0",
|
|
107
|
+
"webpack-cli": "^4.7.2"
|
|
108
|
+
}
|
|
109
109
|
}
|