apify-client 2.3.1-beta.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -115,6 +115,7 @@ export interface RequestQueueClientBatchAddRequestWithRetriesOptions {
|
|
|
115
115
|
forefront?: boolean;
|
|
116
116
|
maxUnprocessedRequestsRetries?: number;
|
|
117
117
|
maxParallel?: number;
|
|
118
|
+
minDelayBetweenUnprocessedRequestsRetriesMillis?: number;
|
|
118
119
|
}
|
|
119
120
|
export interface RequestQueueClientRequestSchema {
|
|
120
121
|
id: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue.d.ts","sourceRoot":"","sources":["../../src/resource_clients/request_queue.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"request_queue.d.ts","sourceRoot":"","sources":["../../src/resource_clients/request_queue.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAezD;;GAEG;AACH,qBAAa,kBAAmB,SAAQ,cAAc;IAClD,OAAO,CAAC,SAAS,CAAC,CAAS;IAE3B,OAAO,CAAC,aAAa,CAAC,CAAS;gBAEnB,OAAO,EAAE,2BAA2B,EAAE,WAAW,GAAE,uBAA4B;IAU3F;;OAEG;IACG,GAAG,IAAI,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC;IAI9C;;OAEG;IACG,MAAM,CAAC,SAAS,EAAE,+BAA+B,GAAG,OAAO,CAAC,YAAY,CAAC;IAM/E;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAI7B;;OAEG;IACG,QAAQ,CAAC,OAAO,GAAE,iCAAsC,GAAG,OAAO,CAAC,gCAAgC,CAAC;IAkB1G;;OAEG;IACG,UAAU,CACZ,OAAO,EAAE,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,EACpD,OAAO,GAAE,mCAAwC,GAClD,OAAO,CAAC,kCAAkC,CAAC;IAuB9C;;;;;;OAMG;cACa,iBAAiB,CAC7B,QAAQ,EAAE,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,EAAE,EACvD,OAAO,GAAE,mCAAwC,GAClD,OAAO,CAAC,8CAA8C,CAAC;cAsB1C,4BAA4B,CACxC,QAAQ,EAAE,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,EAAE,EACvD,OAAO,GAAE,mDAAwD,GAClE,OAAO,CAAC,8CAA8C,CAAC;IA0D1D;;;;;;OAMG;IACG,gBAAgB,CAClB,QAAQ,EAAE,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,EAAE,EACvD,OAAO,GAAE,mDAAwD,GAClE,OAAO,CAAC,8CAA8C,CAAC;IA8C1D;;;;;;OAMG;IACG,mBAAmB,CACrB,QAAQ,EAAE,iCAAiC,EAAE,GAC9C,OAAO,CAAC,8CAA8C,CAAC;IAmB1D;;OAEG;IACG,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,kCAAkC,GAAG,SAAS,CAAC;IAkBrF;;OAEG;IACG,aAAa,CACf,OAAO,EAAE,+BAA+B,EACxC,OAAO,GAAE,mCAAwC,GAClD,OAAO,CAAC,kCAAkC,CAAC;IAuBxC,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAYjD;AAED,MAAM,WAAW,uBAAuB;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,KAAK,EAAE,iBAAiB,CAAC;CAC5B;AAED,MAAM,WAAW,iBAAiB;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,+BAA+B;IAC5C,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iCAAiC;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gCAAgC;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,IAAI,CAAC;IACtB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,KAAK,EAAE,0BAA0B,EAAE,CAAC;CACvC;AAED,MAAM,WAAW,0BAA0B;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;CAC9B;AAED,MAAM,WAAW,mCAAmC;IAChD,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,mDAAmD;IAChE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,6BAA6B,CAAC,EAAE,MAAM,CAAC;IACvC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+CAA+C,CAAC,EAAE,MAAM,CAAC;CAC5D;AAED,MAAM,WAAW,+BAA+B;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kCAAkC;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,iBAAiB,EAAE,OAAO,CAAC;CAC9B;AAED,UAAU,gBAAgB;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,iBAAiB,EAAE,OAAO,CAAC;CAC9B;AAED,UAAU,kBAAkB;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,kBAAkB,CAAC;CAC/B;AAED,MAAM,WAAW,8CAA8C;IAC3D,iBAAiB,EAAE,gBAAgB,EAAE,CAAC;IACtC,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;CAC7C;AAED,oBAAY,iCAAiC,GAAG,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC,+BAA+B,EAAE,WAAW,CAAC,CAAA;AAEhJ,oBAAY,kCAAkC,GAAG,IAAI,CAAC,0BAA0B,EAAE,YAAY,CAAC,CAAA;AAE/F,oBAAY,kBAAkB,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,GAAG,SAAS,GAAG,OAAO,CAAA"}
|
|
@@ -10,6 +10,7 @@ const utils_1 = require("../utils");
|
|
|
10
10
|
const MAX_REQUESTS_PER_BATCH_OPERATION = 25;
|
|
11
11
|
// The number of 50 parallel requests seemed optimal, if it was higher it did not seem to bring any extra value.
|
|
12
12
|
const DEFAULT_PARALLEL_BATCH_ADD_REQUESTS = 50;
|
|
13
|
+
const DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS = 500;
|
|
13
14
|
/**
|
|
14
15
|
* @hideconstructor
|
|
15
16
|
*/
|
|
@@ -120,7 +121,7 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
120
121
|
return (0, utils_1.cast)((0, utils_1.parseDateFields)((0, utils_1.pluckData)(data)));
|
|
121
122
|
}
|
|
122
123
|
async _batchAddRequestsWithRetries(requests, options = {}) {
|
|
123
|
-
const { maxUnprocessedRequestsRetries =
|
|
124
|
+
const { forefront, maxUnprocessedRequestsRetries = 0, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
|
|
124
125
|
// Keep track of the requests that remain to be processed (in parameter format)
|
|
125
126
|
let remainingRequests = requests;
|
|
126
127
|
// Keep track of the requests that have been processed (in api format)
|
|
@@ -159,9 +160,9 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
159
160
|
.map(({ method, uniqueKey, url }) => ({ method, uniqueKey, url }));
|
|
160
161
|
break;
|
|
161
162
|
}
|
|
162
|
-
//
|
|
163
|
-
|
|
164
|
-
await new Promise((resolve) => setTimeout(resolve,
|
|
163
|
+
// Exponential backoff
|
|
164
|
+
const delayMillis = Math.floor((1 + Math.random()) * (2 ** i) * minDelayBetweenUnprocessedRequestsRetriesMillis);
|
|
165
|
+
await new Promise((resolve) => setTimeout(resolve, delayMillis));
|
|
165
166
|
}
|
|
166
167
|
const result = { processedRequests, unprocessedRequests };
|
|
167
168
|
return (0, utils_1.cast)((0, utils_1.parseDateFields)(result));
|
|
@@ -174,26 +175,31 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
174
175
|
* @experimental
|
|
175
176
|
*/
|
|
176
177
|
async batchAddRequests(requests, options = {}) {
|
|
177
|
-
const { forefront, maxUnprocessedRequestsRetries =
|
|
178
|
+
const { forefront, maxUnprocessedRequestsRetries = 0, maxParallel = DEFAULT_PARALLEL_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
|
|
178
179
|
(0, ow_1.default)(requests, ow_1.default.array.ofType(ow_1.default.object.partialShape({
|
|
179
180
|
id: ow_1.default.undefined,
|
|
180
181
|
})).minLength(1));
|
|
181
182
|
(0, ow_1.default)(forefront, ow_1.default.optional.boolean);
|
|
182
183
|
(0, ow_1.default)(maxUnprocessedRequestsRetries, ow_1.default.optional.number);
|
|
183
184
|
(0, ow_1.default)(maxParallel, ow_1.default.optional.number);
|
|
184
|
-
|
|
185
|
+
(0, ow_1.default)(minDelayBetweenUnprocessedRequestsRetriesMillis, ow_1.default.optional.number);
|
|
186
|
+
const executingRequests = new Set();
|
|
185
187
|
const individualResults = [];
|
|
186
|
-
//
|
|
187
|
-
for (let i = 0; i < requests.length; i +=
|
|
188
|
-
const requestsInBatch = requests.slice(i, i +
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
188
|
+
// Keep a pool of up to `maxParallel` requests running at once
|
|
189
|
+
for (let i = 0; i < requests.length; i += MAX_REQUESTS_PER_BATCH_OPERATION) {
|
|
190
|
+
const requestsInBatch = requests.slice(i, i + MAX_REQUESTS_PER_BATCH_OPERATION);
|
|
191
|
+
const requestPromise = this._batchAddRequestsWithRetries(requestsInBatch, options);
|
|
192
|
+
executingRequests.add(requestPromise);
|
|
193
|
+
requestPromise.then((batchAddResult) => {
|
|
194
|
+
executingRequests.delete(requestPromise);
|
|
195
|
+
individualResults.push(batchAddResult);
|
|
196
|
+
});
|
|
197
|
+
if (executingRequests.size >= maxParallel) {
|
|
198
|
+
await Promise.race(executingRequests);
|
|
193
199
|
}
|
|
194
200
|
}
|
|
195
201
|
// Get results from remaining operations
|
|
196
|
-
|
|
202
|
+
await Promise.all(executingRequests);
|
|
197
203
|
// Combine individual results together
|
|
198
204
|
const result = {
|
|
199
205
|
processedRequests: [],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue.js","sourceRoot":"","sources":["../../src/resource_clients/request_queue.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,oDAAoB;AAIpB,6DAAyD;AAEzD,oCAKkB;AAElB,0FAA0F;AAC1F,MAAM,gCAAgC,GAAG,EAAE,CAAC;AAC5C,gHAAgH;AAChH,MAAM,mCAAmC,GAAG,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"request_queue.js","sourceRoot":"","sources":["../../src/resource_clients/request_queue.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,oDAAoB;AAIpB,6DAAyD;AAEzD,oCAKkB;AAElB,0FAA0F;AAC1F,MAAM,gCAAgC,GAAG,EAAE,CAAC;AAC5C,gHAAgH;AAChH,MAAM,mCAAmC,GAAG,EAAE,CAAC;AAC/C,MAAM,6DAA6D,GAAG,GAAG,CAAC;AAE1E;;GAEG;AACH,MAAa,kBAAmB,SAAQ,gCAAc;IAKlD,YAAY,OAAoC,EAAE,cAAuC,EAAE;QACvF,KAAK,CAAC;YACF,YAAY,EAAE,gBAAgB;YAC9B,GAAG,OAAO;SACb,CAAC,CAAC;QARP;;;;;WAA2B;QAE3B;;;;;WAA+B;QAQ3B,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,SAAS,CAAC;QACvC,IAAI,CAAC,aAAa,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7F,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG;QACL,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,SAA0C;QACnD,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAEzB,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,UAA6C,EAAE;QAC1D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC5B,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACxC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YACtB,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;gBACjB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC;SACL,CAAC,CAAC;QAEH,OAAO,IAAA,YAAI,EAAC,IAAA,uBAAe,EAAC,IAAA,iBAAS,EAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CACZ,OAAoD,EACpD,UAA+C,EAAE;QAEjD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAC/B,EAAE,EAAE,YAAE,CAAC,SAAS;SACnB,CAAC,CAAC,CAAC;QAEJ,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACxC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;YAC1B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;gBACjB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC;SACL,CAAC,CAAC;QAEH,OAAO,IAAA,YAAI,EAAC,IAAA,uBAAe,EAAC,IAAA,iBAAS,EAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED;;;;;;OAMG;IACO,KAAK,CAAC,iBAAiB,CAC7B,QAAuD,EACvD,UAA+C,EAAE;QAEjD,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAChD,EAAE,EAAE,YAAE,CAAC,SAAS;SACnB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,gCAAgC,CAAC,CAAC,CAAC;QAC9D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACxC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC;YAChC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;gBACjB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC;SACL,CAAC,CAAC;QAEH,OAAO,IAAA,YAAI,EAAC,IAAA,uBAAe,EAAC,IAAA,iBAAS,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IAES,KAAK,CAAC,4BAA4B,CACxC,QAAuD,EACvD,UAA+D,EAAE;QAEjE,MAAM,EACF,SAAS,EACT,6BAA6B,GAAG,CAAC,EACjC,+CAA+C,GAAG,6DAA6D,GAClH,GAAG,OAAO,CAAC;QACZ,+EAA+E;QAC/E,IAAI,iBAAiB,GAAG,QAAQ,CAAC;QACjC,sEAAsE;QACtE,MAAM,iBAAiB,GAAuB,EAAE,CAAC;QACjD,iEAAiE;QACjE,oDAAoD;QACpD,IAAI,mBAAmB,GAAyB,EAAE,CAAC;QACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,6BAA6B,EAAE,CAAC,EAAE,EAAE;YACxD,IAAI;gBACA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,EAAE;oBAC7D,SAAS;iBACZ,CAAC,CAAC;gBACH,iBAAiB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;gBACtD,mBAAmB,GAAG,QAAQ,CAAC,mBAAmB,CAAC;gBAEnD,8DAA8D;gBAC9D,sHAAsH;gBACtH,IAAI,mBAAmB,CAAC,MAAM,KAAK,CAAC,EAAE;oBAClC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;iBAClD;gBAED,mDAAmD;gBACnD,MAAM,2BAA2B,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC;gBACxF,6DAA6D;gBAC7D,iBAAiB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,2BAA2B,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;gBAEzG,2CAA2C;gBAC3C,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;oBAChC,MAAM;iBACT;aACJ;YAAC,OAAO,GAAG,EAAE;gBACV,aAAG,CAAC,SAAS,CAAC,GAAY,EAAE,6BAA6B,CAAC,CAAC;gBAC3D,0GAA0G;gBAC1G,wEAAwE;gBACxE,MAAM,2BAA2B,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC;gBACxF,mBAAmB,GAAG,QAAQ;qBACzB,MAAM,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,2BAA2B,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;qBAC3E,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;gBAEvE,MAAM;aACT;YAED,sBAAsB;YACtB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,+CAA+C,CAAC,CAAC;YACjH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;SACpE;QAED,MAAM,MAAM,GAAG,EAAE,iBAAiB,EAAE,mBAAmB,EAA2B,CAAC;QAEnF,OAAO,IAAA,YAAI,EAAC,IAAA,uBAAe,EAAC,MAAM,CAAC,CAAC,CAAC;IACzC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,gBAAgB,CAClB,QAAuD,EACvD,UAA+D,EAAE;QAEjE,MAAM,EACF,SAAS,EACT,6BAA6B,GAAG,CAAC,EACjC,WAAW,GAAG,mCAAmC,EACjD,+CAA+C,GAAG,6DAA6D,GAClH,GAAG,OAAO,CAAC;QACZ,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAChD,EAAE,EAAE,YAAE,CAAC,SAAS;SACnB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACnC,IAAA,YAAE,EAAC,6BAA6B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtD,IAAA,YAAE,EAAC,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACpC,IAAA,YAAE,EAAC,+CAA+C,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAExE,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAE,CAAC;QACpC,MAAM,iBAAiB,GAAqD,EAAE,CAAC;QAE/E,8DAA8D;QAC9D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,gCAAgC,EAAE;YACxE,MAAM,eAAe,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gCAAgC,CAAC,CAAC;YAChF,MAAM,cAAc,GAAG,IAAI,CAAC,4BAA4B,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;YACnF,iBAAiB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACtC,cAAc,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,EAAE;gBACnC,iBAAiB,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;gBACzC,iBAAiB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC3C,CAAC,CAAC,CAAC;YACH,IAAI,iBAAiB,CAAC,IAAI,IAAI,WAAW,EAAE;gBACvC,MAAM,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;aACzC;SACJ;QACD,wCAAwC;QACxC,MAAM,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAErC,sCAAsC;QACtC,MAAM,MAAM,GAAmD;YAC3D,iBAAiB,EAAE,EAAE;YACrB,mBAAmB,EAAE,EAAE;SAC1B,CAAC;QACF,iBAAiB,CAAC,OAAO,CAAC,CAAC,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,EAAE,EAAE;YACrE,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,CAAC;YACpD,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,GAAG,mBAAmB,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,mBAAmB,CACrB,QAA6C;QAE7C,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAC/B,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,YAAE,CAAC,MAAM,EAAE,CAAC,EACzC,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,YAAE,CAAC,MAAM,EAAE,CAAC,CACnD,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,gCAAgC,CAAC,CAAC,CAAC;QAE7D,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACxC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC;YAChC,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;gBACjB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC;SACL,CAAC,CAAC;QAEH,OAAO,IAAA,YAAI,EAAC,IAAA,uBAAe,EAAC,IAAA,iBAAS,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,EAAU;QACvB,IAAA,YAAE,EAAC,EAAE,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAClB,MAAM,WAAW,GAAuB;YACpC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YAChC,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE;SACzB,CAAC;QACF,IAAI;YACA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACzD,OAAO,IAAA,YAAI,EAAC,IAAA,uBAAe,EAAC,IAAA,iBAAS,EAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SAC1D;QAAC,OAAO,GAAG,EAAE;YACV,IAAA,4BAAoB,EAAC,GAAoB,CAAC,CAAC;SAC9C;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CACf,OAAwC,EACxC,UAA+C,EAAE;QAEjD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAC/B,EAAE,EAAE,YAAE,CAAC,MAAM;SAChB,CAAC,CAAC,CAAC;QAEJ,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACxC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,OAAO,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;gBACjB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC;SACL,CAAC,CAAC;QAEH,OAAO,IAAA,YAAI,EAAC,IAAA,uBAAe,EAAC,IAAA,iBAAS,EAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,EAAU;QAC1B,IAAA,YAAE,EAAC,EAAE,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAElB,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACvB,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YAChC,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;gBACjB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC;SACL,CAAC,CAAC;IACP,CAAC;CACJ;AAvUD,gDAuUC"}
|
package/package.json
CHANGED
|
@@ -1,109 +1,109 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
2
|
+
"name": "apify-client",
|
|
3
|
+
"version": "2.4.0",
|
|
4
|
+
"description": "Apify API client for JavaScript",
|
|
5
|
+
"main": "dist/index.js",
|
|
6
|
+
"module": "dist/index.mjs",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
"./package.json": "./package.json",
|
|
10
|
+
".": {
|
|
11
|
+
"import": "./dist/index.mjs",
|
|
12
|
+
"require": "./dist/index.js"
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"keywords": [
|
|
16
|
+
"apify",
|
|
17
|
+
"api",
|
|
18
|
+
"apifier",
|
|
19
|
+
"crawler",
|
|
20
|
+
"scraper"
|
|
21
|
+
],
|
|
22
|
+
"author": {
|
|
23
|
+
"name": "Apify",
|
|
24
|
+
"email": "support@apify.com",
|
|
25
|
+
"url": "https://apify.com"
|
|
26
|
+
},
|
|
27
|
+
"contributors": [
|
|
28
|
+
"Jan Curn <jan@apify.com>",
|
|
29
|
+
"Marek Trunkát <marek@apify.com>",
|
|
30
|
+
"Ondra Urban <ondra@apify.com>",
|
|
31
|
+
"Jakub Drobník <jakub.drobnik@apify.com>"
|
|
32
|
+
],
|
|
33
|
+
"license": "Apache-2.0",
|
|
34
|
+
"repository": {
|
|
35
|
+
"type": "git",
|
|
36
|
+
"url": "git+https://github.com/apifytech/apify-client-js"
|
|
37
|
+
},
|
|
38
|
+
"bugs": {
|
|
39
|
+
"url": "https://github.com/apifytech/apify-client-js/issues"
|
|
40
|
+
},
|
|
41
|
+
"homepage": "https://apify.com/docs/sdk/apify-client-js/latest",
|
|
42
|
+
"files": [
|
|
43
|
+
"dist",
|
|
44
|
+
"!dist/*.tsbuildinfo",
|
|
45
|
+
"!dist/bundle.js"
|
|
46
|
+
],
|
|
47
|
+
"scripts": {
|
|
48
|
+
"build": "run-s clean build:node build:browser",
|
|
49
|
+
"postbuild": "gen-esm-wrapper dist/index.js dist/index.mjs",
|
|
50
|
+
"prepublishOnly": "(test $CI || (echo \"Publishing is reserved to CI!\"; exit 1))",
|
|
51
|
+
"clean": "rimraf dist",
|
|
52
|
+
"test": "npm run build && jest",
|
|
53
|
+
"lint": "eslint src test --ext js,jsx,mjs,ts",
|
|
54
|
+
"lint:fix": "eslint src test --ext js,jsx,mjs,ts --fix",
|
|
55
|
+
"build:node": "tsc",
|
|
56
|
+
"build:browser": "webpack",
|
|
57
|
+
"build-docs": "npm run build-toc && node docs/build_docs.js",
|
|
58
|
+
"build-toc": "markdown-toc docs/README.hbs -i"
|
|
59
|
+
},
|
|
60
|
+
"dependencies": {
|
|
61
|
+
"@apify/consts": "^1.4.0",
|
|
62
|
+
"@apify/log": "^1.1.1",
|
|
63
|
+
"agentkeepalive": "^4.1.4",
|
|
64
|
+
"async-retry": "^1.3.1",
|
|
65
|
+
"axios": "^0.21.1",
|
|
66
|
+
"content-type": "^1.0.4",
|
|
67
|
+
"ow": "^0.27.0"
|
|
68
|
+
},
|
|
69
|
+
"devDependencies": {
|
|
70
|
+
"@apify/eslint-config-ts": "^0.1.4",
|
|
71
|
+
"@apify/tsconfig": "^0.1.0",
|
|
72
|
+
"@babel/cli": "^7.14.8",
|
|
73
|
+
"@babel/core": "^7.15.0",
|
|
74
|
+
"@babel/preset-env": "^7.15.0",
|
|
75
|
+
"@babel/register": "^7.15.3",
|
|
76
|
+
"@types/async-retry": "^1.4.3",
|
|
77
|
+
"@types/content-type": "^1.1.5",
|
|
78
|
+
"@types/express": "^4.17.13",
|
|
79
|
+
"@types/fs-extra": "^9.0.12",
|
|
80
|
+
"@types/jest": "^27.0.1",
|
|
81
|
+
"@types/node": "^16.6.1",
|
|
82
|
+
"@typescript-eslint/eslint-plugin": "^4.29.1",
|
|
83
|
+
"@typescript-eslint/parser": "^4.29.1",
|
|
84
|
+
"apify": "^2.0.2",
|
|
85
|
+
"babel-loader": "^8.2.2",
|
|
86
|
+
"body-parser": "^1.19.0",
|
|
87
|
+
"compression": "^1.7.4",
|
|
88
|
+
"eslint": "^7.32.0",
|
|
89
|
+
"express": "^4.17.1",
|
|
90
|
+
"fs-extra": "^10.0.0",
|
|
91
|
+
"gen-esm-wrapper": "^1.1.2",
|
|
92
|
+
"jest": "^27.0.6",
|
|
93
|
+
"jest-circus": "^27.0.6",
|
|
94
|
+
"jsdoc-to-markdown": "^7.0.1",
|
|
95
|
+
"markdown-toc": "^1.2.0",
|
|
96
|
+
"morgan": "^1.10.0",
|
|
97
|
+
"npm-run-all": "^4.1.5",
|
|
98
|
+
"process": "^0.11.10",
|
|
99
|
+
"puppeteer": "^10.2.0",
|
|
100
|
+
"rimraf": "^3.0.2",
|
|
101
|
+
"terser-webpack-plugin": "^5.1.4",
|
|
102
|
+
"ts-jest": "^27.0.4",
|
|
103
|
+
"ts-loader": "^9.2.5",
|
|
104
|
+
"ts-node": "^10.2.0",
|
|
105
|
+
"typescript": "^4.3.5",
|
|
106
|
+
"webpack": "^5.50.0",
|
|
107
|
+
"webpack-cli": "^4.7.2"
|
|
108
|
+
}
|
|
109
109
|
}
|