apify-client 2.23.5-beta.4 → 2.23.5-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apify_api_error.js +37 -84
- package/dist/apify_client.js +6 -36
- package/dist/base/api_client.js +10 -56
- package/dist/bundle.js +1 -1
- package/dist/bundle.js.map +1 -1
- package/dist/http_client.js +15 -77
- package/dist/interceptors.js +5 -17
- package/dist/resource_clients/actor.js +3 -3
- package/dist/resource_clients/actor_collection.d.ts +2 -0
- package/dist/resource_clients/dataset.js +3 -5
- package/dist/resource_clients/dataset_collection.js +1 -1
- package/dist/resource_clients/key_value_store.js +2 -2
- package/dist/resource_clients/key_value_store_collection.js +1 -1
- package/dist/resource_clients/log.js +7 -42
- package/dist/resource_clients/request_queue.js +14 -36
- package/dist/resource_clients/run.js +5 -7
- package/dist/statistics.js +13 -30
- package/dist/utils.js +6 -31
- package/package.json +3 -3
package/dist/http_client.js
CHANGED
|
@@ -11,79 +11,19 @@ const utils_1 = require("./utils");
|
|
|
11
11
|
const { version } = (0, utils_1.getVersionData)();
|
|
12
12
|
const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429;
|
|
13
13
|
class HttpClient {
|
|
14
|
+
stats;
|
|
15
|
+
maxRetries;
|
|
16
|
+
minDelayBetweenRetriesMillis;
|
|
17
|
+
userProvidedRequestInterceptors;
|
|
18
|
+
logger;
|
|
19
|
+
timeoutMillis;
|
|
20
|
+
httpAgent;
|
|
21
|
+
httpsAgent;
|
|
22
|
+
axios;
|
|
23
|
+
workflowKey;
|
|
24
|
+
nodeInitPromise;
|
|
25
|
+
userAgentSuffix;
|
|
14
26
|
constructor(options) {
|
|
15
|
-
Object.defineProperty(this, "stats", {
|
|
16
|
-
enumerable: true,
|
|
17
|
-
configurable: true,
|
|
18
|
-
writable: true,
|
|
19
|
-
value: void 0
|
|
20
|
-
});
|
|
21
|
-
Object.defineProperty(this, "maxRetries", {
|
|
22
|
-
enumerable: true,
|
|
23
|
-
configurable: true,
|
|
24
|
-
writable: true,
|
|
25
|
-
value: void 0
|
|
26
|
-
});
|
|
27
|
-
Object.defineProperty(this, "minDelayBetweenRetriesMillis", {
|
|
28
|
-
enumerable: true,
|
|
29
|
-
configurable: true,
|
|
30
|
-
writable: true,
|
|
31
|
-
value: void 0
|
|
32
|
-
});
|
|
33
|
-
Object.defineProperty(this, "userProvidedRequestInterceptors", {
|
|
34
|
-
enumerable: true,
|
|
35
|
-
configurable: true,
|
|
36
|
-
writable: true,
|
|
37
|
-
value: void 0
|
|
38
|
-
});
|
|
39
|
-
Object.defineProperty(this, "logger", {
|
|
40
|
-
enumerable: true,
|
|
41
|
-
configurable: true,
|
|
42
|
-
writable: true,
|
|
43
|
-
value: void 0
|
|
44
|
-
});
|
|
45
|
-
Object.defineProperty(this, "timeoutMillis", {
|
|
46
|
-
enumerable: true,
|
|
47
|
-
configurable: true,
|
|
48
|
-
writable: true,
|
|
49
|
-
value: void 0
|
|
50
|
-
});
|
|
51
|
-
Object.defineProperty(this, "httpAgent", {
|
|
52
|
-
enumerable: true,
|
|
53
|
-
configurable: true,
|
|
54
|
-
writable: true,
|
|
55
|
-
value: void 0
|
|
56
|
-
});
|
|
57
|
-
Object.defineProperty(this, "httpsAgent", {
|
|
58
|
-
enumerable: true,
|
|
59
|
-
configurable: true,
|
|
60
|
-
writable: true,
|
|
61
|
-
value: void 0
|
|
62
|
-
});
|
|
63
|
-
Object.defineProperty(this, "axios", {
|
|
64
|
-
enumerable: true,
|
|
65
|
-
configurable: true,
|
|
66
|
-
writable: true,
|
|
67
|
-
value: void 0
|
|
68
|
-
});
|
|
69
|
-
Object.defineProperty(this, "workflowKey", {
|
|
70
|
-
enumerable: true,
|
|
71
|
-
configurable: true,
|
|
72
|
-
writable: true,
|
|
73
|
-
value: void 0
|
|
74
|
-
});
|
|
75
|
-
Object.defineProperty(this, "nodeInitPromise", {
|
|
76
|
-
enumerable: true,
|
|
77
|
-
configurable: true,
|
|
78
|
-
writable: true,
|
|
79
|
-
value: void 0
|
|
80
|
-
});
|
|
81
|
-
Object.defineProperty(this, "userAgentSuffix", {
|
|
82
|
-
enumerable: true,
|
|
83
|
-
configurable: true,
|
|
84
|
-
writable: true,
|
|
85
|
-
value: void 0
|
|
86
|
-
});
|
|
87
27
|
const { token } = options;
|
|
88
28
|
this.stats = options.apifyClientStats;
|
|
89
29
|
this.maxRetries = options.maxRetries;
|
|
@@ -137,10 +77,9 @@ class HttpClient {
|
|
|
137
77
|
interceptors_1.responseInterceptors.forEach((i) => this.axios.interceptors.response.use(i));
|
|
138
78
|
}
|
|
139
79
|
async ensureNodeInit() {
|
|
140
|
-
var _a;
|
|
141
80
|
if (!(0, utils_1.isNode)())
|
|
142
81
|
return;
|
|
143
|
-
|
|
82
|
+
this.nodeInitPromise ??= this.initNode();
|
|
144
83
|
return this.nodeInitPromise;
|
|
145
84
|
}
|
|
146
85
|
async initNode() {
|
|
@@ -160,7 +99,7 @@ class HttpClient {
|
|
|
160
99
|
timeout: this.timeoutMillis,
|
|
161
100
|
// Keep alive timeout for free sockets (15 seconds)
|
|
162
101
|
// Node.js will close unused sockets after this period
|
|
163
|
-
keepAliveMsecs:
|
|
102
|
+
keepAliveMsecs: 15_000,
|
|
164
103
|
// Maximum number of sockets per host
|
|
165
104
|
maxSockets: 256,
|
|
166
105
|
maxFreeSockets: 256,
|
|
@@ -210,7 +149,6 @@ class HttpClient {
|
|
|
210
149
|
*/
|
|
211
150
|
_createRequestHandler(config) {
|
|
212
151
|
const makeRequest = async (stopTrying, attempt) => {
|
|
213
|
-
var _a;
|
|
214
152
|
this.stats.requests++;
|
|
215
153
|
let response;
|
|
216
154
|
const requestIsStream = (0, utils_1.isStream)(config.data);
|
|
@@ -222,7 +160,7 @@ class HttpClient {
|
|
|
222
160
|
config = { ...config, maxRedirects: 0 };
|
|
223
161
|
}
|
|
224
162
|
// Increase timeout with each attempt. Max timeout is bounded by the client timeout.
|
|
225
|
-
config.timeout = Math.min(this.timeoutMillis, (
|
|
163
|
+
config.timeout = Math.min(this.timeoutMillis, (config.timeout ?? this.timeoutMillis) * 2 ** (attempt - 1));
|
|
226
164
|
response = await this.axios.request(config);
|
|
227
165
|
if (this._isStatusOk(response.status))
|
|
228
166
|
return response;
|
package/dist/interceptors.js
CHANGED
|
@@ -14,20 +14,10 @@ const utils_1 = require("./utils");
|
|
|
14
14
|
* The properties mimic AxiosError for easier integration in HttpClient error handling.
|
|
15
15
|
*/
|
|
16
16
|
class InvalidResponseBodyError extends Error {
|
|
17
|
+
code;
|
|
18
|
+
response;
|
|
17
19
|
constructor(response, cause) {
|
|
18
20
|
super(`Response body could not be parsed.\nCause:${cause.message}`);
|
|
19
|
-
Object.defineProperty(this, "code", {
|
|
20
|
-
enumerable: true,
|
|
21
|
-
configurable: true,
|
|
22
|
-
writable: true,
|
|
23
|
-
value: void 0
|
|
24
|
-
});
|
|
25
|
-
Object.defineProperty(this, "response", {
|
|
26
|
-
enumerable: true,
|
|
27
|
-
configurable: true,
|
|
28
|
-
writable: true,
|
|
29
|
-
value: void 0
|
|
30
|
-
});
|
|
31
21
|
this.name = this.constructor.name;
|
|
32
22
|
this.code = 'invalid-response-body';
|
|
33
23
|
this.response = response;
|
|
@@ -36,7 +26,6 @@ class InvalidResponseBodyError extends Error {
|
|
|
36
26
|
}
|
|
37
27
|
exports.InvalidResponseBodyError = InvalidResponseBodyError;
|
|
38
28
|
function serializeRequest(config) {
|
|
39
|
-
var _a, _b;
|
|
40
29
|
const [defaultTransform] = axios_1.default.defaults.transformRequest;
|
|
41
30
|
// The function not only serializes data, but it also adds correct headers.
|
|
42
31
|
const data = defaultTransform(config.data, config.headers);
|
|
@@ -46,7 +35,7 @@ function serializeRequest(config) {
|
|
|
46
35
|
// it's a small price to pay. The axios default transform does a lot
|
|
47
36
|
// of body type checks and we would have to copy all of them to the resource clients.
|
|
48
37
|
if (config.stringifyFunctions) {
|
|
49
|
-
const contentTypeHeader =
|
|
38
|
+
const contentTypeHeader = config.headers?.['Content-Type'] || config.headers?.['content-type'];
|
|
50
39
|
try {
|
|
51
40
|
const { type } = content_type_1.default.parse(contentTypeHeader);
|
|
52
41
|
if (type === 'application/json' && typeof config.data === 'object') {
|
|
@@ -81,12 +70,11 @@ function stringifyWithFunctions(obj) {
|
|
|
81
70
|
});
|
|
82
71
|
}
|
|
83
72
|
async function maybeGzipRequest(config) {
|
|
84
|
-
|
|
85
|
-
if ((_a = config.headers) === null || _a === void 0 ? void 0 : _a['content-encoding'])
|
|
73
|
+
if (config.headers?.['content-encoding'])
|
|
86
74
|
return config;
|
|
87
75
|
const maybeZippedData = await (0, utils_1.maybeGzipValue)(config.data);
|
|
88
76
|
if (maybeZippedData) {
|
|
89
|
-
|
|
77
|
+
config.headers ??= {};
|
|
90
78
|
config.headers['content-encoding'] = 'gzip';
|
|
91
79
|
config.data = maybeZippedData;
|
|
92
80
|
}
|
|
@@ -212,13 +212,13 @@ class ActorClient extends resource_client_1.ResourceClient {
|
|
|
212
212
|
// Creating a new instance of RunClient here would only allow
|
|
213
213
|
// setting it up as a nested route under actor API.
|
|
214
214
|
const newRunClient = this.apifyClient.run(id);
|
|
215
|
-
const streamedLog = await newRunClient.getStreamedLog({ toLog: options
|
|
216
|
-
streamedLog
|
|
215
|
+
const streamedLog = await newRunClient.getStreamedLog({ toLog: options?.log });
|
|
216
|
+
streamedLog?.start();
|
|
217
217
|
return this.apifyClient
|
|
218
218
|
.run(id)
|
|
219
219
|
.waitForFinish({ waitSecs })
|
|
220
220
|
.finally(async () => {
|
|
221
|
-
await
|
|
221
|
+
await streamedLog?.stop();
|
|
222
222
|
});
|
|
223
223
|
}
|
|
224
224
|
/**
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { ACTOR_PERMISSION_LEVEL } from '@apify/consts';
|
|
1
2
|
import type { ApiClientSubResourceOptions } from '../base/api_client';
|
|
2
3
|
import { ResourceCollectionClient } from '../base/resource_collection_client';
|
|
3
4
|
import type { PaginatedIterator, PaginatedList, PaginationOptions } from '../utils';
|
|
@@ -95,4 +96,5 @@ export interface ActorCollectionCreateOptions {
|
|
|
95
96
|
actorStandby?: ActorStandby & {
|
|
96
97
|
isEnabled: boolean;
|
|
97
98
|
};
|
|
99
|
+
actorPermissionLevel?: ACTOR_PERMISSION_LEVEL;
|
|
98
100
|
}
|
|
@@ -134,14 +134,13 @@ class DatasetClient extends resource_client_1.ResourceClient {
|
|
|
134
134
|
signature: ow_1.default.optional.string,
|
|
135
135
|
}));
|
|
136
136
|
const fetchItems = async (datasetListOptions = {}) => {
|
|
137
|
-
var _a;
|
|
138
137
|
const response = await this.httpClient.call({
|
|
139
138
|
url: this._url('items'),
|
|
140
139
|
method: 'GET',
|
|
141
140
|
params: this._params(datasetListOptions),
|
|
142
141
|
timeout: resource_client_1.DEFAULT_TIMEOUT_MILLIS,
|
|
143
142
|
});
|
|
144
|
-
return this._createPaginationList(response,
|
|
143
|
+
return this._createPaginationList(response, datasetListOptions.desc ?? false);
|
|
145
144
|
};
|
|
146
145
|
return this._listPaginatedFromCallback(fetchItems, options);
|
|
147
146
|
}
|
|
@@ -339,7 +338,7 @@ class DatasetClient extends resource_client_1.ResourceClient {
|
|
|
339
338
|
const dataset = await this.get();
|
|
340
339
|
const { expiresInSecs, ...queryOptions } = options;
|
|
341
340
|
let createdItemsPublicUrl = new URL(this._publicUrl('items'));
|
|
342
|
-
if (dataset
|
|
341
|
+
if (dataset?.urlSigningSecretKey) {
|
|
343
342
|
const signature = await (0, utilities_1.createStorageContentSignatureAsync)({
|
|
344
343
|
resourceId: dataset.id,
|
|
345
344
|
urlSigningSecretKey: dataset.urlSigningSecretKey,
|
|
@@ -351,7 +350,6 @@ class DatasetClient extends resource_client_1.ResourceClient {
|
|
|
351
350
|
return createdItemsPublicUrl.toString();
|
|
352
351
|
}
|
|
353
352
|
_createPaginationList(response, userProvidedDesc) {
|
|
354
|
-
var _a;
|
|
355
353
|
return {
|
|
356
354
|
items: response.data,
|
|
357
355
|
total: Number(response.headers['x-apify-pagination-total']),
|
|
@@ -359,7 +357,7 @@ class DatasetClient extends resource_client_1.ResourceClient {
|
|
|
359
357
|
count: response.data.length, // because x-apify-pagination-count returns invalid values when hidden/empty items are skipped
|
|
360
358
|
limit: Number(response.headers['x-apify-pagination-limit']), // API returns 999999999999 when no limit is used
|
|
361
359
|
// TODO: Replace this once https://github.com/apify/apify-core/issues/3503 is solved
|
|
362
|
-
desc: JSON.parse(
|
|
360
|
+
desc: JSON.parse(response.headers['x-apify-pagination-desc'] ?? userProvidedDesc),
|
|
363
361
|
};
|
|
364
362
|
}
|
|
365
363
|
}
|
|
@@ -75,7 +75,7 @@ class DatasetCollectionClient extends resource_collection_client_1.ResourceColle
|
|
|
75
75
|
*/
|
|
76
76
|
async getOrCreate(name, options) {
|
|
77
77
|
(0, ow_1.default)(name, ow_1.default.optional.string);
|
|
78
|
-
(0, ow_1.default)(options
|
|
78
|
+
(0, ow_1.default)(options?.schema, ow_1.default.optional.object); // TODO: Add schema validatioon
|
|
79
79
|
return this._getOrCreate(name, options);
|
|
80
80
|
}
|
|
81
81
|
}
|
|
@@ -174,7 +174,7 @@ class KeyValueStoreClient extends resource_client_1.ResourceClient {
|
|
|
174
174
|
(0, ow_1.default)(key, ow_1.default.string.nonEmpty);
|
|
175
175
|
const store = await this.get();
|
|
176
176
|
const recordPublicUrl = new URL(this._publicUrl(`records/${key}`));
|
|
177
|
-
if (store
|
|
177
|
+
if (store?.urlSigningSecretKey) {
|
|
178
178
|
const signature = await (0, utilities_1.createHmacSignatureAsync)(store.urlSigningSecretKey, key);
|
|
179
179
|
recordPublicUrl.searchParams.append('signature', signature);
|
|
180
180
|
}
|
|
@@ -213,7 +213,7 @@ class KeyValueStoreClient extends resource_client_1.ResourceClient {
|
|
|
213
213
|
const store = await this.get();
|
|
214
214
|
const { expiresInSecs, ...queryOptions } = options;
|
|
215
215
|
let createdPublicKeysUrl = new URL(this._publicUrl('keys'));
|
|
216
|
-
if (store
|
|
216
|
+
if (store?.urlSigningSecretKey) {
|
|
217
217
|
const signature = await (0, utilities_1.createStorageContentSignatureAsync)({
|
|
218
218
|
resourceId: store.id,
|
|
219
219
|
urlSigningSecretKey: store.urlSigningSecretKey,
|
|
@@ -75,7 +75,7 @@ class KeyValueStoreCollectionClient extends resource_collection_client_1.Resourc
|
|
|
75
75
|
*/
|
|
76
76
|
async getOrCreate(name, options) {
|
|
77
77
|
(0, ow_1.default)(name, ow_1.default.optional.string);
|
|
78
|
-
(0, ow_1.default)(options
|
|
78
|
+
(0, ow_1.default)(options?.schema, ow_1.default.optional.object); // TODO: Add schema validatioon
|
|
79
79
|
return this._getOrCreate(name, options);
|
|
80
80
|
}
|
|
81
81
|
}
|
|
@@ -123,49 +123,14 @@ exports.LoggerActorRedirect = LoggerActorRedirect;
|
|
|
123
123
|
* Helper class for redirecting streamed Actor logs to another log.
|
|
124
124
|
*/
|
|
125
125
|
class StreamedLog {
|
|
126
|
+
destinationLog;
|
|
127
|
+
streamBuffer = [];
|
|
128
|
+
splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g;
|
|
129
|
+
relevancyTimeLimit;
|
|
130
|
+
logClient;
|
|
131
|
+
streamingTask = null;
|
|
132
|
+
stopLogging = false;
|
|
126
133
|
constructor(options) {
|
|
127
|
-
Object.defineProperty(this, "destinationLog", {
|
|
128
|
-
enumerable: true,
|
|
129
|
-
configurable: true,
|
|
130
|
-
writable: true,
|
|
131
|
-
value: void 0
|
|
132
|
-
});
|
|
133
|
-
Object.defineProperty(this, "streamBuffer", {
|
|
134
|
-
enumerable: true,
|
|
135
|
-
configurable: true,
|
|
136
|
-
writable: true,
|
|
137
|
-
value: []
|
|
138
|
-
});
|
|
139
|
-
Object.defineProperty(this, "splitMarker", {
|
|
140
|
-
enumerable: true,
|
|
141
|
-
configurable: true,
|
|
142
|
-
writable: true,
|
|
143
|
-
value: /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g
|
|
144
|
-
});
|
|
145
|
-
Object.defineProperty(this, "relevancyTimeLimit", {
|
|
146
|
-
enumerable: true,
|
|
147
|
-
configurable: true,
|
|
148
|
-
writable: true,
|
|
149
|
-
value: void 0
|
|
150
|
-
});
|
|
151
|
-
Object.defineProperty(this, "logClient", {
|
|
152
|
-
enumerable: true,
|
|
153
|
-
configurable: true,
|
|
154
|
-
writable: true,
|
|
155
|
-
value: void 0
|
|
156
|
-
});
|
|
157
|
-
Object.defineProperty(this, "streamingTask", {
|
|
158
|
-
enumerable: true,
|
|
159
|
-
configurable: true,
|
|
160
|
-
writable: true,
|
|
161
|
-
value: null
|
|
162
|
-
});
|
|
163
|
-
Object.defineProperty(this, "stopLogging", {
|
|
164
|
-
enumerable: true,
|
|
165
|
-
configurable: true,
|
|
166
|
-
writable: true,
|
|
167
|
-
value: false
|
|
168
|
-
});
|
|
169
134
|
const { toLog, logClient, fromStart = true } = options;
|
|
170
135
|
this.destinationLog = toLog;
|
|
171
136
|
this.logClient = logClient;
|
|
@@ -43,6 +43,8 @@ const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
|
|
|
43
43
|
* @see https://docs.apify.com/platform/storage/request-queue
|
|
44
44
|
*/
|
|
45
45
|
class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
46
|
+
clientKey;
|
|
47
|
+
timeoutMillis;
|
|
46
48
|
/**
|
|
47
49
|
* @hidden
|
|
48
50
|
*/
|
|
@@ -51,18 +53,6 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
51
53
|
resourcePath: 'request-queues',
|
|
52
54
|
...options,
|
|
53
55
|
});
|
|
54
|
-
Object.defineProperty(this, "clientKey", {
|
|
55
|
-
enumerable: true,
|
|
56
|
-
configurable: true,
|
|
57
|
-
writable: true,
|
|
58
|
-
value: void 0
|
|
59
|
-
});
|
|
60
|
-
Object.defineProperty(this, "timeoutMillis", {
|
|
61
|
-
enumerable: true,
|
|
62
|
-
configurable: true,
|
|
63
|
-
writable: true,
|
|
64
|
-
value: void 0
|
|
65
|
-
});
|
|
66
56
|
this.clientKey = userOptions.clientKey;
|
|
67
57
|
this.timeoutMillis = userOptions.timeoutSecs ? userOptions.timeoutSecs * 1e3 : undefined;
|
|
68
58
|
}
|
|
@@ -105,14 +95,13 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
105
95
|
* @see https://docs.apify.com/api/v2/request-queue-head-get
|
|
106
96
|
*/
|
|
107
97
|
async listHead(options = {}) {
|
|
108
|
-
var _a;
|
|
109
98
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
110
99
|
limit: ow_1.default.optional.number.not.negative,
|
|
111
100
|
}));
|
|
112
101
|
const response = await this.httpClient.call({
|
|
113
102
|
url: this._url('head'),
|
|
114
103
|
method: 'GET',
|
|
115
|
-
timeout: Math.min(resource_client_1.SMALL_TIMEOUT_MILLIS,
|
|
104
|
+
timeout: Math.min(resource_client_1.SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
|
|
116
105
|
params: this._params({
|
|
117
106
|
limit: options.limit,
|
|
118
107
|
clientKey: this.clientKey,
|
|
@@ -153,7 +142,6 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
153
142
|
* ```
|
|
154
143
|
*/
|
|
155
144
|
async listAndLockHead(options) {
|
|
156
|
-
var _a;
|
|
157
145
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
158
146
|
lockSecs: ow_1.default.number,
|
|
159
147
|
limit: ow_1.default.optional.number.not.negative,
|
|
@@ -161,7 +149,7 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
161
149
|
const response = await this.httpClient.call({
|
|
162
150
|
url: this._url('head/lock'),
|
|
163
151
|
method: 'POST',
|
|
164
|
-
timeout: Math.min(resource_client_1.MEDIUM_TIMEOUT_MILLIS,
|
|
152
|
+
timeout: Math.min(resource_client_1.MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
|
|
165
153
|
params: this._params({
|
|
166
154
|
limit: options.limit,
|
|
167
155
|
lockSecs: options.lockSecs,
|
|
@@ -209,7 +197,6 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
209
197
|
* ```
|
|
210
198
|
*/
|
|
211
199
|
async addRequest(request, options = {}) {
|
|
212
|
-
var _a;
|
|
213
200
|
(0, ow_1.default)(request, ow_1.default.object.partialShape({
|
|
214
201
|
id: ow_1.default.undefined,
|
|
215
202
|
}));
|
|
@@ -219,7 +206,7 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
219
206
|
const response = await this.httpClient.call({
|
|
220
207
|
url: this._url('requests'),
|
|
221
208
|
method: 'POST',
|
|
222
|
-
timeout: Math.min(resource_client_1.SMALL_TIMEOUT_MILLIS,
|
|
209
|
+
timeout: Math.min(resource_client_1.SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
|
|
223
210
|
data: request,
|
|
224
211
|
params: this._params({
|
|
225
212
|
forefront: options.forefront,
|
|
@@ -234,7 +221,6 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
234
221
|
* @private
|
|
235
222
|
*/
|
|
236
223
|
async _batchAddRequests(requests, options = {}) {
|
|
237
|
-
var _a;
|
|
238
224
|
(0, ow_1.default)(requests, ow_1.default.array
|
|
239
225
|
.ofType(ow_1.default.object.partialShape({
|
|
240
226
|
id: ow_1.default.undefined,
|
|
@@ -247,7 +233,7 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
247
233
|
const { data } = await this.httpClient.call({
|
|
248
234
|
url: this._url('requests/batch'),
|
|
249
235
|
method: 'POST',
|
|
250
|
-
timeout: Math.min(resource_client_1.MEDIUM_TIMEOUT_MILLIS,
|
|
236
|
+
timeout: Math.min(resource_client_1.MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
|
|
251
237
|
data: requests,
|
|
252
238
|
params: this._params({
|
|
253
239
|
forefront: options.forefront,
|
|
@@ -395,7 +381,6 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
395
381
|
* @see https://docs.apify.com/api/v2/request-queue-requests-batch-delete
|
|
396
382
|
*/
|
|
397
383
|
async batchDeleteRequests(requests) {
|
|
398
|
-
var _a;
|
|
399
384
|
(0, ow_1.default)(requests, ow_1.default.array
|
|
400
385
|
.ofType(ow_1.default.any(ow_1.default.object.partialShape({ id: ow_1.default.string }), ow_1.default.object.partialShape({ uniqueKey: ow_1.default.string })))
|
|
401
386
|
.minLength(1)
|
|
@@ -403,7 +388,7 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
403
388
|
const { data } = await this.httpClient.call({
|
|
404
389
|
url: this._url('requests/batch'),
|
|
405
390
|
method: 'DELETE',
|
|
406
|
-
timeout: Math.min(resource_client_1.SMALL_TIMEOUT_MILLIS,
|
|
391
|
+
timeout: Math.min(resource_client_1.SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
|
|
407
392
|
data: requests,
|
|
408
393
|
params: this._params({
|
|
409
394
|
clientKey: this.clientKey,
|
|
@@ -419,12 +404,11 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
419
404
|
* @see https://docs.apify.com/api/v2/request-queue-request-get
|
|
420
405
|
*/
|
|
421
406
|
async getRequest(id) {
|
|
422
|
-
var _a;
|
|
423
407
|
(0, ow_1.default)(id, ow_1.default.string);
|
|
424
408
|
const requestOpts = {
|
|
425
409
|
url: this._url(`requests/${id}`),
|
|
426
410
|
method: 'GET',
|
|
427
|
-
timeout: Math.min(resource_client_1.SMALL_TIMEOUT_MILLIS,
|
|
411
|
+
timeout: Math.min(resource_client_1.SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
|
|
428
412
|
params: this._params(),
|
|
429
413
|
};
|
|
430
414
|
try {
|
|
@@ -445,7 +429,6 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
445
429
|
* @see https://docs.apify.com/api/v2/request-queue-request-put
|
|
446
430
|
*/
|
|
447
431
|
async updateRequest(request, options = {}) {
|
|
448
|
-
var _a;
|
|
449
432
|
(0, ow_1.default)(request, ow_1.default.object.partialShape({
|
|
450
433
|
id: ow_1.default.string,
|
|
451
434
|
}));
|
|
@@ -455,7 +438,7 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
455
438
|
const response = await this.httpClient.call({
|
|
456
439
|
url: this._url(`requests/${request.id}`),
|
|
457
440
|
method: 'PUT',
|
|
458
|
-
timeout: Math.min(resource_client_1.MEDIUM_TIMEOUT_MILLIS,
|
|
441
|
+
timeout: Math.min(resource_client_1.MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
|
|
459
442
|
data: request,
|
|
460
443
|
params: this._params({
|
|
461
444
|
forefront: options.forefront,
|
|
@@ -470,12 +453,11 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
470
453
|
* @param id - Request ID
|
|
471
454
|
*/
|
|
472
455
|
async deleteRequest(id) {
|
|
473
|
-
var _a;
|
|
474
456
|
(0, ow_1.default)(id, ow_1.default.string);
|
|
475
457
|
await this.httpClient.call({
|
|
476
458
|
url: this._url(`requests/${id}`),
|
|
477
459
|
method: 'DELETE',
|
|
478
|
-
timeout: Math.min(resource_client_1.SMALL_TIMEOUT_MILLIS,
|
|
460
|
+
timeout: Math.min(resource_client_1.SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
|
|
479
461
|
params: this._params({
|
|
480
462
|
clientKey: this.clientKey,
|
|
481
463
|
}),
|
|
@@ -506,7 +488,6 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
506
488
|
* ```
|
|
507
489
|
*/
|
|
508
490
|
async prolongRequestLock(id, options) {
|
|
509
|
-
var _a;
|
|
510
491
|
(0, ow_1.default)(id, ow_1.default.string);
|
|
511
492
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
512
493
|
lockSecs: ow_1.default.number,
|
|
@@ -515,7 +496,7 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
515
496
|
const response = await this.httpClient.call({
|
|
516
497
|
url: this._url(`requests/${id}/lock`),
|
|
517
498
|
method: 'PUT',
|
|
518
|
-
timeout: Math.min(resource_client_1.MEDIUM_TIMEOUT_MILLIS,
|
|
499
|
+
timeout: Math.min(resource_client_1.MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
|
|
519
500
|
params: this._params({
|
|
520
501
|
forefront: options.forefront,
|
|
521
502
|
lockSecs: options.lockSecs,
|
|
@@ -535,7 +516,6 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
535
516
|
* @see https://docs.apify.com/api/v2/request-queue-request-lock-delete
|
|
536
517
|
*/
|
|
537
518
|
async deleteRequestLock(id, options = {}) {
|
|
538
|
-
var _a;
|
|
539
519
|
(0, ow_1.default)(id, ow_1.default.string);
|
|
540
520
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
541
521
|
forefront: ow_1.default.optional.boolean,
|
|
@@ -543,7 +523,7 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
543
523
|
await this.httpClient.call({
|
|
544
524
|
url: this._url(`requests/${id}/lock`),
|
|
545
525
|
method: 'DELETE',
|
|
546
|
-
timeout: Math.min(resource_client_1.SMALL_TIMEOUT_MILLIS,
|
|
526
|
+
timeout: Math.min(resource_client_1.SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
|
|
547
527
|
params: this._params({
|
|
548
528
|
forefront: options.forefront,
|
|
549
529
|
clientKey: this.clientKey,
|
|
@@ -570,11 +550,10 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
570
550
|
})
|
|
571
551
|
.validate((0, utils_1.mutuallyExclusive)('exclusiveStartId', 'cursor')));
|
|
572
552
|
const getPaginatedList = async (rqListOptions = {}) => {
|
|
573
|
-
var _a;
|
|
574
553
|
const response = await this.httpClient.call({
|
|
575
554
|
url: this._url('requests'),
|
|
576
555
|
method: 'GET',
|
|
577
|
-
timeout: Math.min(resource_client_1.MEDIUM_TIMEOUT_MILLIS,
|
|
556
|
+
timeout: Math.min(resource_client_1.MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
|
|
578
557
|
params: this._params({
|
|
579
558
|
...rqListOptions,
|
|
580
559
|
filter: rqListOptions.filter ? rqListOptions.filter.join(',') : undefined,
|
|
@@ -622,11 +601,10 @@ class RequestQueueClient extends resource_client_1.ResourceClient {
|
|
|
622
601
|
* @see https://docs.apify.com/api/v2/request-queue-requests-unlock-post
|
|
623
602
|
*/
|
|
624
603
|
async unlockRequests() {
|
|
625
|
-
var _a;
|
|
626
604
|
const response = await this.httpClient.call({
|
|
627
605
|
url: this._url('requests/unlock'),
|
|
628
606
|
method: 'POST',
|
|
629
|
-
timeout: Math.min(resource_client_1.MEDIUM_TIMEOUT_MILLIS,
|
|
607
|
+
timeout: Math.min(resource_client_1.MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
|
|
630
608
|
params: this._params({
|
|
631
609
|
clientKey: this.clientKey,
|
|
632
610
|
}),
|
|
@@ -254,16 +254,15 @@ class RunClient extends resource_client_1.ResourceClient {
|
|
|
254
254
|
* @see https://docs.apify.com/api/v2/post-charge-run
|
|
255
255
|
*/
|
|
256
256
|
async charge(options) {
|
|
257
|
-
var _a, _b;
|
|
258
257
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
259
258
|
eventName: ow_1.default.string,
|
|
260
259
|
count: ow_1.default.optional.number,
|
|
261
260
|
idempotencyKey: ow_1.default.optional.string,
|
|
262
261
|
}));
|
|
263
|
-
const count =
|
|
262
|
+
const count = options.count ?? 1;
|
|
264
263
|
/** To avoid duplicates during the same milisecond, doesn't need to by crypto-secure. */
|
|
265
264
|
const randomSuffix = (Math.random() + 1).toString(36).slice(3, 8);
|
|
266
|
-
const idempotencyKey =
|
|
265
|
+
const idempotencyKey = options.idempotencyKey ?? `${this.id}-${options.eventName}-${Date.now()}-${randomSuffix}`;
|
|
267
266
|
const request = {
|
|
268
267
|
url: this._url('charge'),
|
|
269
268
|
method: 'POST',
|
|
@@ -385,7 +384,6 @@ class RunClient extends resource_client_1.ResourceClient {
|
|
|
385
384
|
* Get StreamedLog for convenient streaming of the run log and their redirection.
|
|
386
385
|
*/
|
|
387
386
|
async getStreamedLog(options = {}) {
|
|
388
|
-
var _a, _b, _c;
|
|
389
387
|
const { fromStart = true } = options;
|
|
390
388
|
let { toLog } = options;
|
|
391
389
|
if (toLog === null || !(0, utils_1.isNode)()) {
|
|
@@ -396,10 +394,10 @@ class RunClient extends resource_client_1.ResourceClient {
|
|
|
396
394
|
// Create default StreamedLog
|
|
397
395
|
// Get actor name and run id
|
|
398
396
|
const runData = await this.get();
|
|
399
|
-
const runId =
|
|
400
|
-
const actorId =
|
|
397
|
+
const runId = runData?.id ?? '';
|
|
398
|
+
const actorId = runData?.actId ?? '';
|
|
401
399
|
const actorData = (await this.apifyClient.actor(actorId).get()) || { name: '' };
|
|
402
|
-
const actorName =
|
|
400
|
+
const actorName = actorData?.name ?? '';
|
|
403
401
|
const name = [actorName, `runId:${runId}`].filter(Boolean).join(' ');
|
|
404
402
|
toLog = new log_1.Log({ level: log_1.LEVELS.DEBUG, prefix: `${name} -> `, logger: new log_2.LoggerActorRedirect() });
|
|
405
403
|
}
|
package/dist/statistics.js
CHANGED
|
@@ -4,36 +4,19 @@ exports.Statistics = void 0;
|
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
6
6
|
class Statistics {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
Object.defineProperty(this, "requests", {
|
|
21
|
-
enumerable: true,
|
|
22
|
-
configurable: true,
|
|
23
|
-
writable: true,
|
|
24
|
-
value: 0
|
|
25
|
-
});
|
|
26
|
-
/**
|
|
27
|
-
* Number of times the API returned 429 error. Errors on first attempt are
|
|
28
|
-
* counted at index 0. First retry error counts are on index 1 and so on.
|
|
29
|
-
*/
|
|
30
|
-
Object.defineProperty(this, "rateLimitErrors", {
|
|
31
|
-
enumerable: true,
|
|
32
|
-
configurable: true,
|
|
33
|
-
writable: true,
|
|
34
|
-
value: []
|
|
35
|
-
});
|
|
36
|
-
}
|
|
7
|
+
/**
|
|
8
|
+
* Number of Apify client function calls
|
|
9
|
+
*/
|
|
10
|
+
calls = 0;
|
|
11
|
+
/**
|
|
12
|
+
* Number of Apify API requests
|
|
13
|
+
*/
|
|
14
|
+
requests = 0;
|
|
15
|
+
/**
|
|
16
|
+
* Number of times the API returned 429 error. Errors on first attempt are
|
|
17
|
+
* counted at index 0. First retry error counts are on index 1 and so on.
|
|
18
|
+
*/
|
|
19
|
+
rateLimitErrors = [];
|
|
37
20
|
addRateLimitError(attempt) {
|
|
38
21
|
(0, ow_1.default)(attempt, ow_1.default.number.greaterThan(0));
|
|
39
22
|
// attempt is never 0,
|