apify-client 3.0.0-beta.2 → 3.0.0-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/apify_api_error.d.ts +60 -19
- package/dist/apify_api_error.js +88 -22
- package/dist/apify_client.d.ts +22 -3
- package/dist/apify_client.js +37 -31
- package/dist/base/api_client.d.ts +6 -6
- package/dist/base/api_client.js +24 -16
- package/dist/base/resource_client.d.ts +23 -9
- package/dist/base/resource_client.js +49 -25
- package/dist/base/resource_collection_client.d.ts +7 -4
- package/dist/base/resource_collection_client.js +21 -15
- package/dist/body_parser.d.ts +3 -4
- package/dist/body_parser.js +17 -24
- package/dist/bundle.js +38 -48
- package/dist/bundle.js.map +1 -1
- package/dist/generated/api.d.ts +287 -214
- package/dist/generated/schemas.d.ts +6699 -0
- package/dist/generated/schemas.js +1521 -0
- package/dist/http_client.d.ts +18 -51
- package/dist/http_client.js +90 -85
- package/dist/index.d.ts +3 -1
- package/dist/index.js +2 -1
- package/dist/interceptors.js +24 -6
- package/dist/lazy_schema.d.ts +8 -0
- package/dist/lazy_schema.js +11 -0
- package/dist/models.d.ts +32 -52
- package/dist/resource_clients/actor.d.ts +46 -31
- package/dist/resource_clients/actor.js +71 -53
- package/dist/resource_clients/actor_collection.d.ts +9 -5
- package/dist/resource_clients/actor_collection.js +12 -3
- package/dist/resource_clients/actor_env_var.d.ts +10 -3
- package/dist/resource_clients/actor_env_var.js +17 -6
- package/dist/resource_clients/actor_env_var_collection.d.ts +14 -18
- package/dist/resource_clients/actor_env_var_collection.js +18 -11
- package/dist/resource_clients/actor_version.d.ts +18 -3
- package/dist/resource_clients/actor_version.js +20 -9
- package/dist/resource_clients/actor_version_collection.d.ts +16 -17
- package/dist/resource_clients/actor_version_collection.js +20 -11
- package/dist/resource_clients/build.d.ts +18 -6
- package/dist/resource_clients/build.js +35 -17
- package/dist/resource_clients/build_collection.d.ts +3 -1
- package/dist/resource_clients/build_collection.js +5 -1
- package/dist/resource_clients/dataset.d.ts +23 -9
- package/dist/resource_clients/dataset.js +70 -48
- package/dist/resource_clients/dataset_collection.d.ts +6 -2
- package/dist/resource_clients/dataset_collection.js +13 -2
- package/dist/resource_clients/key_value_store.d.ts +48 -15
- package/dist/resource_clients/key_value_store.js +71 -40
- package/dist/resource_clients/key_value_store_collection.d.ts +6 -2
- package/dist/resource_clients/key_value_store_collection.js +13 -2
- package/dist/resource_clients/log.d.ts +9 -19
- package/dist/resource_clients/log.js +57 -45
- package/dist/resource_clients/request_queue.d.ts +56 -26
- package/dist/resource_clients/request_queue.js +178 -113
- package/dist/resource_clients/request_queue_collection.d.ts +6 -2
- package/dist/resource_clients/request_queue_collection.js +10 -3
- package/dist/resource_clients/run.d.ts +51 -17
- package/dist/resource_clients/run.js +85 -44
- package/dist/resource_clients/run_collection.d.ts +3 -1
- package/dist/resource_clients/run_collection.js +5 -1
- package/dist/resource_clients/schedule.d.ts +16 -7
- package/dist/resource_clients/schedule.js +31 -21
- package/dist/resource_clients/schedule_collection.d.ts +6 -2
- package/dist/resource_clients/schedule_collection.js +10 -3
- package/dist/resource_clients/store_collection.d.ts +3 -1
- package/dist/resource_clients/store_collection.js +5 -1
- package/dist/resource_clients/task.d.ts +28 -10
- package/dist/resource_clients/task.js +67 -44
- package/dist/resource_clients/task_collection.d.ts +6 -2
- package/dist/resource_clients/task_collection.js +10 -3
- package/dist/resource_clients/user.d.ts +17 -8
- package/dist/resource_clients/user.js +40 -39
- package/dist/resource_clients/webhook.d.ts +14 -5
- package/dist/resource_clients/webhook.js +30 -22
- package/dist/resource_clients/webhook_collection.d.ts +6 -2
- package/dist/resource_clients/webhook_collection.js +10 -3
- package/dist/resource_clients/webhook_dispatch.d.ts +4 -1
- package/dist/resource_clients/webhook_dispatch.js +8 -2
- package/dist/resource_clients/webhook_dispatch_collection.d.ts +3 -1
- package/dist/resource_clients/webhook_dispatch_collection.js +5 -1
- package/dist/response_validation_error.d.ts +26 -0
- package/dist/response_validation_error.js +37 -0
- package/dist/runtime/node.d.ts +6 -0
- package/dist/runtime/node.js +58 -0
- package/dist/runtime/types.d.ts +50 -0
- package/dist/runtime/types.js +1 -0
- package/dist/runtime/web.d.ts +5 -0
- package/dist/runtime/web.js +17 -0
- package/dist/schemas.d.ts +15 -0
- package/dist/schemas.js +15 -0
- package/dist/statistics.d.ts +1 -6
- package/dist/statistics.js +2 -2
- package/dist/timeouts.d.ts +72 -0
- package/dist/timeouts.js +31 -0
- package/dist/utils.d.ts +75 -55
- package/dist/utils.js +201 -180
- package/package.json +42 -35
- package/dist/argument_validation_error.d.ts +0 -17
- package/dist/argument_validation_error.js +0 -153
|
@@ -1,23 +1,26 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { MAX_PAYLOAD_SIZE_BYTES, REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION } from '@apify/consts';
|
|
3
3
|
import log from '@apify/log';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { ResourceClient } from '../base/resource_client.js';
|
|
5
|
+
import { ResponseValidationError } from '../response_validation_error.js';
|
|
6
|
+
import * as schemas from '../schemas.js';
|
|
7
|
+
import { optionalTimeoutSchema, timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
|
|
8
|
+
import { anyObjectSchema, catchNotFoundOrThrow, isNonArrayObject, parseArgument, parseResponse, RequestQueuePaginationIterator, splitIntoJsonArrayBatches, utf8ByteLength, } from '../utils.js';
|
|
6
9
|
const DEFAULT_PARALLEL_BATCH_ADD_REQUESTS = 5;
|
|
7
10
|
const DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS = 3;
|
|
8
11
|
const DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS = 500;
|
|
9
12
|
const DEFAULT_REQUEST_QUEUE_REQUEST_PAGE_LIMIT = 1000;
|
|
10
13
|
const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
|
|
11
|
-
const listHeadOptionsSchema = z.strictObject({ limit: z.number().min(0).optional() });
|
|
14
|
+
const listHeadOptionsSchema = z.strictObject({ limit: z.number().min(0).optional(), ...timeoutOptionsShape });
|
|
12
15
|
const listAndLockHeadOptionsSchema = z.strictObject({
|
|
13
16
|
lockSecs: z.number(),
|
|
14
17
|
limit: z.number().min(0).optional(),
|
|
18
|
+
...timeoutOptionsShape,
|
|
15
19
|
});
|
|
16
20
|
// Predicates, not `z.looseObject` arms: these run over a whole batch, and an object arm would copy
|
|
17
21
|
// every key of every request. `id` is assigned by the API, so a new request must not carry one.
|
|
18
22
|
const newRequestSchema = z.custom((value) => isNonArrayObject(value) && value.id === undefined, 'Expected a request object without an `id`');
|
|
19
|
-
const forefrontOptionsSchema = z.strictObject({ forefront: z.boolean().optional() });
|
|
20
|
-
const batchAddRequestsSchema = z.array(newRequestSchema).min(1).max(REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION);
|
|
23
|
+
const forefrontOptionsSchema = z.strictObject({ forefront: z.boolean().optional(), ...timeoutOptionsShape });
|
|
21
24
|
const batchAddRequestsWithRetriesSchema = z.array(newRequestSchema).min(1);
|
|
22
25
|
const optionalBooleanSchema = z.boolean().optional();
|
|
23
26
|
const optionalNumberSchema = z.number().optional();
|
|
@@ -31,25 +34,22 @@ const existingRequestSchema = z.custom((value) => isNonArrayObject(value) && typ
|
|
|
31
34
|
const prolongRequestLockOptionsSchema = z.strictObject({
|
|
32
35
|
lockSecs: z.number(),
|
|
33
36
|
forefront: z.boolean().optional(),
|
|
37
|
+
...timeoutOptionsShape,
|
|
34
38
|
});
|
|
35
39
|
const requestFilterSchema = z.array(z.enum(['locked', 'pending'])).min(1);
|
|
36
|
-
const listRequestsOptionsSchema = z
|
|
37
|
-
.strictObject({
|
|
40
|
+
const listRequestsOptionsSchema = z.strictObject({
|
|
38
41
|
limit: z.number().min(0).optional(),
|
|
39
|
-
exclusiveStartId: z.string().optional(),
|
|
40
42
|
cursor: z.string().optional(),
|
|
41
43
|
filter: requestFilterSchema.optional(),
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
const paginateRequestsOptionsSchema = z
|
|
45
|
-
.strictObject({
|
|
44
|
+
...timeoutOptionsShape,
|
|
45
|
+
});
|
|
46
|
+
const paginateRequestsOptionsSchema = z.strictObject({
|
|
46
47
|
limit: z.number().min(0).optional(),
|
|
47
48
|
maxPageLimit: z.number().default(DEFAULT_REQUEST_QUEUE_REQUEST_PAGE_LIMIT),
|
|
48
|
-
exclusiveStartId: z.string().optional(),
|
|
49
49
|
cursor: z.string().optional(),
|
|
50
50
|
filter: requestFilterSchema.optional(),
|
|
51
|
-
|
|
52
|
-
|
|
51
|
+
...timeoutOptionsShape,
|
|
52
|
+
});
|
|
53
53
|
/**
|
|
54
54
|
* Client for managing a specific Request queue.
|
|
55
55
|
*
|
|
@@ -81,8 +81,8 @@ const paginateRequestsOptionsSchema = z
|
|
|
81
81
|
* @see https://docs.apify.com/platform/storage/request-queue
|
|
82
82
|
*/
|
|
83
83
|
export class RequestQueueClient extends ResourceClient {
|
|
84
|
-
clientKey;
|
|
85
|
-
|
|
84
|
+
#clientKey;
|
|
85
|
+
#timeoutSecs;
|
|
86
86
|
/**
|
|
87
87
|
* @hidden
|
|
88
88
|
*/
|
|
@@ -91,36 +91,57 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
91
91
|
resourcePath: 'request-queues',
|
|
92
92
|
...options,
|
|
93
93
|
});
|
|
94
|
-
this
|
|
95
|
-
this
|
|
94
|
+
this.#clientKey = userOptions.clientKey;
|
|
95
|
+
this.#timeoutSecs = userOptions.timeoutSecs;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Picks the timeout of a request: an explicit per-call `timeoutSecs` as is, otherwise the method's default
|
|
99
|
+
* tier, capped at the queue-wide `timeoutSecs` when one was given.
|
|
100
|
+
*/
|
|
101
|
+
#resolveTimeout(timeoutSecs, defaultTier) {
|
|
102
|
+
if (timeoutSecs !== undefined)
|
|
103
|
+
return timeoutSecs;
|
|
104
|
+
if (this.#timeoutSecs === undefined)
|
|
105
|
+
return defaultTier;
|
|
106
|
+
const tierSecs = this.httpClient.timeoutMillis[defaultTier] / 1000;
|
|
107
|
+
return tierSecs <= this.#timeoutSecs ? defaultTier : this.#timeoutSecs;
|
|
96
108
|
}
|
|
97
109
|
/**
|
|
98
110
|
* Gets the Request queue object from the Apify API.
|
|
99
111
|
*
|
|
112
|
+
* @param options - Request options
|
|
113
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
100
114
|
* @returns The RequestQueue object, or `undefined` if it does not exist
|
|
101
115
|
* @see https://docs.apify.com/api/v2/request-queue-get
|
|
102
116
|
*/
|
|
103
|
-
async get() {
|
|
104
|
-
|
|
117
|
+
async get(options = {}) {
|
|
118
|
+
const { timeoutSecs } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
119
|
+
return this.getResource(schemas.RequestQueue(), {}, this.#resolveTimeout(timeoutSecs, 'short'));
|
|
105
120
|
}
|
|
106
121
|
/**
|
|
107
122
|
* Updates the Request queue with specified fields.
|
|
108
123
|
*
|
|
109
124
|
* @param newFields - Fields to update in the Request queue
|
|
125
|
+
* @param options - Request options
|
|
126
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
110
127
|
* @returns The updated RequestQueue object
|
|
111
128
|
* @see https://docs.apify.com/api/v2/request-queue-put
|
|
112
129
|
*/
|
|
113
|
-
async update(newFields) {
|
|
130
|
+
async update(newFields, options = {}) {
|
|
114
131
|
parseArgument(newFields, anyObjectSchema);
|
|
115
|
-
|
|
132
|
+
const { timeoutSecs } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
133
|
+
return this.updateResource(schemas.RequestQueue(), newFields, this.#resolveTimeout(timeoutSecs, 'short'));
|
|
116
134
|
}
|
|
117
135
|
/**
|
|
118
136
|
* Deletes the Request queue.
|
|
119
137
|
*
|
|
138
|
+
* @param options - Request options
|
|
139
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
120
140
|
* @see https://docs.apify.com/api/v2/request-queue-delete
|
|
121
141
|
*/
|
|
122
|
-
async delete() {
|
|
123
|
-
|
|
142
|
+
async delete(options = {}) {
|
|
143
|
+
const { timeoutSecs } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
144
|
+
return this.deleteResource(this.#resolveTimeout(timeoutSecs, 'short'));
|
|
124
145
|
}
|
|
125
146
|
/**
|
|
126
147
|
* Lists requests from the beginning of the queue (head).
|
|
@@ -129,21 +150,23 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
129
150
|
* inspecting what requests are waiting to be processed.
|
|
130
151
|
*
|
|
131
152
|
* @param options - Options for listing (e.g., limit)
|
|
153
|
+
* @param options.limit - Maximum number of requests to return.
|
|
154
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
132
155
|
* @returns List of requests from the queue head
|
|
133
156
|
* @see https://docs.apify.com/api/v2/request-queue-head-get
|
|
134
157
|
*/
|
|
135
158
|
async listHead(options = {}) {
|
|
136
159
|
const parsed = parseArgument(options, listHeadOptionsSchema, 'RequestQueueClientListHeadOptions');
|
|
137
160
|
const response = await this.httpClient.call({
|
|
138
|
-
url: this.
|
|
161
|
+
url: this.buildUrl('head'),
|
|
139
162
|
method: 'GET',
|
|
140
|
-
|
|
141
|
-
params: this.
|
|
163
|
+
timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
|
|
164
|
+
params: this.buildParams({
|
|
142
165
|
limit: parsed.limit,
|
|
143
|
-
clientKey: this
|
|
166
|
+
clientKey: this.#clientKey,
|
|
144
167
|
}),
|
|
145
168
|
});
|
|
146
|
-
return
|
|
169
|
+
return parseResponse(response, schemas.RequestQueueHead());
|
|
147
170
|
}
|
|
148
171
|
/**
|
|
149
172
|
* Gets and locks the next requests from the queue head for processing.
|
|
@@ -157,6 +180,7 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
157
180
|
* @param options - Lock configuration
|
|
158
181
|
* @param options.lockSecs - **Required.** Duration in seconds to lock the requests. After this time, the locks expire and requests can be retrieved by other clients.
|
|
159
182
|
* @param options.limit - Maximum number of requests to return. Default is 25.
|
|
183
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
|
|
160
184
|
* @returns Object containing `items` (locked requests), `queueModifiedAt`, `hadMultipleClients`, and lock information
|
|
161
185
|
* @see https://docs.apify.com/api/v2/request-queue-head-lock-post
|
|
162
186
|
*
|
|
@@ -181,16 +205,16 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
181
205
|
async listAndLockHead(options) {
|
|
182
206
|
const parsed = parseArgument(options, listAndLockHeadOptionsSchema, 'RequestQueueClientListAndLockHeadOptions');
|
|
183
207
|
const response = await this.httpClient.call({
|
|
184
|
-
url: this.
|
|
208
|
+
url: this.buildUrl('head/lock'),
|
|
185
209
|
method: 'POST',
|
|
186
|
-
|
|
187
|
-
params: this.
|
|
210
|
+
timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
|
|
211
|
+
params: this.buildParams({
|
|
188
212
|
limit: parsed.limit,
|
|
189
213
|
lockSecs: parsed.lockSecs,
|
|
190
|
-
clientKey: this
|
|
214
|
+
clientKey: this.#clientKey,
|
|
191
215
|
}),
|
|
192
216
|
});
|
|
193
|
-
return
|
|
217
|
+
return parseResponse(response, schemas.LockedRequestQueueHead());
|
|
194
218
|
}
|
|
195
219
|
/**
|
|
196
220
|
* Adds a single request to the queue.
|
|
@@ -208,6 +232,7 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
208
232
|
* @param request.payload - HTTP payload for POST/PUT requests (string).
|
|
209
233
|
* @param options - Additional options
|
|
210
234
|
* @param options.forefront - If `true`, adds the request to the beginning of the queue. Default is `false` (adds to the end).
|
|
235
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
211
236
|
* @returns Object with `requestId`, `wasAlreadyPresent`, and `wasAlreadyHandled` flags
|
|
212
237
|
* @see https://docs.apify.com/api/v2/request-queue-requests-post
|
|
213
238
|
*
|
|
@@ -234,39 +259,41 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
234
259
|
parseArgument(request, newRequestSchema);
|
|
235
260
|
const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
|
|
236
261
|
const response = await this.httpClient.call({
|
|
237
|
-
url: this.
|
|
262
|
+
url: this.buildUrl('requests'),
|
|
238
263
|
method: 'POST',
|
|
239
|
-
|
|
264
|
+
timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
|
|
240
265
|
data: request,
|
|
241
|
-
params: this.
|
|
266
|
+
params: this.buildParams({
|
|
242
267
|
forefront: parsed.forefront,
|
|
243
|
-
clientKey: this
|
|
268
|
+
clientKey: this.#clientKey,
|
|
244
269
|
}),
|
|
245
270
|
});
|
|
246
|
-
return
|
|
271
|
+
return parseResponse(response, schemas.RequestRegistration());
|
|
247
272
|
}
|
|
248
273
|
/**
|
|
249
274
|
* Writes requests to Request queue in batch.
|
|
250
275
|
*
|
|
251
276
|
* @private
|
|
252
277
|
*/
|
|
253
|
-
async
|
|
254
|
-
parseArgument(requests, batchAddRequestsSchema);
|
|
278
|
+
async addRequestBatch(requests, options = {}) {
|
|
255
279
|
const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
|
|
256
|
-
const
|
|
257
|
-
url: this.
|
|
280
|
+
const response = await this.httpClient.call({
|
|
281
|
+
url: this.buildUrl('requests/batch'),
|
|
258
282
|
method: 'POST',
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
283
|
+
timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
|
|
284
|
+
// The body is assembled from the requests as `batchAddRequests` serialized them; the explicit content type
|
|
285
|
+
// makes the request interceptor send the string as it is instead of serializing the requests again.
|
|
286
|
+
headers: { 'content-type': 'application/json' },
|
|
287
|
+
data: `[${requests.map(({ json }) => json).join(',')}]`,
|
|
288
|
+
params: this.buildParams({
|
|
262
289
|
forefront: parsed.forefront,
|
|
263
|
-
clientKey: this
|
|
290
|
+
clientKey: this.#clientKey,
|
|
264
291
|
}),
|
|
265
292
|
});
|
|
266
|
-
return
|
|
293
|
+
return parseResponse(response, schemas.BatchAddResult());
|
|
267
294
|
}
|
|
268
|
-
async
|
|
269
|
-
const { forefront, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
|
|
295
|
+
async addRequestBatchWithRetries(requests, options = {}) {
|
|
296
|
+
const { forefront, timeoutSecs, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
|
|
270
297
|
// Keep track of the requests that remain to be processed (in parameter format)
|
|
271
298
|
let remainingRequests = requests;
|
|
272
299
|
// Keep track of the requests that have been processed (in api format)
|
|
@@ -276,8 +303,9 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
276
303
|
let unprocessedRequests = [];
|
|
277
304
|
for (let i = 0; i < 1 + maxUnprocessedRequestsRetries; i++) {
|
|
278
305
|
try {
|
|
279
|
-
const response = await this.
|
|
306
|
+
const response = await this.addRequestBatch(remainingRequests, {
|
|
280
307
|
forefront,
|
|
308
|
+
timeoutSecs,
|
|
281
309
|
});
|
|
282
310
|
processedRequests.push(...response.processedRequests);
|
|
283
311
|
unprocessedRequests = response.unprocessedRequests;
|
|
@@ -289,20 +317,24 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
289
317
|
// Get unique keys of all requests processed so far
|
|
290
318
|
const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
|
|
291
319
|
// Requests remaining to be processed are the all that remain
|
|
292
|
-
remainingRequests = requests.filter(({
|
|
320
|
+
remainingRequests = requests.filter(({ request }) => !processedRequestsUniqueKeys.includes(request.uniqueKey));
|
|
293
321
|
// Stop if all requests have been processed
|
|
294
322
|
if (remainingRequests.length === 0) {
|
|
295
323
|
break;
|
|
296
324
|
}
|
|
297
325
|
}
|
|
298
326
|
catch (err) {
|
|
327
|
+
// A response the specification does not describe is reported, as everywhere else: the server may well
|
|
328
|
+
// have added the batch, so calling it unprocessed would hide the mismatch behind a wrong answer.
|
|
329
|
+
if (err instanceof ResponseValidationError)
|
|
330
|
+
throw err;
|
|
299
331
|
log.exception(err, 'Request batch insert failed');
|
|
300
332
|
// When something fails and http client does not retry, the remaining requests are treated as unprocessed.
|
|
301
333
|
// This ensures that this method does not throw and keeps the signature.
|
|
302
334
|
const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
|
|
303
335
|
unprocessedRequests = requests
|
|
304
|
-
.filter(({
|
|
305
|
-
.map(({ method, uniqueKey, url }) => ({ method, uniqueKey, url }));
|
|
336
|
+
.filter(({ request }) => !processedRequestsUniqueKeys.includes(request.uniqueKey))
|
|
337
|
+
.map(({ request: { method, uniqueKey, url } }) => ({ method, uniqueKey, url }));
|
|
306
338
|
break;
|
|
307
339
|
}
|
|
308
340
|
// Exponential backoff
|
|
@@ -311,8 +343,7 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
311
343
|
setTimeout(resolve, delayMillis);
|
|
312
344
|
});
|
|
313
345
|
}
|
|
314
|
-
|
|
315
|
-
return cast(parseDateFields(result));
|
|
346
|
+
return { processedRequests, unprocessedRequests };
|
|
316
347
|
}
|
|
317
348
|
/**
|
|
318
349
|
* Adds multiple requests to the queue in a single operation.
|
|
@@ -329,6 +360,7 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
329
360
|
* @param options.maxUnprocessedRequestsRetries - Maximum number of retry attempts for rate-limited requests. Default is 3.
|
|
330
361
|
* @param options.maxParallel - Maximum number of parallel batch API calls. Default is 5.
|
|
331
362
|
* @param options.minDelayBetweenUnprocessedRequestsRetriesMillis - Minimum delay before retrying rate-limited requests. Default is 500ms.
|
|
363
|
+
* @param options.timeoutSecs - Timeout for each batch API request. Default is `'medium'`.
|
|
332
364
|
* @returns Object with `processedRequests` (successfully added) and `unprocessedRequests` (failed after all retries)
|
|
333
365
|
* @see https://docs.apify.com/api/v2/request-queue-requests-batch-post
|
|
334
366
|
*
|
|
@@ -353,30 +385,45 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
353
385
|
* @since Added in 2.1.0
|
|
354
386
|
*/
|
|
355
387
|
async batchAddRequests(requests, options = {}) {
|
|
356
|
-
const { forefront, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, maxParallel = DEFAULT_PARALLEL_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
|
|
388
|
+
const { forefront, timeoutSecs, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, maxParallel = DEFAULT_PARALLEL_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
|
|
357
389
|
parseArgument(requests, batchAddRequestsWithRetriesSchema);
|
|
358
390
|
parseArgument(forefront, optionalBooleanSchema);
|
|
391
|
+
parseArgument(timeoutSecs, optionalTimeoutSchema);
|
|
359
392
|
parseArgument(maxUnprocessedRequestsRetries, optionalNumberSchema);
|
|
360
393
|
parseArgument(maxParallel, optionalNumberSchema);
|
|
361
394
|
parseArgument(minDelayBetweenUnprocessedRequestsRetriesMillis, optionalNumberSchema);
|
|
362
395
|
const executingRequests = new Set();
|
|
363
396
|
const individualResults = [];
|
|
364
397
|
const payloadSizeLimitBytes = MAX_PAYLOAD_SIZE_BYTES - Math.ceil(MAX_PAYLOAD_SIZE_BYTES * SAFETY_BUFFER_PERCENT);
|
|
398
|
+
// Serialize every request once: the byte lengths decide the batch boundaries, and the same strings are joined
|
|
399
|
+
// into the batch bodies, so nothing is stringified a second time when it is sent.
|
|
400
|
+
const serializedRequests = requests.map((request, index) => {
|
|
401
|
+
const json = JSON.stringify(request);
|
|
402
|
+
const byteLength = utf8ByteLength(json);
|
|
403
|
+
// Two more bytes for the brackets, which even a batch of one request carries.
|
|
404
|
+
if (byteLength + 2 > payloadSizeLimitBytes) {
|
|
405
|
+
throw new Error(`RequestQueueClient.batchAddRequests: The size of the request with index: ${index} ` +
|
|
406
|
+
`exceeds the maximum allowed size (${payloadSizeLimitBytes} bytes).`);
|
|
407
|
+
}
|
|
408
|
+
return { request, json, byteLength };
|
|
409
|
+
});
|
|
410
|
+
const batches = splitIntoJsonArrayBatches(serializedRequests, {
|
|
411
|
+
maxCount: REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION,
|
|
412
|
+
maxByteLength: payloadSizeLimitBytes,
|
|
413
|
+
});
|
|
365
414
|
// Keep a pool of up to `maxParallel` requests running at once
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
const slicedRequests = requests.slice(i, i + REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION);
|
|
369
|
-
const requestsInBatch = sliceArrayByByteLength(slicedRequests, payloadSizeLimitBytes, i);
|
|
370
|
-
const requestPromise = this._batchAddRequestsWithRetries(requestsInBatch, options);
|
|
415
|
+
for (const batch of batches) {
|
|
416
|
+
const requestPromise = this.addRequestBatchWithRetries(batch, options);
|
|
371
417
|
executingRequests.add(requestPromise);
|
|
418
|
+
// A rejection reaches the caller through the awaits below; this bookkeeping chain only has to avoid
|
|
419
|
+
// turning it into an unhandled one of its own.
|
|
372
420
|
void requestPromise.then((batchAddResult) => {
|
|
373
421
|
executingRequests.delete(requestPromise);
|
|
374
422
|
individualResults.push(batchAddResult);
|
|
375
|
-
});
|
|
423
|
+
}, () => undefined);
|
|
376
424
|
if (executingRequests.size >= maxParallel) {
|
|
377
425
|
await Promise.race(executingRequests);
|
|
378
426
|
}
|
|
379
|
-
i += requestsInBatch.length;
|
|
380
427
|
}
|
|
381
428
|
// Get results from remaining operations
|
|
382
429
|
await Promise.all(executingRequests);
|
|
@@ -397,41 +444,47 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
397
444
|
* Requests can be identified by either their ID or unique key.
|
|
398
445
|
*
|
|
399
446
|
* @param requests - Array of requests to delete (by id or uniqueKey)
|
|
447
|
+
* @param options - Request options
|
|
448
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
400
449
|
* @returns Result containing processed and unprocessed requests
|
|
401
450
|
* @see https://docs.apify.com/api/v2/request-queue-requests-batch-delete
|
|
402
451
|
* @since Added in 2.3.0
|
|
403
452
|
*/
|
|
404
|
-
async batchDeleteRequests(requests) {
|
|
453
|
+
async batchDeleteRequests(requests, options = {}) {
|
|
405
454
|
parseArgument(requests, batchDeleteRequestsSchema);
|
|
406
|
-
const {
|
|
407
|
-
|
|
455
|
+
const { timeoutSecs } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
456
|
+
const response = await this.httpClient.call({
|
|
457
|
+
url: this.buildUrl('requests/batch'),
|
|
408
458
|
method: 'DELETE',
|
|
409
|
-
|
|
459
|
+
timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
|
|
410
460
|
data: requests,
|
|
411
|
-
params: this.
|
|
412
|
-
clientKey: this
|
|
461
|
+
params: this.buildParams({
|
|
462
|
+
clientKey: this.#clientKey,
|
|
413
463
|
}),
|
|
414
464
|
});
|
|
415
|
-
return
|
|
465
|
+
return parseResponse(response, schemas.BatchDeleteResult());
|
|
416
466
|
}
|
|
417
467
|
/**
|
|
418
468
|
* Gets a specific request from the queue by its ID.
|
|
419
469
|
*
|
|
420
470
|
* @param id - Request ID
|
|
471
|
+
* @param options - Request options
|
|
472
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
421
473
|
* @returns The request object, or `undefined` if not found
|
|
422
474
|
* @see https://docs.apify.com/api/v2/request-queue-request-get
|
|
423
475
|
*/
|
|
424
|
-
async getRequest(id) {
|
|
476
|
+
async getRequest(id, options = {}) {
|
|
425
477
|
parseArgument(id, requestIdSchema);
|
|
478
|
+
const { timeoutSecs } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
426
479
|
const requestOpts = {
|
|
427
|
-
url: this.
|
|
480
|
+
url: this.buildUrl(['requests', id]),
|
|
428
481
|
method: 'GET',
|
|
429
|
-
|
|
430
|
-
params: this.
|
|
482
|
+
timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
|
|
483
|
+
params: this.buildParams(),
|
|
431
484
|
};
|
|
432
485
|
try {
|
|
433
486
|
const response = await this.httpClient.call(requestOpts);
|
|
434
|
-
return
|
|
487
|
+
return parseResponse(response, schemas.Request());
|
|
435
488
|
}
|
|
436
489
|
catch (err) {
|
|
437
490
|
catchNotFoundOrThrow(err);
|
|
@@ -443,6 +496,8 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
443
496
|
*
|
|
444
497
|
* @param request - The updated request object (must include id)
|
|
445
498
|
* @param options - Update options such as whether to move to front
|
|
499
|
+
* @param options.forefront - If `true`, moves the request to the beginning of the queue. Default is `false`.
|
|
500
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
|
|
446
501
|
* @returns Information about the updated request
|
|
447
502
|
* @see https://docs.apify.com/api/v2/request-queue-request-put
|
|
448
503
|
*/
|
|
@@ -450,30 +505,33 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
450
505
|
parseArgument(request, existingRequestSchema);
|
|
451
506
|
const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
|
|
452
507
|
const response = await this.httpClient.call({
|
|
453
|
-
url: this.
|
|
508
|
+
url: this.buildUrl(['requests', request.id]),
|
|
454
509
|
method: 'PUT',
|
|
455
|
-
|
|
510
|
+
timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
|
|
456
511
|
data: request,
|
|
457
|
-
params: this.
|
|
512
|
+
params: this.buildParams({
|
|
458
513
|
forefront: parsed.forefront,
|
|
459
|
-
clientKey: this
|
|
514
|
+
clientKey: this.#clientKey,
|
|
460
515
|
}),
|
|
461
516
|
});
|
|
462
|
-
return
|
|
517
|
+
return parseResponse(response, schemas.RequestRegistration());
|
|
463
518
|
}
|
|
464
519
|
/**
|
|
465
520
|
* Deletes a specific request from the queue.
|
|
466
521
|
*
|
|
467
522
|
* @param id - Request ID
|
|
523
|
+
* @param options - Request options
|
|
524
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
468
525
|
*/
|
|
469
|
-
async deleteRequest(id) {
|
|
526
|
+
async deleteRequest(id, options = {}) {
|
|
470
527
|
parseArgument(id, requestIdSchema);
|
|
528
|
+
const { timeoutSecs } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
471
529
|
await this.httpClient.call({
|
|
472
|
-
url: this.
|
|
530
|
+
url: this.buildUrl(['requests', id]),
|
|
473
531
|
method: 'DELETE',
|
|
474
|
-
|
|
475
|
-
params: this.
|
|
476
|
-
clientKey: this
|
|
532
|
+
timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
|
|
533
|
+
params: this.buildParams({
|
|
534
|
+
clientKey: this.#clientKey,
|
|
477
535
|
}),
|
|
478
536
|
});
|
|
479
537
|
}
|
|
@@ -488,6 +546,7 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
488
546
|
* @param options - Lock extension options
|
|
489
547
|
* @param options.lockSecs - **Required.** New lock duration in seconds from now.
|
|
490
548
|
* @param options.forefront - If `true`, moves the request to the beginning of the queue when the lock expires. Default is `false`.
|
|
549
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
|
|
491
550
|
* @returns Object with new `lockExpiresAt` timestamp
|
|
492
551
|
* @see https://docs.apify.com/api/v2/request-queue-request-lock-put
|
|
493
552
|
*
|
|
@@ -506,16 +565,16 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
506
565
|
parseArgument(id, requestIdSchema);
|
|
507
566
|
const parsed = parseArgument(options, prolongRequestLockOptionsSchema, 'RequestQueueClientProlongRequestLockOptions');
|
|
508
567
|
const response = await this.httpClient.call({
|
|
509
|
-
url: this.
|
|
568
|
+
url: this.buildUrl(['requests', id, 'lock']),
|
|
510
569
|
method: 'PUT',
|
|
511
|
-
|
|
512
|
-
params: this.
|
|
570
|
+
timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
|
|
571
|
+
params: this.buildParams({
|
|
513
572
|
forefront: parsed.forefront,
|
|
514
573
|
lockSecs: parsed.lockSecs,
|
|
515
|
-
clientKey: this
|
|
574
|
+
clientKey: this.#clientKey,
|
|
516
575
|
}),
|
|
517
576
|
});
|
|
518
|
-
return
|
|
577
|
+
return parseResponse(response, schemas.RequestLockInfo());
|
|
519
578
|
}
|
|
520
579
|
/**
|
|
521
580
|
* Releases the lock on a request, allowing other clients to process it.
|
|
@@ -525,6 +584,8 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
525
584
|
*
|
|
526
585
|
* @param id - Request ID
|
|
527
586
|
* @param options - Options such as whether to move to front
|
|
587
|
+
* @param options.forefront - If `true`, moves the request to the beginning of the queue. Default is `false`.
|
|
588
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
528
589
|
* @see https://docs.apify.com/api/v2/request-queue-request-lock-delete
|
|
529
590
|
* @since Added in 2.4.1
|
|
530
591
|
*/
|
|
@@ -532,12 +593,12 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
532
593
|
parseArgument(id, requestIdSchema);
|
|
533
594
|
const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientDeleteRequestLockOptions');
|
|
534
595
|
await this.httpClient.call({
|
|
535
|
-
url: this.
|
|
596
|
+
url: this.buildUrl(['requests', id, 'lock']),
|
|
536
597
|
method: 'DELETE',
|
|
537
|
-
|
|
538
|
-
params: this.
|
|
598
|
+
timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
|
|
599
|
+
params: this.buildParams({
|
|
539
600
|
forefront: parsed.forefront,
|
|
540
|
-
clientKey: this
|
|
601
|
+
clientKey: this.#clientKey,
|
|
541
602
|
}),
|
|
542
603
|
});
|
|
543
604
|
}
|
|
@@ -548,24 +609,27 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
548
609
|
* queue contents.
|
|
549
610
|
*
|
|
550
611
|
* @param options - Pagination options
|
|
612
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
|
|
551
613
|
* @returns List of requests with pagination information
|
|
552
614
|
* @see https://docs.apify.com/api/v2/request-queue-requests-get
|
|
553
615
|
* @since Added in 2.5.1
|
|
554
616
|
*/
|
|
555
617
|
listRequests(options = {}) {
|
|
556
|
-
|
|
618
|
+
// `timeoutSecs` times every page request; it is not an API parameter, so it must not reach the query string.
|
|
619
|
+
const { timeoutSecs, ...parsed } = parseArgument(options, listRequestsOptionsSchema, 'RequestQueueClientListRequestsOptions');
|
|
620
|
+
const pageTimeout = this.#resolveTimeout(timeoutSecs, 'medium');
|
|
557
621
|
const getPaginatedList = async (rqListOptions = {}) => {
|
|
558
622
|
const response = await this.httpClient.call({
|
|
559
|
-
url: this.
|
|
623
|
+
url: this.buildUrl('requests'),
|
|
560
624
|
method: 'GET',
|
|
561
|
-
|
|
562
|
-
params: this.
|
|
625
|
+
timeoutSecs: pageTimeout,
|
|
626
|
+
params: this.buildParams({
|
|
563
627
|
...rqListOptions,
|
|
564
628
|
filter: rqListOptions.filter ? rqListOptions.filter.join(',') : undefined,
|
|
565
|
-
clientKey: this
|
|
629
|
+
clientKey: this.#clientKey,
|
|
566
630
|
}),
|
|
567
631
|
});
|
|
568
|
-
return
|
|
632
|
+
return parseResponse(response, schemas.ListOfRequests());
|
|
569
633
|
};
|
|
570
634
|
const paginatedListPromise = getPaginatedList(parsed);
|
|
571
635
|
async function* asyncGenerator() {
|
|
@@ -581,8 +645,6 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
581
645
|
const newOptions = {
|
|
582
646
|
...parsed,
|
|
583
647
|
limit: remainingItems,
|
|
584
|
-
// remove original exclusiveStartId, if there was any, and use cursor-based pagination
|
|
585
|
-
exclusiveStartId: undefined,
|
|
586
648
|
cursor: currentPage.nextCursor,
|
|
587
649
|
};
|
|
588
650
|
currentPage = await getPaginatedList(newOptions);
|
|
@@ -602,20 +664,23 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
602
664
|
* This is useful for releasing all locks at once, for example when shutting down
|
|
603
665
|
* a crawler gracefully.
|
|
604
666
|
*
|
|
667
|
+
* @param options - Request options
|
|
668
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
605
669
|
* @returns Number of requests that were unlocked
|
|
606
670
|
* @see https://docs.apify.com/api/v2/request-queue-requests-unlock-post
|
|
607
671
|
* @since Added in 2.12.5
|
|
608
672
|
*/
|
|
609
|
-
async unlockRequests() {
|
|
673
|
+
async unlockRequests(options = {}) {
|
|
674
|
+
const { timeoutSecs } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
610
675
|
const response = await this.httpClient.call({
|
|
611
|
-
url: this.
|
|
676
|
+
url: this.buildUrl('requests/unlock'),
|
|
612
677
|
method: 'POST',
|
|
613
|
-
|
|
614
|
-
params: this.
|
|
615
|
-
clientKey: this
|
|
678
|
+
timeoutSecs: this.#resolveTimeout(timeoutSecs, 'long'),
|
|
679
|
+
params: this.buildParams({
|
|
680
|
+
clientKey: this.#clientKey,
|
|
616
681
|
}),
|
|
617
682
|
});
|
|
618
|
-
return
|
|
683
|
+
return parseResponse(response, schemas.UnlockRequestsResult());
|
|
619
684
|
}
|
|
620
685
|
/**
|
|
621
686
|
* Returns an async iterable for paginating through all requests in the queue.
|
|
@@ -624,6 +689,7 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
624
689
|
* automatically handling pagination behind the scenes.
|
|
625
690
|
*
|
|
626
691
|
* @param options - Pagination options
|
|
692
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
|
|
627
693
|
* @returns An async iterable of request pages
|
|
628
694
|
* @see https://docs.apify.com/api/v2/request-queue-requests-get
|
|
629
695
|
*
|
|
@@ -636,11 +702,10 @@ export class RequestQueueClient extends ResourceClient {
|
|
|
636
702
|
* @since Added in 2.5.1
|
|
637
703
|
*/
|
|
638
704
|
paginateRequests(options = {}) {
|
|
639
|
-
const { limit,
|
|
705
|
+
const { limit, cursor, filter, maxPageLimit, timeoutSecs } = parseArgument(options, paginateRequestsOptionsSchema, 'RequestQueueClientPaginateRequestsOptions');
|
|
640
706
|
return new RequestQueuePaginationIterator({
|
|
641
|
-
getPage: async (pageOptions) => this.listRequests({ ...pageOptions, filter }),
|
|
707
|
+
getPage: async (pageOptions) => this.listRequests({ ...pageOptions, filter, timeoutSecs }),
|
|
642
708
|
limit,
|
|
643
|
-
exclusiveStartId,
|
|
644
709
|
cursor,
|
|
645
710
|
maxPageLimit,
|
|
646
711
|
});
|