apify-client 3.0.0-beta.7 → 3.0.0-beta.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/README.md +10 -2
  2. package/dist/apify_api_error.d.ts +13 -27
  3. package/dist/apify_api_error.js +33 -23
  4. package/dist/apify_client.d.ts +125 -10
  5. package/dist/apify_client.js +133 -48
  6. package/dist/apify_client_error.d.ts +10 -0
  7. package/dist/apify_client_error.js +10 -0
  8. package/dist/base/api_client.d.ts +7 -7
  9. package/dist/base/api_client.js +36 -36
  10. package/dist/base/resource_client.d.ts +21 -8
  11. package/dist/base/resource_client.js +67 -33
  12. package/dist/base/resource_collection_client.d.ts +12 -4
  13. package/dist/base/resource_collection_client.js +34 -11
  14. package/dist/body_parser.d.ts +3 -4
  15. package/dist/body_parser.js +19 -26
  16. package/dist/bundle.js +27 -22
  17. package/dist/bundle.js.map +1 -1
  18. package/dist/generated/api.d.ts +918 -372
  19. package/dist/generated/schemas.d.ts +889 -871
  20. package/dist/generated/schemas.js +234 -209
  21. package/dist/http_clients/axios.d.ts +60 -0
  22. package/dist/http_clients/axios.js +117 -0
  23. package/dist/http_clients/base.d.ts +331 -0
  24. package/dist/http_clients/base.js +563 -0
  25. package/dist/http_clients/index.d.ts +4 -0
  26. package/dist/http_clients/index.js +2 -0
  27. package/dist/http_compressors/base.d.ts +40 -0
  28. package/dist/http_compressors/base.js +9 -0
  29. package/dist/http_compressors/brotli.d.ts +33 -0
  30. package/dist/http_compressors/brotli.js +39 -0
  31. package/dist/http_compressors/gzip.d.ts +33 -0
  32. package/dist/http_compressors/gzip.js +39 -0
  33. package/dist/http_compressors/index.d.ts +6 -0
  34. package/dist/http_compressors/index.js +2 -0
  35. package/dist/http_compressors/resolve.d.ts +23 -0
  36. package/dist/http_compressors/resolve.js +27 -0
  37. package/dist/index.d.ts +5 -1
  38. package/dist/index.js +4 -1
  39. package/dist/invalid_response_body_error.d.ts +14 -0
  40. package/dist/invalid_response_body_error.js +18 -0
  41. package/dist/models.d.ts +167 -194
  42. package/dist/models.js +6 -4
  43. package/dist/resource_clients/actor.d.ts +74 -34
  44. package/dist/resource_clients/actor.js +87 -56
  45. package/dist/resource_clients/actor_collection.d.ts +18 -12
  46. package/dist/resource_clients/actor_collection.js +11 -3
  47. package/dist/resource_clients/actor_env_var.d.ts +11 -4
  48. package/dist/resource_clients/actor_env_var.js +16 -6
  49. package/dist/resource_clients/actor_env_var_collection.d.ts +9 -4
  50. package/dist/resource_clients/actor_env_var_collection.js +11 -4
  51. package/dist/resource_clients/actor_version.d.ts +20 -5
  52. package/dist/resource_clients/actor_version.js +19 -9
  53. package/dist/resource_clients/actor_version_collection.d.ts +11 -4
  54. package/dist/resource_clients/actor_version_collection.js +13 -4
  55. package/dist/resource_clients/build.d.ts +20 -7
  56. package/dist/resource_clients/build.js +35 -15
  57. package/dist/resource_clients/build_collection.d.ts +7 -5
  58. package/dist/resource_clients/build_collection.js +4 -1
  59. package/dist/resource_clients/dataset.d.ts +81 -36
  60. package/dist/resource_clients/dataset.js +109 -120
  61. package/dist/resource_clients/dataset_collection.d.ts +10 -6
  62. package/dist/resource_clients/dataset_collection.js +13 -2
  63. package/dist/resource_clients/key_value_store.d.ts +101 -23
  64. package/dist/resource_clients/key_value_store.js +121 -45
  65. package/dist/resource_clients/key_value_store_collection.d.ts +10 -6
  66. package/dist/resource_clients/key_value_store_collection.js +13 -2
  67. package/dist/resource_clients/log.d.ts +40 -21
  68. package/dist/resource_clients/log.js +144 -66
  69. package/dist/resource_clients/request_queue.d.ts +76 -31
  70. package/dist/resource_clients/request_queue.js +178 -93
  71. package/dist/resource_clients/request_queue_collection.d.ts +9 -5
  72. package/dist/resource_clients/request_queue_collection.js +9 -3
  73. package/dist/resource_clients/run.d.ts +119 -28
  74. package/dist/resource_clients/run.js +190 -49
  75. package/dist/resource_clients/run_collection.d.ts +5 -3
  76. package/dist/resource_clients/run_collection.js +4 -1
  77. package/dist/resource_clients/schedule.d.ts +16 -7
  78. package/dist/resource_clients/schedule.js +30 -22
  79. package/dist/resource_clients/schedule_collection.d.ts +6 -2
  80. package/dist/resource_clients/schedule_collection.js +9 -3
  81. package/dist/resource_clients/store_collection.d.ts +6 -4
  82. package/dist/resource_clients/store_collection.js +4 -1
  83. package/dist/resource_clients/task.d.ts +36 -13
  84. package/dist/resource_clients/task.js +79 -46
  85. package/dist/resource_clients/task_collection.d.ts +9 -5
  86. package/dist/resource_clients/task_collection.js +9 -3
  87. package/dist/resource_clients/user.d.ts +19 -10
  88. package/dist/resource_clients/user.js +42 -38
  89. package/dist/resource_clients/webhook.d.ts +17 -8
  90. package/dist/resource_clients/webhook.js +30 -22
  91. package/dist/resource_clients/webhook_collection.d.ts +8 -4
  92. package/dist/resource_clients/webhook_collection.js +9 -3
  93. package/dist/resource_clients/webhook_dispatch.d.ts +4 -1
  94. package/dist/resource_clients/webhook_dispatch.js +7 -2
  95. package/dist/resource_clients/webhook_dispatch_collection.d.ts +3 -1
  96. package/dist/resource_clients/webhook_dispatch_collection.js +4 -1
  97. package/dist/response_validation_error.d.ts +2 -1
  98. package/dist/response_validation_error.js +2 -1
  99. package/dist/runtime/node.d.ts +6 -0
  100. package/dist/runtime/node.js +49 -0
  101. package/dist/runtime/types.d.ts +52 -0
  102. package/dist/runtime/types.js +1 -0
  103. package/dist/runtime/web.d.ts +5 -0
  104. package/dist/runtime/web.js +18 -0
  105. package/dist/schemas.d.ts +201 -4
  106. package/dist/schemas.js +19 -4
  107. package/dist/statistics.d.ts +1 -6
  108. package/dist/statistics.js +2 -2
  109. package/dist/timeouts.d.ts +84 -0
  110. package/dist/timeouts.js +37 -0
  111. package/dist/timezones.d.ts +1 -1
  112. package/dist/utils.d.ts +88 -57
  113. package/dist/utils.js +219 -157
  114. package/dist/wait_for_resources.d.ts +8 -0
  115. package/dist/wait_for_resources.js +39 -0
  116. package/package.json +32 -29
  117. package/dist/http_client.d.ts +0 -88
  118. package/dist/http_client.js +0 -255
  119. package/dist/interceptors.d.ts +0 -19
  120. package/dist/interceptors.js +0 -105
@@ -1,25 +1,26 @@
1
1
  import { z } from 'zod';
2
2
  import { MAX_PAYLOAD_SIZE_BYTES, REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION } from '@apify/consts';
3
3
  import log from '@apify/log';
4
- import { MEDIUM_TIMEOUT_MILLIS, ResourceClient, SMALL_TIMEOUT_MILLIS } from '../base/resource_client.js';
4
+ import { ResourceClient } from '../base/resource_client.js';
5
5
  import { ResponseValidationError } from '../response_validation_error.js';
6
6
  import * as schemas from '../schemas.js';
7
- import { anyObjectSchema, cast, catchNotFoundOrThrow, isNonArrayObject, parseArgument, parseDateFields, parseResponse, RequestQueuePaginationIterator, sliceArrayByByteLength, } from '../utils.js';
7
+ import { optionalSignalSchema, optionalTimeoutSchema, timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
8
+ import { anyObjectSchema, catchNotFoundOrThrow, isNonArrayObject, minForLimitParam, parseArgument, parseResponse, RequestQueuePaginationIterator, sleep, splitIntoJsonArrayBatches, utf8ByteLength, } from '../utils.js';
8
9
  const DEFAULT_PARALLEL_BATCH_ADD_REQUESTS = 5;
9
10
  const DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS = 3;
10
11
  const DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS = 500;
11
12
  const DEFAULT_REQUEST_QUEUE_REQUEST_PAGE_LIMIT = 1000;
12
13
  const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
13
- const listHeadOptionsSchema = z.strictObject({ limit: z.number().min(0).optional() });
14
+ const listHeadOptionsSchema = z.strictObject({ limit: z.number().min(0).optional(), ...timeoutOptionsShape });
14
15
  const listAndLockHeadOptionsSchema = z.strictObject({
15
16
  lockSecs: z.number(),
16
17
  limit: z.number().min(0).optional(),
18
+ ...timeoutOptionsShape,
17
19
  });
18
20
  // Predicates, not `z.looseObject` arms: these run over a whole batch, and an object arm would copy
19
21
  // every key of every request. `id` is assigned by the API, so a new request must not carry one.
20
22
  const newRequestSchema = z.custom((value) => isNonArrayObject(value) && value.id === undefined, 'Expected a request object without an `id`');
21
- const forefrontOptionsSchema = z.strictObject({ forefront: z.boolean().optional() });
22
- const batchAddRequestsSchema = z.array(newRequestSchema).min(1).max(REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION);
23
+ const forefrontOptionsSchema = z.strictObject({ forefront: z.boolean().optional(), ...timeoutOptionsShape });
23
24
  const batchAddRequestsWithRetriesSchema = z.array(newRequestSchema).min(1);
24
25
  const optionalBooleanSchema = z.boolean().optional();
25
26
  const optionalNumberSchema = z.number().optional();
@@ -33,18 +34,22 @@ const existingRequestSchema = z.custom((value) => isNonArrayObject(value) && typ
33
34
  const prolongRequestLockOptionsSchema = z.strictObject({
34
35
  lockSecs: z.number(),
35
36
  forefront: z.boolean().optional(),
37
+ ...timeoutOptionsShape,
36
38
  });
37
39
  const requestFilterSchema = z.array(z.enum(['locked', 'pending'])).min(1);
38
40
  const listRequestsOptionsSchema = z.strictObject({
39
41
  limit: z.number().min(0).optional(),
42
+ chunkSize: z.number().positive().optional(),
40
43
  cursor: z.string().optional(),
41
44
  filter: requestFilterSchema.optional(),
45
+ ...timeoutOptionsShape,
42
46
  });
43
47
  const paginateRequestsOptionsSchema = z.strictObject({
44
48
  limit: z.number().min(0).optional(),
45
49
  maxPageLimit: z.number().default(DEFAULT_REQUEST_QUEUE_REQUEST_PAGE_LIMIT),
46
50
  cursor: z.string().optional(),
47
51
  filter: requestFilterSchema.optional(),
52
+ ...timeoutOptionsShape,
48
53
  });
49
54
  /**
50
55
  * Client for managing a specific Request queue.
@@ -77,8 +82,8 @@ const paginateRequestsOptionsSchema = z.strictObject({
77
82
  * @see https://docs.apify.com/platform/storage/request-queue
78
83
  */
79
84
  export class RequestQueueClient extends ResourceClient {
80
- clientKey;
81
- timeoutMillis;
85
+ #clientKey;
86
+ #timeoutSecs;
82
87
  /**
83
88
  * @hidden
84
89
  */
@@ -87,36 +92,57 @@ export class RequestQueueClient extends ResourceClient {
87
92
  resourcePath: 'request-queues',
88
93
  ...options,
89
94
  });
90
- this.clientKey = userOptions.clientKey;
91
- this.timeoutMillis = userOptions.timeoutSecs ? userOptions.timeoutSecs * 1e3 : undefined;
95
+ this.#clientKey = userOptions.clientKey;
96
+ this.#timeoutSecs = userOptions.timeoutSecs;
97
+ }
98
+ /**
99
+ * Picks the timeout of a request: an explicit per-call `timeoutSecs` as is, otherwise the method's default
100
+ * tier, capped at the queue-wide `timeoutSecs` when one was given.
101
+ */
102
+ #resolveTimeout(timeoutSecs, defaultTier) {
103
+ if (timeoutSecs !== undefined)
104
+ return timeoutSecs;
105
+ if (this.#timeoutSecs === undefined)
106
+ return defaultTier;
107
+ const tierSecs = this.httpClient.timeoutMillis[defaultTier] / 1000;
108
+ return tierSecs <= this.#timeoutSecs ? defaultTier : this.#timeoutSecs;
92
109
  }
93
110
  /**
94
111
  * Gets the Request queue object from the Apify API.
95
112
  *
113
+ * @param options - Request options
114
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
96
115
  * @returns The RequestQueue object, or `undefined` if it does not exist
97
116
  * @see https://docs.apify.com/api/v2/request-queue-get
98
117
  */
99
- async get() {
100
- return this._get(schemas.RequestQueue(), {}, SMALL_TIMEOUT_MILLIS);
118
+ async get(options = {}) {
119
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
120
+ return this.getResource(schemas.RequestQueueResource(), {}, this.#resolveTimeout(timeoutSecs, 'short'), signal);
101
121
  }
102
122
  /**
103
123
  * Updates the Request queue with specified fields.
104
124
  *
105
125
  * @param newFields - Fields to update in the Request queue
126
+ * @param options - Request options
127
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
106
128
  * @returns The updated RequestQueue object
107
129
  * @see https://docs.apify.com/api/v2/request-queue-put
108
130
  */
109
- async update(newFields) {
131
+ async update(newFields, options = {}) {
110
132
  parseArgument(newFields, anyObjectSchema);
111
- return this._update(schemas.RequestQueue(), newFields, SMALL_TIMEOUT_MILLIS);
133
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
134
+ return this.updateResource(schemas.RequestQueueResource(), newFields, this.#resolveTimeout(timeoutSecs, 'short'), signal);
112
135
  }
113
136
  /**
114
137
  * Deletes the Request queue.
115
138
  *
139
+ * @param options - Request options
140
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
116
141
  * @see https://docs.apify.com/api/v2/request-queue-delete
117
142
  */
118
- async delete() {
119
- return this._delete(SMALL_TIMEOUT_MILLIS);
143
+ async delete(options = {}) {
144
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
145
+ return this.deleteResource(this.#resolveTimeout(timeoutSecs, 'short'), signal);
120
146
  }
121
147
  /**
122
148
  * Lists requests from the beginning of the queue (head).
@@ -125,18 +151,21 @@ export class RequestQueueClient extends ResourceClient {
125
151
  * inspecting what requests are waiting to be processed.
126
152
  *
127
153
  * @param options - Options for listing (e.g., limit)
154
+ * @param options.limit - Maximum number of requests to return.
155
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
128
156
  * @returns List of requests from the queue head
129
157
  * @see https://docs.apify.com/api/v2/request-queue-head-get
130
158
  */
131
159
  async listHead(options = {}) {
132
160
  const parsed = parseArgument(options, listHeadOptionsSchema, 'RequestQueueClientListHeadOptions');
133
161
  const response = await this.httpClient.call({
134
- url: this._url('head'),
162
+ url: this.buildUrl('head'),
135
163
  method: 'GET',
136
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
137
- params: this._params({
164
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
165
+ signal: parsed.signal,
166
+ params: this.buildParams({
138
167
  limit: parsed.limit,
139
- clientKey: this.clientKey,
168
+ clientKey: this.#clientKey,
140
169
  }),
141
170
  });
142
171
  return parseResponse(response, schemas.RequestQueueHead());
@@ -153,6 +182,7 @@ export class RequestQueueClient extends ResourceClient {
153
182
  * @param options - Lock configuration
154
183
  * @param options.lockSecs - **Required.** Duration in seconds to lock the requests. After this time, the locks expire and requests can be retrieved by other clients.
155
184
  * @param options.limit - Maximum number of requests to return. Default is 25.
185
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
156
186
  * @returns Object containing `items` (locked requests), `queueModifiedAt`, `hadMultipleClients`, and lock information
157
187
  * @see https://docs.apify.com/api/v2/request-queue-head-lock-post
158
188
  *
@@ -177,13 +207,14 @@ export class RequestQueueClient extends ResourceClient {
177
207
  async listAndLockHead(options) {
178
208
  const parsed = parseArgument(options, listAndLockHeadOptionsSchema, 'RequestQueueClientListAndLockHeadOptions');
179
209
  const response = await this.httpClient.call({
180
- url: this._url('head/lock'),
210
+ url: this.buildUrl('head/lock'),
181
211
  method: 'POST',
182
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
183
- params: this._params({
212
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
213
+ signal: parsed.signal,
214
+ params: this.buildParams({
184
215
  limit: parsed.limit,
185
216
  lockSecs: parsed.lockSecs,
186
- clientKey: this.clientKey,
217
+ clientKey: this.#clientKey,
187
218
  }),
188
219
  });
189
220
  return parseResponse(response, schemas.LockedRequestQueueHead());
@@ -204,6 +235,7 @@ export class RequestQueueClient extends ResourceClient {
204
235
  * @param request.payload - HTTP payload for POST/PUT requests (string).
205
236
  * @param options - Additional options
206
237
  * @param options.forefront - If `true`, adds the request to the beginning of the queue. Default is `false` (adds to the end).
238
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
207
239
  * @returns Object with `requestId`, `wasAlreadyPresent`, and `wasAlreadyHandled` flags
208
240
  * @see https://docs.apify.com/api/v2/request-queue-requests-post
209
241
  *
@@ -230,13 +262,14 @@ export class RequestQueueClient extends ResourceClient {
230
262
  parseArgument(request, newRequestSchema);
231
263
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
232
264
  const response = await this.httpClient.call({
233
- url: this._url('requests'),
265
+ url: this.buildUrl('requests'),
234
266
  method: 'POST',
235
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
267
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
268
+ signal: parsed.signal,
236
269
  data: request,
237
- params: this._params({
270
+ params: this.buildParams({
238
271
  forefront: parsed.forefront,
239
- clientKey: this.clientKey,
272
+ clientKey: this.#clientKey,
240
273
  }),
241
274
  });
242
275
  return parseResponse(response, schemas.RequestRegistration());
@@ -246,23 +279,26 @@ export class RequestQueueClient extends ResourceClient {
246
279
  *
247
280
  * @private
248
281
  */
249
- async _batchAddRequests(requests, options = {}) {
250
- parseArgument(requests, batchAddRequestsSchema);
282
+ async addRequestBatch(requests, options = {}) {
251
283
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
252
284
  const response = await this.httpClient.call({
253
- url: this._url('requests/batch'),
285
+ url: this.buildUrl('requests/batch'),
254
286
  method: 'POST',
255
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
256
- data: requests,
257
- params: this._params({
287
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
288
+ signal: parsed.signal,
289
+ // The body is assembled from the requests as `batchAddRequests` serialized them; the explicit content type
290
+ // makes the request interceptor send the string as it is instead of serializing the requests again.
291
+ headers: { 'content-type': 'application/json' },
292
+ data: `[${requests.map(({ json }) => json).join(',')}]`,
293
+ params: this.buildParams({
258
294
  forefront: parsed.forefront,
259
- clientKey: this.clientKey,
295
+ clientKey: this.#clientKey,
260
296
  }),
261
297
  });
262
298
  return parseResponse(response, schemas.BatchAddResult());
263
299
  }
264
- async _batchAddRequestsWithRetries(requests, options = {}) {
265
- const { forefront, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
300
+ async addRequestBatchWithRetries(requests, options = {}) {
301
+ const { forefront, timeoutSecs, signal, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
266
302
  // Keep track of the requests that remain to be processed (in parameter format)
267
303
  let remainingRequests = requests;
268
304
  // Keep track of the requests that have been processed (in api format)
@@ -272,8 +308,10 @@ export class RequestQueueClient extends ResourceClient {
272
308
  let unprocessedRequests = [];
273
309
  for (let i = 0; i < 1 + maxUnprocessedRequestsRetries; i++) {
274
310
  try {
275
- const response = await this._batchAddRequests(remainingRequests, {
311
+ const response = await this.addRequestBatch(remainingRequests, {
276
312
  forefront,
313
+ timeoutSecs,
314
+ signal,
277
315
  });
278
316
  processedRequests.push(...response.processedRequests);
279
317
  unprocessedRequests = response.unprocessedRequests;
@@ -285,13 +323,15 @@ export class RequestQueueClient extends ResourceClient {
285
323
  // Get unique keys of all requests processed so far
286
324
  const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
287
325
  // Requests remaining to be processed are the all that remain
288
- remainingRequests = requests.filter(({ uniqueKey }) => !processedRequestsUniqueKeys.includes(uniqueKey));
326
+ remainingRequests = requests.filter(({ request }) => !processedRequestsUniqueKeys.includes(request.uniqueKey));
289
327
  // Stop if all requests have been processed
290
328
  if (remainingRequests.length === 0) {
291
329
  break;
292
330
  }
293
331
  }
294
332
  catch (err) {
333
+ // An abort rejects the whole call, since the caller asked to stop and not for a partial result.
334
+ signal?.throwIfAborted();
295
335
  // A response the specification does not describe is reported, as everywhere else: the server may well
296
336
  // have added the batch, so calling it unprocessed would hide the mismatch behind a wrong answer.
297
337
  if (err instanceof ResponseValidationError)
@@ -301,18 +341,16 @@ export class RequestQueueClient extends ResourceClient {
301
341
  // This ensures that this method does not throw and keeps the signature.
302
342
  const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
303
343
  unprocessedRequests = requests
304
- .filter(({ uniqueKey }) => !processedRequestsUniqueKeys.includes(uniqueKey))
305
- .map(({ method, uniqueKey, url }) => ({ method, uniqueKey, url }));
344
+ .filter(({ request }) => !processedRequestsUniqueKeys.includes(request.uniqueKey))
345
+ .map(({ request: { method, uniqueKey, url } }) => ({ method, uniqueKey, url }));
306
346
  break;
307
347
  }
308
348
  // Exponential backoff
309
349
  const delayMillis = Math.floor((1 + Math.random()) * 2 ** i * minDelayBetweenUnprocessedRequestsRetriesMillis);
310
- await new Promise((resolve) => {
311
- setTimeout(resolve, delayMillis);
312
- });
350
+ await sleep(delayMillis, signal);
351
+ signal?.throwIfAborted();
313
352
  }
314
- const result = { processedRequests, unprocessedRequests };
315
- return cast(parseDateFields(result));
353
+ return { processedRequests, unprocessedRequests };
316
354
  }
317
355
  /**
318
356
  * Adds multiple requests to the queue in a single operation.
@@ -329,6 +367,7 @@ export class RequestQueueClient extends ResourceClient {
329
367
  * @param options.maxUnprocessedRequestsRetries - Maximum number of retry attempts for rate-limited requests. Default is 3.
330
368
  * @param options.maxParallel - Maximum number of parallel batch API calls. Default is 5.
331
369
  * @param options.minDelayBetweenUnprocessedRequestsRetriesMillis - Minimum delay before retrying rate-limited requests. Default is 500ms.
370
+ * @param options.timeoutSecs - Timeout for each batch API request. Default is `'medium'`.
332
371
  * @returns Object with `processedRequests` (successfully added) and `unprocessedRequests` (failed after all retries)
333
372
  * @see https://docs.apify.com/api/v2/request-queue-requests-batch-post
334
373
  *
@@ -353,21 +392,36 @@ export class RequestQueueClient extends ResourceClient {
353
392
  * @since Added in 2.1.0
354
393
  */
355
394
  async batchAddRequests(requests, options = {}) {
356
- const { forefront, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, maxParallel = DEFAULT_PARALLEL_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
395
+ const { forefront, timeoutSecs, signal, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, maxParallel = DEFAULT_PARALLEL_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
357
396
  parseArgument(requests, batchAddRequestsWithRetriesSchema);
358
397
  parseArgument(forefront, optionalBooleanSchema);
398
+ parseArgument(timeoutSecs, optionalTimeoutSchema);
399
+ parseArgument(signal, optionalSignalSchema);
359
400
  parseArgument(maxUnprocessedRequestsRetries, optionalNumberSchema);
360
401
  parseArgument(maxParallel, optionalNumberSchema);
361
402
  parseArgument(minDelayBetweenUnprocessedRequestsRetriesMillis, optionalNumberSchema);
362
403
  const executingRequests = new Set();
363
404
  const individualResults = [];
364
405
  const payloadSizeLimitBytes = MAX_PAYLOAD_SIZE_BYTES - Math.ceil(MAX_PAYLOAD_SIZE_BYTES * SAFETY_BUFFER_PERCENT);
406
+ // Serialize every request once: the byte lengths decide the batch boundaries, and the same strings are joined
407
+ // into the batch bodies, so nothing is stringified a second time when it is sent.
408
+ const serializedRequests = requests.map((request, index) => {
409
+ const json = JSON.stringify(request);
410
+ const byteLength = utf8ByteLength(json);
411
+ // Two more bytes for the brackets, which even a batch of one request carries.
412
+ if (byteLength + 2 > payloadSizeLimitBytes) {
413
+ throw new Error(`RequestQueueClient.batchAddRequests: The size of the request with index: ${index} ` +
414
+ `exceeds the maximum allowed size (${payloadSizeLimitBytes} bytes).`);
415
+ }
416
+ return { request, json, byteLength };
417
+ });
418
+ const batches = splitIntoJsonArrayBatches(serializedRequests, {
419
+ maxCount: REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION,
420
+ maxByteLength: payloadSizeLimitBytes,
421
+ });
365
422
  // Keep a pool of up to `maxParallel` requests running at once
366
- let i = 0;
367
- while (i < requests.length) {
368
- const slicedRequests = requests.slice(i, i + REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION);
369
- const requestsInBatch = sliceArrayByByteLength(slicedRequests, payloadSizeLimitBytes, i);
370
- const requestPromise = this._batchAddRequestsWithRetries(requestsInBatch, options);
423
+ for (const batch of batches) {
424
+ const requestPromise = this.addRequestBatchWithRetries(batch, options);
371
425
  executingRequests.add(requestPromise);
372
426
  // A rejection reaches the caller through the awaits below; this bookkeeping chain only has to avoid
373
427
  // turning it into an unhandled one of its own.
@@ -378,7 +432,6 @@ export class RequestQueueClient extends ResourceClient {
378
432
  if (executingRequests.size >= maxParallel) {
379
433
  await Promise.race(executingRequests);
380
434
  }
381
- i += requestsInBatch.length;
382
435
  }
383
436
  // Get results from remaining operations
384
437
  await Promise.all(executingRequests);
@@ -399,19 +452,23 @@ export class RequestQueueClient extends ResourceClient {
399
452
  * Requests can be identified by either their ID or unique key.
400
453
  *
401
454
  * @param requests - Array of requests to delete (by id or uniqueKey)
455
+ * @param options - Request options
456
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
402
457
  * @returns Result containing processed and unprocessed requests
403
458
  * @see https://docs.apify.com/api/v2/request-queue-requests-batch-delete
404
459
  * @since Added in 2.3.0
405
460
  */
406
- async batchDeleteRequests(requests) {
461
+ async batchDeleteRequests(requests, options = {}) {
407
462
  parseArgument(requests, batchDeleteRequestsSchema);
463
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
408
464
  const response = await this.httpClient.call({
409
- url: this._url('requests/batch'),
465
+ url: this.buildUrl('requests/batch'),
410
466
  method: 'DELETE',
411
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
467
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
468
+ signal,
412
469
  data: requests,
413
- params: this._params({
414
- clientKey: this.clientKey,
470
+ params: this.buildParams({
471
+ clientKey: this.#clientKey,
415
472
  }),
416
473
  });
417
474
  return parseResponse(response, schemas.BatchDeleteResult());
@@ -420,20 +477,24 @@ export class RequestQueueClient extends ResourceClient {
420
477
  * Gets a specific request from the queue by its ID.
421
478
  *
422
479
  * @param id - Request ID
480
+ * @param options - Request options
481
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
423
482
  * @returns The request object, or `undefined` if not found
424
483
  * @see https://docs.apify.com/api/v2/request-queue-request-get
425
484
  */
426
- async getRequest(id) {
485
+ async getRequest(id, options = {}) {
427
486
  parseArgument(id, requestIdSchema);
487
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
428
488
  const requestOpts = {
429
- url: this._url(['requests', id]),
489
+ url: this.buildUrl(['requests', id]),
430
490
  method: 'GET',
431
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
432
- params: this._params(),
491
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
492
+ signal,
493
+ params: this.buildParams(),
433
494
  };
434
495
  try {
435
496
  const response = await this.httpClient.call(requestOpts);
436
- return parseResponse(response, schemas.Request());
497
+ return parseResponse(response, schemas.RequestResource());
437
498
  }
438
499
  catch (err) {
439
500
  catchNotFoundOrThrow(err);
@@ -445,6 +506,8 @@ export class RequestQueueClient extends ResourceClient {
445
506
  *
446
507
  * @param request - The updated request object (must include id)
447
508
  * @param options - Update options such as whether to move to front
509
+ * @param options.forefront - If `true`, moves the request to the beginning of the queue. Default is `false`.
510
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
448
511
  * @returns Information about the updated request
449
512
  * @see https://docs.apify.com/api/v2/request-queue-request-put
450
513
  */
@@ -452,13 +515,14 @@ export class RequestQueueClient extends ResourceClient {
452
515
  parseArgument(request, existingRequestSchema);
453
516
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
454
517
  const response = await this.httpClient.call({
455
- url: this._url(['requests', request.id]),
518
+ url: this.buildUrl(['requests', request.id]),
456
519
  method: 'PUT',
457
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
520
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
521
+ signal: parsed.signal,
458
522
  data: request,
459
- params: this._params({
523
+ params: this.buildParams({
460
524
  forefront: parsed.forefront,
461
- clientKey: this.clientKey,
525
+ clientKey: this.#clientKey,
462
526
  }),
463
527
  });
464
528
  return parseResponse(response, schemas.RequestRegistration());
@@ -467,15 +531,19 @@ export class RequestQueueClient extends ResourceClient {
467
531
  * Deletes a specific request from the queue.
468
532
  *
469
533
  * @param id - Request ID
534
+ * @param options - Request options
535
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
470
536
  */
471
- async deleteRequest(id) {
537
+ async deleteRequest(id, options = {}) {
472
538
  parseArgument(id, requestIdSchema);
539
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
473
540
  await this.httpClient.call({
474
- url: this._url(['requests', id]),
541
+ url: this.buildUrl(['requests', id]),
475
542
  method: 'DELETE',
476
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
477
- params: this._params({
478
- clientKey: this.clientKey,
543
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
544
+ signal,
545
+ params: this.buildParams({
546
+ clientKey: this.#clientKey,
479
547
  }),
480
548
  });
481
549
  }
@@ -490,6 +558,7 @@ export class RequestQueueClient extends ResourceClient {
490
558
  * @param options - Lock extension options
491
559
  * @param options.lockSecs - **Required.** New lock duration in seconds from now.
492
560
  * @param options.forefront - If `true`, moves the request to the beginning of the queue when the lock expires. Default is `false`.
561
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
493
562
  * @returns Object with new `lockExpiresAt` timestamp
494
563
  * @see https://docs.apify.com/api/v2/request-queue-request-lock-put
495
564
  *
@@ -508,13 +577,14 @@ export class RequestQueueClient extends ResourceClient {
508
577
  parseArgument(id, requestIdSchema);
509
578
  const parsed = parseArgument(options, prolongRequestLockOptionsSchema, 'RequestQueueClientProlongRequestLockOptions');
510
579
  const response = await this.httpClient.call({
511
- url: this._url(['requests', id, 'lock']),
580
+ url: this.buildUrl(['requests', id, 'lock']),
512
581
  method: 'PUT',
513
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
514
- params: this._params({
582
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
583
+ signal: parsed.signal,
584
+ params: this.buildParams({
515
585
  forefront: parsed.forefront,
516
586
  lockSecs: parsed.lockSecs,
517
- clientKey: this.clientKey,
587
+ clientKey: this.#clientKey,
518
588
  }),
519
589
  });
520
590
  return parseResponse(response, schemas.RequestLockInfo());
@@ -527,6 +597,8 @@ export class RequestQueueClient extends ResourceClient {
527
597
  *
528
598
  * @param id - Request ID
529
599
  * @param options - Options such as whether to move to front
600
+ * @param options.forefront - If `true`, moves the request to the beginning of the queue. Default is `false`.
601
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
530
602
  * @see https://docs.apify.com/api/v2/request-queue-request-lock-delete
531
603
  * @since Added in 2.4.1
532
604
  */
@@ -534,12 +606,13 @@ export class RequestQueueClient extends ResourceClient {
534
606
  parseArgument(id, requestIdSchema);
535
607
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientDeleteRequestLockOptions');
536
608
  await this.httpClient.call({
537
- url: this._url(['requests', id, 'lock']),
609
+ url: this.buildUrl(['requests', id, 'lock']),
538
610
  method: 'DELETE',
539
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
540
- params: this._params({
611
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
612
+ signal: parsed.signal,
613
+ params: this.buildParams({
541
614
  forefront: parsed.forefront,
542
- clientKey: this.clientKey,
615
+ clientKey: this.#clientKey,
543
616
  }),
544
617
  });
545
618
  }
@@ -550,26 +623,33 @@ export class RequestQueueClient extends ResourceClient {
550
623
  * queue contents.
551
624
  *
552
625
  * @param options - Pagination options
626
+ * @param options.chunkSize - Maximum number of requests fetched per API call when iterating across pages. `limit`
627
+ * then caps the total across all pages.
628
+ * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
553
629
  * @returns List of requests with pagination information
554
630
  * @see https://docs.apify.com/api/v2/request-queue-requests-get
555
631
  * @since Added in 2.5.1
556
632
  */
557
633
  listRequests(options = {}) {
558
- const parsed = parseArgument(options, listRequestsOptionsSchema, 'RequestQueueClientListRequestsOptions');
634
+ // `chunkSize` only sizes the page requests, and `timeoutSecs` and `signal` apply to every one of them; none of
635
+ // them is an API parameter, so they must not reach the query string.
636
+ const { chunkSize, timeoutSecs, signal, ...parsed } = parseArgument(options, listRequestsOptionsSchema, 'RequestQueueClientListRequestsOptions');
637
+ const pageTimeout = this.#resolveTimeout(timeoutSecs, 'medium');
559
638
  const getPaginatedList = async (rqListOptions = {}) => {
560
639
  const response = await this.httpClient.call({
561
- url: this._url('requests'),
640
+ url: this.buildUrl('requests'),
562
641
  method: 'GET',
563
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
564
- params: this._params({
642
+ timeoutSecs: pageTimeout,
643
+ signal,
644
+ params: this.buildParams({
565
645
  ...rqListOptions,
566
646
  filter: rqListOptions.filter ? rqListOptions.filter.join(',') : undefined,
567
- clientKey: this.clientKey,
647
+ clientKey: this.#clientKey,
568
648
  }),
569
649
  });
570
650
  return parseResponse(response, schemas.ListOfRequests());
571
651
  };
572
- const paginatedListPromise = getPaginatedList(parsed);
652
+ const paginatedListPromise = getPaginatedList({ ...parsed, limit: minForLimitParam(parsed.limit, chunkSize) });
573
653
  async function* asyncGenerator() {
574
654
  let currentPage = await paginatedListPromise;
575
655
  yield* currentPage.items;
@@ -582,7 +662,7 @@ export class RequestQueueClient extends ResourceClient {
582
662
  ) {
583
663
  const newOptions = {
584
664
  ...parsed,
585
- limit: remainingItems,
665
+ limit: minForLimitParam(remainingItems, chunkSize),
586
666
  cursor: currentPage.nextCursor,
587
667
  };
588
668
  currentPage = await getPaginatedList(newOptions);
@@ -602,17 +682,21 @@ export class RequestQueueClient extends ResourceClient {
602
682
  * This is useful for releasing all locks at once, for example when shutting down
603
683
  * a crawler gracefully.
604
684
  *
685
+ * @param options - Request options
686
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
605
687
  * @returns Number of requests that were unlocked
606
688
  * @see https://docs.apify.com/api/v2/request-queue-requests-unlock-post
607
689
  * @since Added in 2.12.5
608
690
  */
609
- async unlockRequests() {
691
+ async unlockRequests(options = {}) {
692
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
610
693
  const response = await this.httpClient.call({
611
- url: this._url('requests/unlock'),
694
+ url: this.buildUrl('requests/unlock'),
612
695
  method: 'POST',
613
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
614
- params: this._params({
615
- clientKey: this.clientKey,
696
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'long'),
697
+ signal,
698
+ params: this.buildParams({
699
+ clientKey: this.#clientKey,
616
700
  }),
617
701
  });
618
702
  return parseResponse(response, schemas.UnlockRequestsResult());
@@ -624,6 +708,7 @@ export class RequestQueueClient extends ResourceClient {
624
708
  * automatically handling pagination behind the scenes.
625
709
  *
626
710
  * @param options - Pagination options
711
+ * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
627
712
  * @returns An async iterable of request pages
628
713
  * @see https://docs.apify.com/api/v2/request-queue-requests-get
629
714
  *
@@ -636,9 +721,9 @@ export class RequestQueueClient extends ResourceClient {
636
721
  * @since Added in 2.5.1
637
722
  */
638
723
  paginateRequests(options = {}) {
639
- const { limit, cursor, filter, maxPageLimit } = parseArgument(options, paginateRequestsOptionsSchema, 'RequestQueueClientPaginateRequestsOptions');
724
+ const { limit, cursor, filter, maxPageLimit, timeoutSecs, signal } = parseArgument(options, paginateRequestsOptionsSchema, 'RequestQueueClientPaginateRequestsOptions');
640
725
  return new RequestQueuePaginationIterator({
641
- getPage: async (pageOptions) => this.listRequests({ ...pageOptions, filter }),
726
+ getPage: async (pageOptions) => this.listRequests({ ...pageOptions, filter, timeoutSecs, signal }),
642
727
  limit,
643
728
  cursor,
644
729
  maxPageLimit,