apify-client 3.0.0-beta.8 → 3.0.0-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/README.md +10 -2
  2. package/dist/apify_api_error.d.ts +11 -26
  3. package/dist/apify_api_error.js +31 -22
  4. package/dist/apify_client.d.ts +125 -10
  5. package/dist/apify_client.js +133 -48
  6. package/dist/base/api_client.d.ts +7 -7
  7. package/dist/base/api_client.js +36 -24
  8. package/dist/base/resource_client.d.ts +21 -8
  9. package/dist/base/resource_client.js +67 -33
  10. package/dist/base/resource_collection_client.d.ts +12 -4
  11. package/dist/base/resource_collection_client.js +34 -11
  12. package/dist/body_parser.d.ts +3 -4
  13. package/dist/body_parser.js +19 -26
  14. package/dist/bundle.js +27 -22
  15. package/dist/bundle.js.map +1 -1
  16. package/dist/generated/api.d.ts +785 -290
  17. package/dist/generated/schemas.d.ts +720 -747
  18. package/dist/generated/schemas.js +166 -162
  19. package/dist/http_clients/axios.d.ts +60 -0
  20. package/dist/http_clients/axios.js +117 -0
  21. package/dist/http_clients/base.d.ts +331 -0
  22. package/dist/http_clients/base.js +563 -0
  23. package/dist/http_clients/index.d.ts +4 -0
  24. package/dist/http_clients/index.js +2 -0
  25. package/dist/http_compressors/base.d.ts +40 -0
  26. package/dist/http_compressors/base.js +9 -0
  27. package/dist/http_compressors/brotli.d.ts +33 -0
  28. package/dist/http_compressors/brotli.js +39 -0
  29. package/dist/http_compressors/gzip.d.ts +33 -0
  30. package/dist/http_compressors/gzip.js +39 -0
  31. package/dist/http_compressors/index.d.ts +6 -0
  32. package/dist/http_compressors/index.js +2 -0
  33. package/dist/http_compressors/resolve.d.ts +23 -0
  34. package/dist/http_compressors/resolve.js +27 -0
  35. package/dist/index.d.ts +4 -1
  36. package/dist/index.js +3 -1
  37. package/dist/invalid_response_body_error.d.ts +13 -0
  38. package/dist/invalid_response_body_error.js +17 -0
  39. package/dist/models.d.ts +162 -194
  40. package/dist/models.js +6 -4
  41. package/dist/resource_clients/actor.d.ts +74 -34
  42. package/dist/resource_clients/actor.js +87 -56
  43. package/dist/resource_clients/actor_collection.d.ts +18 -12
  44. package/dist/resource_clients/actor_collection.js +11 -3
  45. package/dist/resource_clients/actor_env_var.d.ts +11 -4
  46. package/dist/resource_clients/actor_env_var.js +16 -6
  47. package/dist/resource_clients/actor_env_var_collection.d.ts +9 -4
  48. package/dist/resource_clients/actor_env_var_collection.js +11 -4
  49. package/dist/resource_clients/actor_version.d.ts +20 -5
  50. package/dist/resource_clients/actor_version.js +19 -9
  51. package/dist/resource_clients/actor_version_collection.d.ts +11 -4
  52. package/dist/resource_clients/actor_version_collection.js +13 -4
  53. package/dist/resource_clients/build.d.ts +20 -7
  54. package/dist/resource_clients/build.js +35 -15
  55. package/dist/resource_clients/build_collection.d.ts +7 -5
  56. package/dist/resource_clients/build_collection.js +4 -1
  57. package/dist/resource_clients/dataset.d.ts +53 -35
  58. package/dist/resource_clients/dataset.js +105 -80
  59. package/dist/resource_clients/dataset_collection.d.ts +10 -6
  60. package/dist/resource_clients/dataset_collection.js +13 -2
  61. package/dist/resource_clients/key_value_store.d.ts +62 -20
  62. package/dist/resource_clients/key_value_store.js +78 -40
  63. package/dist/resource_clients/key_value_store_collection.d.ts +10 -6
  64. package/dist/resource_clients/key_value_store_collection.js +13 -2
  65. package/dist/resource_clients/log.d.ts +13 -20
  66. package/dist/resource_clients/log.js +141 -59
  67. package/dist/resource_clients/request_queue.d.ts +68 -31
  68. package/dist/resource_clients/request_queue.js +173 -91
  69. package/dist/resource_clients/request_queue_collection.d.ts +9 -5
  70. package/dist/resource_clients/request_queue_collection.js +9 -3
  71. package/dist/resource_clients/run.d.ts +119 -28
  72. package/dist/resource_clients/run.js +190 -49
  73. package/dist/resource_clients/run_collection.d.ts +5 -3
  74. package/dist/resource_clients/run_collection.js +4 -1
  75. package/dist/resource_clients/schedule.d.ts +16 -7
  76. package/dist/resource_clients/schedule.js +30 -22
  77. package/dist/resource_clients/schedule_collection.d.ts +6 -2
  78. package/dist/resource_clients/schedule_collection.js +9 -3
  79. package/dist/resource_clients/store_collection.d.ts +6 -4
  80. package/dist/resource_clients/store_collection.js +4 -1
  81. package/dist/resource_clients/task.d.ts +36 -13
  82. package/dist/resource_clients/task.js +79 -46
  83. package/dist/resource_clients/task_collection.d.ts +9 -5
  84. package/dist/resource_clients/task_collection.js +9 -3
  85. package/dist/resource_clients/user.d.ts +19 -10
  86. package/dist/resource_clients/user.js +42 -38
  87. package/dist/resource_clients/webhook.d.ts +17 -8
  88. package/dist/resource_clients/webhook.js +30 -22
  89. package/dist/resource_clients/webhook_collection.d.ts +8 -4
  90. package/dist/resource_clients/webhook_collection.js +9 -3
  91. package/dist/resource_clients/webhook_dispatch.d.ts +4 -1
  92. package/dist/resource_clients/webhook_dispatch.js +7 -2
  93. package/dist/resource_clients/webhook_dispatch_collection.d.ts +3 -1
  94. package/dist/resource_clients/webhook_dispatch_collection.js +4 -1
  95. package/dist/runtime/node.d.ts +6 -0
  96. package/dist/runtime/node.js +49 -0
  97. package/dist/runtime/types.d.ts +52 -0
  98. package/dist/runtime/types.js +1 -0
  99. package/dist/runtime/web.d.ts +5 -0
  100. package/dist/runtime/web.js +18 -0
  101. package/dist/schemas.d.ts +1 -1
  102. package/dist/schemas.js +1 -1
  103. package/dist/statistics.d.ts +1 -6
  104. package/dist/statistics.js +2 -2
  105. package/dist/timeouts.d.ts +84 -0
  106. package/dist/timeouts.js +37 -0
  107. package/dist/timezones.d.ts +1 -1
  108. package/dist/utils.d.ts +82 -45
  109. package/dist/utils.js +197 -146
  110. package/dist/wait_for_resources.d.ts +8 -0
  111. package/dist/wait_for_resources.js +39 -0
  112. package/package.json +32 -29
  113. package/dist/http_client.d.ts +0 -88
  114. package/dist/http_client.js +0 -255
  115. package/dist/interceptors.d.ts +0 -19
  116. package/dist/interceptors.js +0 -105
@@ -1,25 +1,26 @@
1
1
  import { z } from 'zod';
2
2
  import { MAX_PAYLOAD_SIZE_BYTES, REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION } from '@apify/consts';
3
3
  import log from '@apify/log';
4
- import { MEDIUM_TIMEOUT_MILLIS, ResourceClient, SMALL_TIMEOUT_MILLIS } from '../base/resource_client.js';
4
+ import { ResourceClient } from '../base/resource_client.js';
5
5
  import { ResponseValidationError } from '../response_validation_error.js';
6
6
  import * as schemas from '../schemas.js';
7
- import { anyObjectSchema, cast, catchNotFoundOrThrow, isNonArrayObject, parseArgument, parseDateFields, parseResponse, RequestQueuePaginationIterator, sliceArrayByByteLength, } from '../utils.js';
7
+ import { optionalSignalSchema, optionalTimeoutSchema, timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
8
+ import { anyObjectSchema, catchNotFoundOrThrow, isNonArrayObject, parseArgument, parseResponse, RequestQueuePaginationIterator, sleep, splitIntoJsonArrayBatches, utf8ByteLength, } from '../utils.js';
8
9
  const DEFAULT_PARALLEL_BATCH_ADD_REQUESTS = 5;
9
10
  const DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS = 3;
10
11
  const DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS = 500;
11
12
  const DEFAULT_REQUEST_QUEUE_REQUEST_PAGE_LIMIT = 1000;
12
13
  const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
13
- const listHeadOptionsSchema = z.strictObject({ limit: z.number().min(0).optional() });
14
+ const listHeadOptionsSchema = z.strictObject({ limit: z.number().min(0).optional(), ...timeoutOptionsShape });
14
15
  const listAndLockHeadOptionsSchema = z.strictObject({
15
16
  lockSecs: z.number(),
16
17
  limit: z.number().min(0).optional(),
18
+ ...timeoutOptionsShape,
17
19
  });
18
20
  // Predicates, not `z.looseObject` arms: these run over a whole batch, and an object arm would copy
19
21
  // every key of every request. `id` is assigned by the API, so a new request must not carry one.
20
22
  const newRequestSchema = z.custom((value) => isNonArrayObject(value) && value.id === undefined, 'Expected a request object without an `id`');
21
- const forefrontOptionsSchema = z.strictObject({ forefront: z.boolean().optional() });
22
- const batchAddRequestsSchema = z.array(newRequestSchema).min(1).max(REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION);
23
+ const forefrontOptionsSchema = z.strictObject({ forefront: z.boolean().optional(), ...timeoutOptionsShape });
23
24
  const batchAddRequestsWithRetriesSchema = z.array(newRequestSchema).min(1);
24
25
  const optionalBooleanSchema = z.boolean().optional();
25
26
  const optionalNumberSchema = z.number().optional();
@@ -33,18 +34,21 @@ const existingRequestSchema = z.custom((value) => isNonArrayObject(value) && typ
33
34
  const prolongRequestLockOptionsSchema = z.strictObject({
34
35
  lockSecs: z.number(),
35
36
  forefront: z.boolean().optional(),
37
+ ...timeoutOptionsShape,
36
38
  });
37
39
  const requestFilterSchema = z.array(z.enum(['locked', 'pending'])).min(1);
38
40
  const listRequestsOptionsSchema = z.strictObject({
39
41
  limit: z.number().min(0).optional(),
40
42
  cursor: z.string().optional(),
41
43
  filter: requestFilterSchema.optional(),
44
+ ...timeoutOptionsShape,
42
45
  });
43
46
  const paginateRequestsOptionsSchema = z.strictObject({
44
47
  limit: z.number().min(0).optional(),
45
48
  maxPageLimit: z.number().default(DEFAULT_REQUEST_QUEUE_REQUEST_PAGE_LIMIT),
46
49
  cursor: z.string().optional(),
47
50
  filter: requestFilterSchema.optional(),
51
+ ...timeoutOptionsShape,
48
52
  });
49
53
  /**
50
54
  * Client for managing a specific Request queue.
@@ -77,8 +81,8 @@ const paginateRequestsOptionsSchema = z.strictObject({
77
81
  * @see https://docs.apify.com/platform/storage/request-queue
78
82
  */
79
83
  export class RequestQueueClient extends ResourceClient {
80
- clientKey;
81
- timeoutMillis;
84
+ #clientKey;
85
+ #timeoutSecs;
82
86
  /**
83
87
  * @hidden
84
88
  */
@@ -87,36 +91,57 @@ export class RequestQueueClient extends ResourceClient {
87
91
  resourcePath: 'request-queues',
88
92
  ...options,
89
93
  });
90
- this.clientKey = userOptions.clientKey;
91
- this.timeoutMillis = userOptions.timeoutSecs ? userOptions.timeoutSecs * 1e3 : undefined;
94
+ this.#clientKey = userOptions.clientKey;
95
+ this.#timeoutSecs = userOptions.timeoutSecs;
96
+ }
97
+ /**
98
+ * Picks the timeout of a request: an explicit per-call `timeoutSecs` as is, otherwise the method's default
99
+ * tier, capped at the queue-wide `timeoutSecs` when one was given.
100
+ */
101
+ #resolveTimeout(timeoutSecs, defaultTier) {
102
+ if (timeoutSecs !== undefined)
103
+ return timeoutSecs;
104
+ if (this.#timeoutSecs === undefined)
105
+ return defaultTier;
106
+ const tierSecs = this.httpClient.timeoutMillis[defaultTier] / 1000;
107
+ return tierSecs <= this.#timeoutSecs ? defaultTier : this.#timeoutSecs;
92
108
  }
93
109
  /**
94
110
  * Gets the Request queue object from the Apify API.
95
111
  *
112
+ * @param options - Request options
113
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
96
114
  * @returns The RequestQueue object, or `undefined` if it does not exist
97
115
  * @see https://docs.apify.com/api/v2/request-queue-get
98
116
  */
99
- async get() {
100
- return this._get(schemas.RequestQueue(), {}, SMALL_TIMEOUT_MILLIS);
117
+ async get(options = {}) {
118
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
119
+ return this.getResource(schemas.RequestQueueResource(), {}, this.#resolveTimeout(timeoutSecs, 'short'), signal);
101
120
  }
102
121
  /**
103
122
  * Updates the Request queue with specified fields.
104
123
  *
105
124
  * @param newFields - Fields to update in the Request queue
125
+ * @param options - Request options
126
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
106
127
  * @returns The updated RequestQueue object
107
128
  * @see https://docs.apify.com/api/v2/request-queue-put
108
129
  */
109
- async update(newFields) {
130
+ async update(newFields, options = {}) {
110
131
  parseArgument(newFields, anyObjectSchema);
111
- return this._update(schemas.RequestQueue(), newFields, SMALL_TIMEOUT_MILLIS);
132
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
133
+ return this.updateResource(schemas.RequestQueueResource(), newFields, this.#resolveTimeout(timeoutSecs, 'short'), signal);
112
134
  }
113
135
  /**
114
136
  * Deletes the Request queue.
115
137
  *
138
+ * @param options - Request options
139
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
116
140
  * @see https://docs.apify.com/api/v2/request-queue-delete
117
141
  */
118
- async delete() {
119
- return this._delete(SMALL_TIMEOUT_MILLIS);
142
+ async delete(options = {}) {
143
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
144
+ return this.deleteResource(this.#resolveTimeout(timeoutSecs, 'short'), signal);
120
145
  }
121
146
  /**
122
147
  * Lists requests from the beginning of the queue (head).
@@ -125,18 +150,21 @@ export class RequestQueueClient extends ResourceClient {
125
150
  * inspecting what requests are waiting to be processed.
126
151
  *
127
152
  * @param options - Options for listing (e.g., limit)
153
+ * @param options.limit - Maximum number of requests to return.
154
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
128
155
  * @returns List of requests from the queue head
129
156
  * @see https://docs.apify.com/api/v2/request-queue-head-get
130
157
  */
131
158
  async listHead(options = {}) {
132
159
  const parsed = parseArgument(options, listHeadOptionsSchema, 'RequestQueueClientListHeadOptions');
133
160
  const response = await this.httpClient.call({
134
- url: this._url('head'),
161
+ url: this.buildUrl('head'),
135
162
  method: 'GET',
136
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
137
- params: this._params({
163
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
164
+ signal: parsed.signal,
165
+ params: this.buildParams({
138
166
  limit: parsed.limit,
139
- clientKey: this.clientKey,
167
+ clientKey: this.#clientKey,
140
168
  }),
141
169
  });
142
170
  return parseResponse(response, schemas.RequestQueueHead());
@@ -153,6 +181,7 @@ export class RequestQueueClient extends ResourceClient {
153
181
  * @param options - Lock configuration
154
182
  * @param options.lockSecs - **Required.** Duration in seconds to lock the requests. After this time, the locks expire and requests can be retrieved by other clients.
155
183
  * @param options.limit - Maximum number of requests to return. Default is 25.
184
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
156
185
  * @returns Object containing `items` (locked requests), `queueModifiedAt`, `hadMultipleClients`, and lock information
157
186
  * @see https://docs.apify.com/api/v2/request-queue-head-lock-post
158
187
  *
@@ -177,13 +206,14 @@ export class RequestQueueClient extends ResourceClient {
177
206
  async listAndLockHead(options) {
178
207
  const parsed = parseArgument(options, listAndLockHeadOptionsSchema, 'RequestQueueClientListAndLockHeadOptions');
179
208
  const response = await this.httpClient.call({
180
- url: this._url('head/lock'),
209
+ url: this.buildUrl('head/lock'),
181
210
  method: 'POST',
182
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
183
- params: this._params({
211
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
212
+ signal: parsed.signal,
213
+ params: this.buildParams({
184
214
  limit: parsed.limit,
185
215
  lockSecs: parsed.lockSecs,
186
- clientKey: this.clientKey,
216
+ clientKey: this.#clientKey,
187
217
  }),
188
218
  });
189
219
  return parseResponse(response, schemas.LockedRequestQueueHead());
@@ -204,6 +234,7 @@ export class RequestQueueClient extends ResourceClient {
204
234
  * @param request.payload - HTTP payload for POST/PUT requests (string).
205
235
  * @param options - Additional options
206
236
  * @param options.forefront - If `true`, adds the request to the beginning of the queue. Default is `false` (adds to the end).
237
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
207
238
  * @returns Object with `requestId`, `wasAlreadyPresent`, and `wasAlreadyHandled` flags
208
239
  * @see https://docs.apify.com/api/v2/request-queue-requests-post
209
240
  *
@@ -230,13 +261,14 @@ export class RequestQueueClient extends ResourceClient {
230
261
  parseArgument(request, newRequestSchema);
231
262
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
232
263
  const response = await this.httpClient.call({
233
- url: this._url('requests'),
264
+ url: this.buildUrl('requests'),
234
265
  method: 'POST',
235
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
266
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
267
+ signal: parsed.signal,
236
268
  data: request,
237
- params: this._params({
269
+ params: this.buildParams({
238
270
  forefront: parsed.forefront,
239
- clientKey: this.clientKey,
271
+ clientKey: this.#clientKey,
240
272
  }),
241
273
  });
242
274
  return parseResponse(response, schemas.RequestRegistration());
@@ -246,23 +278,26 @@ export class RequestQueueClient extends ResourceClient {
246
278
  *
247
279
  * @private
248
280
  */
249
- async _batchAddRequests(requests, options = {}) {
250
- parseArgument(requests, batchAddRequestsSchema);
281
+ async addRequestBatch(requests, options = {}) {
251
282
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
252
283
  const response = await this.httpClient.call({
253
- url: this._url('requests/batch'),
284
+ url: this.buildUrl('requests/batch'),
254
285
  method: 'POST',
255
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
256
- data: requests,
257
- params: this._params({
286
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
287
+ signal: parsed.signal,
288
+ // The body is assembled from the requests as `batchAddRequests` serialized them; the explicit content type
289
+ // makes the request interceptor send the string as it is instead of serializing the requests again.
290
+ headers: { 'content-type': 'application/json' },
291
+ data: `[${requests.map(({ json }) => json).join(',')}]`,
292
+ params: this.buildParams({
258
293
  forefront: parsed.forefront,
259
- clientKey: this.clientKey,
294
+ clientKey: this.#clientKey,
260
295
  }),
261
296
  });
262
297
  return parseResponse(response, schemas.BatchAddResult());
263
298
  }
264
- async _batchAddRequestsWithRetries(requests, options = {}) {
265
- const { forefront, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
299
+ async addRequestBatchWithRetries(requests, options = {}) {
300
+ const { forefront, timeoutSecs, signal, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
266
301
  // Keep track of the requests that remain to be processed (in parameter format)
267
302
  let remainingRequests = requests;
268
303
  // Keep track of the requests that have been processed (in api format)
@@ -272,8 +307,10 @@ export class RequestQueueClient extends ResourceClient {
272
307
  let unprocessedRequests = [];
273
308
  for (let i = 0; i < 1 + maxUnprocessedRequestsRetries; i++) {
274
309
  try {
275
- const response = await this._batchAddRequests(remainingRequests, {
310
+ const response = await this.addRequestBatch(remainingRequests, {
276
311
  forefront,
312
+ timeoutSecs,
313
+ signal,
277
314
  });
278
315
  processedRequests.push(...response.processedRequests);
279
316
  unprocessedRequests = response.unprocessedRequests;
@@ -285,13 +322,15 @@ export class RequestQueueClient extends ResourceClient {
285
322
  // Get unique keys of all requests processed so far
286
323
  const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
287
324
  // Requests remaining to be processed are the all that remain
288
- remainingRequests = requests.filter(({ uniqueKey }) => !processedRequestsUniqueKeys.includes(uniqueKey));
325
+ remainingRequests = requests.filter(({ request }) => !processedRequestsUniqueKeys.includes(request.uniqueKey));
289
326
  // Stop if all requests have been processed
290
327
  if (remainingRequests.length === 0) {
291
328
  break;
292
329
  }
293
330
  }
294
331
  catch (err) {
332
+ // An abort rejects the whole call, since the caller asked to stop and not for a partial result.
333
+ signal?.throwIfAborted();
295
334
  // A response the specification does not describe is reported, as everywhere else: the server may well
296
335
  // have added the batch, so calling it unprocessed would hide the mismatch behind a wrong answer.
297
336
  if (err instanceof ResponseValidationError)
@@ -301,18 +340,16 @@ export class RequestQueueClient extends ResourceClient {
301
340
  // This ensures that this method does not throw and keeps the signature.
302
341
  const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
303
342
  unprocessedRequests = requests
304
- .filter(({ uniqueKey }) => !processedRequestsUniqueKeys.includes(uniqueKey))
305
- .map(({ method, uniqueKey, url }) => ({ method, uniqueKey, url }));
343
+ .filter(({ request }) => !processedRequestsUniqueKeys.includes(request.uniqueKey))
344
+ .map(({ request: { method, uniqueKey, url } }) => ({ method, uniqueKey, url }));
306
345
  break;
307
346
  }
308
347
  // Exponential backoff
309
348
  const delayMillis = Math.floor((1 + Math.random()) * 2 ** i * minDelayBetweenUnprocessedRequestsRetriesMillis);
310
- await new Promise((resolve) => {
311
- setTimeout(resolve, delayMillis);
312
- });
349
+ await sleep(delayMillis, signal);
350
+ signal?.throwIfAborted();
313
351
  }
314
- const result = { processedRequests, unprocessedRequests };
315
- return cast(parseDateFields(result));
352
+ return { processedRequests, unprocessedRequests };
316
353
  }
317
354
  /**
318
355
  * Adds multiple requests to the queue in a single operation.
@@ -329,6 +366,7 @@ export class RequestQueueClient extends ResourceClient {
329
366
  * @param options.maxUnprocessedRequestsRetries - Maximum number of retry attempts for rate-limited requests. Default is 3.
330
367
  * @param options.maxParallel - Maximum number of parallel batch API calls. Default is 5.
331
368
  * @param options.minDelayBetweenUnprocessedRequestsRetriesMillis - Minimum delay before retrying rate-limited requests. Default is 500ms.
369
+ * @param options.timeoutSecs - Timeout for each batch API request. Default is `'medium'`.
332
370
  * @returns Object with `processedRequests` (successfully added) and `unprocessedRequests` (failed after all retries)
333
371
  * @see https://docs.apify.com/api/v2/request-queue-requests-batch-post
334
372
  *
@@ -353,21 +391,36 @@ export class RequestQueueClient extends ResourceClient {
353
391
  * @since Added in 2.1.0
354
392
  */
355
393
  async batchAddRequests(requests, options = {}) {
356
- const { forefront, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, maxParallel = DEFAULT_PARALLEL_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
394
+ const { forefront, timeoutSecs, signal, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, maxParallel = DEFAULT_PARALLEL_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
357
395
  parseArgument(requests, batchAddRequestsWithRetriesSchema);
358
396
  parseArgument(forefront, optionalBooleanSchema);
397
+ parseArgument(timeoutSecs, optionalTimeoutSchema);
398
+ parseArgument(signal, optionalSignalSchema);
359
399
  parseArgument(maxUnprocessedRequestsRetries, optionalNumberSchema);
360
400
  parseArgument(maxParallel, optionalNumberSchema);
361
401
  parseArgument(minDelayBetweenUnprocessedRequestsRetriesMillis, optionalNumberSchema);
362
402
  const executingRequests = new Set();
363
403
  const individualResults = [];
364
404
  const payloadSizeLimitBytes = MAX_PAYLOAD_SIZE_BYTES - Math.ceil(MAX_PAYLOAD_SIZE_BYTES * SAFETY_BUFFER_PERCENT);
405
+ // Serialize every request once: the byte lengths decide the batch boundaries, and the same strings are joined
406
+ // into the batch bodies, so nothing is stringified a second time when it is sent.
407
+ const serializedRequests = requests.map((request, index) => {
408
+ const json = JSON.stringify(request);
409
+ const byteLength = utf8ByteLength(json);
410
+ // Two more bytes for the brackets, which even a batch of one request carries.
411
+ if (byteLength + 2 > payloadSizeLimitBytes) {
412
+ throw new Error(`RequestQueueClient.batchAddRequests: The size of the request with index: ${index} ` +
413
+ `exceeds the maximum allowed size (${payloadSizeLimitBytes} bytes).`);
414
+ }
415
+ return { request, json, byteLength };
416
+ });
417
+ const batches = splitIntoJsonArrayBatches(serializedRequests, {
418
+ maxCount: REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION,
419
+ maxByteLength: payloadSizeLimitBytes,
420
+ });
365
421
  // Keep a pool of up to `maxParallel` requests running at once
366
- let i = 0;
367
- while (i < requests.length) {
368
- const slicedRequests = requests.slice(i, i + REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION);
369
- const requestsInBatch = sliceArrayByByteLength(slicedRequests, payloadSizeLimitBytes, i);
370
- const requestPromise = this._batchAddRequestsWithRetries(requestsInBatch, options);
422
+ for (const batch of batches) {
423
+ const requestPromise = this.addRequestBatchWithRetries(batch, options);
371
424
  executingRequests.add(requestPromise);
372
425
  // A rejection reaches the caller through the awaits below; this bookkeeping chain only has to avoid
373
426
  // turning it into an unhandled one of its own.
@@ -378,7 +431,6 @@ export class RequestQueueClient extends ResourceClient {
378
431
  if (executingRequests.size >= maxParallel) {
379
432
  await Promise.race(executingRequests);
380
433
  }
381
- i += requestsInBatch.length;
382
434
  }
383
435
  // Get results from remaining operations
384
436
  await Promise.all(executingRequests);
@@ -399,19 +451,23 @@ export class RequestQueueClient extends ResourceClient {
399
451
  * Requests can be identified by either their ID or unique key.
400
452
  *
401
453
  * @param requests - Array of requests to delete (by id or uniqueKey)
454
+ * @param options - Request options
455
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
402
456
  * @returns Result containing processed and unprocessed requests
403
457
  * @see https://docs.apify.com/api/v2/request-queue-requests-batch-delete
404
458
  * @since Added in 2.3.0
405
459
  */
406
- async batchDeleteRequests(requests) {
460
+ async batchDeleteRequests(requests, options = {}) {
407
461
  parseArgument(requests, batchDeleteRequestsSchema);
462
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
408
463
  const response = await this.httpClient.call({
409
- url: this._url('requests/batch'),
464
+ url: this.buildUrl('requests/batch'),
410
465
  method: 'DELETE',
411
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
466
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
467
+ signal,
412
468
  data: requests,
413
- params: this._params({
414
- clientKey: this.clientKey,
469
+ params: this.buildParams({
470
+ clientKey: this.#clientKey,
415
471
  }),
416
472
  });
417
473
  return parseResponse(response, schemas.BatchDeleteResult());
@@ -420,20 +476,24 @@ export class RequestQueueClient extends ResourceClient {
420
476
  * Gets a specific request from the queue by its ID.
421
477
  *
422
478
  * @param id - Request ID
479
+ * @param options - Request options
480
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
423
481
  * @returns The request object, or `undefined` if not found
424
482
  * @see https://docs.apify.com/api/v2/request-queue-request-get
425
483
  */
426
- async getRequest(id) {
484
+ async getRequest(id, options = {}) {
427
485
  parseArgument(id, requestIdSchema);
486
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
428
487
  const requestOpts = {
429
- url: this._url(['requests', id]),
488
+ url: this.buildUrl(['requests', id]),
430
489
  method: 'GET',
431
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
432
- params: this._params(),
490
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
491
+ signal,
492
+ params: this.buildParams(),
433
493
  };
434
494
  try {
435
495
  const response = await this.httpClient.call(requestOpts);
436
- return parseResponse(response, schemas.Request());
496
+ return parseResponse(response, schemas.RequestResource());
437
497
  }
438
498
  catch (err) {
439
499
  catchNotFoundOrThrow(err);
@@ -445,6 +505,8 @@ export class RequestQueueClient extends ResourceClient {
445
505
  *
446
506
  * @param request - The updated request object (must include id)
447
507
  * @param options - Update options such as whether to move to front
508
+ * @param options.forefront - If `true`, moves the request to the beginning of the queue. Default is `false`.
509
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
448
510
  * @returns Information about the updated request
449
511
  * @see https://docs.apify.com/api/v2/request-queue-request-put
450
512
  */
@@ -452,13 +514,14 @@ export class RequestQueueClient extends ResourceClient {
452
514
  parseArgument(request, existingRequestSchema);
453
515
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
454
516
  const response = await this.httpClient.call({
455
- url: this._url(['requests', request.id]),
517
+ url: this.buildUrl(['requests', request.id]),
456
518
  method: 'PUT',
457
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
519
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
520
+ signal: parsed.signal,
458
521
  data: request,
459
- params: this._params({
522
+ params: this.buildParams({
460
523
  forefront: parsed.forefront,
461
- clientKey: this.clientKey,
524
+ clientKey: this.#clientKey,
462
525
  }),
463
526
  });
464
527
  return parseResponse(response, schemas.RequestRegistration());
@@ -467,15 +530,19 @@ export class RequestQueueClient extends ResourceClient {
467
530
  * Deletes a specific request from the queue.
468
531
  *
469
532
  * @param id - Request ID
533
+ * @param options - Request options
534
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
470
535
  */
471
- async deleteRequest(id) {
536
+ async deleteRequest(id, options = {}) {
472
537
  parseArgument(id, requestIdSchema);
538
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
473
539
  await this.httpClient.call({
474
- url: this._url(['requests', id]),
540
+ url: this.buildUrl(['requests', id]),
475
541
  method: 'DELETE',
476
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
477
- params: this._params({
478
- clientKey: this.clientKey,
542
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
543
+ signal,
544
+ params: this.buildParams({
545
+ clientKey: this.#clientKey,
479
546
  }),
480
547
  });
481
548
  }
@@ -490,6 +557,7 @@ export class RequestQueueClient extends ResourceClient {
490
557
  * @param options - Lock extension options
491
558
  * @param options.lockSecs - **Required.** New lock duration in seconds from now.
492
559
  * @param options.forefront - If `true`, moves the request to the beginning of the queue when the lock expires. Default is `false`.
560
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
493
561
  * @returns Object with new `lockExpiresAt` timestamp
494
562
  * @see https://docs.apify.com/api/v2/request-queue-request-lock-put
495
563
  *
@@ -508,13 +576,14 @@ export class RequestQueueClient extends ResourceClient {
508
576
  parseArgument(id, requestIdSchema);
509
577
  const parsed = parseArgument(options, prolongRequestLockOptionsSchema, 'RequestQueueClientProlongRequestLockOptions');
510
578
  const response = await this.httpClient.call({
511
- url: this._url(['requests', id, 'lock']),
579
+ url: this.buildUrl(['requests', id, 'lock']),
512
580
  method: 'PUT',
513
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
514
- params: this._params({
581
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
582
+ signal: parsed.signal,
583
+ params: this.buildParams({
515
584
  forefront: parsed.forefront,
516
585
  lockSecs: parsed.lockSecs,
517
- clientKey: this.clientKey,
586
+ clientKey: this.#clientKey,
518
587
  }),
519
588
  });
520
589
  return parseResponse(response, schemas.RequestLockInfo());
@@ -527,6 +596,8 @@ export class RequestQueueClient extends ResourceClient {
527
596
  *
528
597
  * @param id - Request ID
529
598
  * @param options - Options such as whether to move to front
599
+ * @param options.forefront - If `true`, moves the request to the beginning of the queue. Default is `false`.
600
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
530
601
  * @see https://docs.apify.com/api/v2/request-queue-request-lock-delete
531
602
  * @since Added in 2.4.1
532
603
  */
@@ -534,12 +605,13 @@ export class RequestQueueClient extends ResourceClient {
534
605
  parseArgument(id, requestIdSchema);
535
606
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientDeleteRequestLockOptions');
536
607
  await this.httpClient.call({
537
- url: this._url(['requests', id, 'lock']),
608
+ url: this.buildUrl(['requests', id, 'lock']),
538
609
  method: 'DELETE',
539
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
540
- params: this._params({
610
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
611
+ signal: parsed.signal,
612
+ params: this.buildParams({
541
613
  forefront: parsed.forefront,
542
- clientKey: this.clientKey,
614
+ clientKey: this.#clientKey,
543
615
  }),
544
616
  });
545
617
  }
@@ -550,21 +622,26 @@ export class RequestQueueClient extends ResourceClient {
550
622
  * queue contents.
551
623
  *
552
624
  * @param options - Pagination options
625
+ * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
553
626
  * @returns List of requests with pagination information
554
627
  * @see https://docs.apify.com/api/v2/request-queue-requests-get
555
628
  * @since Added in 2.5.1
556
629
  */
557
630
  listRequests(options = {}) {
558
- const parsed = parseArgument(options, listRequestsOptionsSchema, 'RequestQueueClientListRequestsOptions');
631
+ // `timeoutSecs` and `signal` apply to every page request; they are not API parameters, so they must not reach
632
+ // the query string.
633
+ const { timeoutSecs, signal, ...parsed } = parseArgument(options, listRequestsOptionsSchema, 'RequestQueueClientListRequestsOptions');
634
+ const pageTimeout = this.#resolveTimeout(timeoutSecs, 'medium');
559
635
  const getPaginatedList = async (rqListOptions = {}) => {
560
636
  const response = await this.httpClient.call({
561
- url: this._url('requests'),
637
+ url: this.buildUrl('requests'),
562
638
  method: 'GET',
563
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
564
- params: this._params({
639
+ timeoutSecs: pageTimeout,
640
+ signal,
641
+ params: this.buildParams({
565
642
  ...rqListOptions,
566
643
  filter: rqListOptions.filter ? rqListOptions.filter.join(',') : undefined,
567
- clientKey: this.clientKey,
644
+ clientKey: this.#clientKey,
568
645
  }),
569
646
  });
570
647
  return parseResponse(response, schemas.ListOfRequests());
@@ -602,17 +679,21 @@ export class RequestQueueClient extends ResourceClient {
602
679
  * This is useful for releasing all locks at once, for example when shutting down
603
680
  * a crawler gracefully.
604
681
  *
682
+ * @param options - Request options
683
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
605
684
  * @returns Number of requests that were unlocked
606
685
  * @see https://docs.apify.com/api/v2/request-queue-requests-unlock-post
607
686
  * @since Added in 2.12.5
608
687
  */
609
- async unlockRequests() {
688
+ async unlockRequests(options = {}) {
689
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
610
690
  const response = await this.httpClient.call({
611
- url: this._url('requests/unlock'),
691
+ url: this.buildUrl('requests/unlock'),
612
692
  method: 'POST',
613
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
614
- params: this._params({
615
- clientKey: this.clientKey,
693
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'long'),
694
+ signal,
695
+ params: this.buildParams({
696
+ clientKey: this.#clientKey,
616
697
  }),
617
698
  });
618
699
  return parseResponse(response, schemas.UnlockRequestsResult());
@@ -624,6 +705,7 @@ export class RequestQueueClient extends ResourceClient {
624
705
  * automatically handling pagination behind the scenes.
625
706
  *
626
707
  * @param options - Pagination options
708
+ * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
627
709
  * @returns An async iterable of request pages
628
710
  * @see https://docs.apify.com/api/v2/request-queue-requests-get
629
711
  *
@@ -636,9 +718,9 @@ export class RequestQueueClient extends ResourceClient {
636
718
  * @since Added in 2.5.1
637
719
  */
638
720
  paginateRequests(options = {}) {
639
- const { limit, cursor, filter, maxPageLimit } = parseArgument(options, paginateRequestsOptionsSchema, 'RequestQueueClientPaginateRequestsOptions');
721
+ const { limit, cursor, filter, maxPageLimit, timeoutSecs, signal } = parseArgument(options, paginateRequestsOptionsSchema, 'RequestQueueClientPaginateRequestsOptions');
640
722
  return new RequestQueuePaginationIterator({
641
- getPage: async (pageOptions) => this.listRequests({ ...pageOptions, filter }),
723
+ getPage: async (pageOptions) => this.listRequests({ ...pageOptions, filter, timeoutSecs, signal }),
642
724
  limit,
643
725
  cursor,
644
726
  maxPageLimit,
@@ -1,8 +1,9 @@
1
1
  import { STORAGE_OWNERSHIP_FILTER } from '@apify/consts';
2
2
  import type { ApiClientSubResourceOptions } from '../base/api_client.js';
3
3
  import { ResourceCollectionClient } from '../base/resource_collection_client.js';
4
+ import type { TimeoutOptions } from '../timeouts.js';
4
5
  import type { PaginatedList, PaginationOptions } from '../utils.js';
5
- import type { RequestQueue } from './request_queue.js';
6
+ import type { RequestQueueResource } from './request_queue.js';
6
7
  /**
7
8
  * Client for managing the collection of Request queues in your account.
8
9
  *
@@ -45,20 +46,23 @@ export declare class RequestQueueCollectionClient extends ResourceCollectionClie
45
46
  * ```
46
47
  *
47
48
  * @param options - Pagination options.
49
+ * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
48
50
  * @returns A paginated iterator of Request queues.
49
51
  * @see https://docs.apify.com/api/v2/request-queues-get
50
52
  */
51
- list(options?: RequestQueueCollectionListOptions): Promise<RequestQueueCollectionListResult> & AsyncIterable<RequestQueue>;
53
+ list(options?: RequestQueueCollectionListOptions): Promise<RequestQueueCollectionListResult> & AsyncIterable<RequestQueueResource>;
52
54
  /**
53
55
  * Gets or creates a Request queue with the specified name.
54
56
  *
55
57
  * @param name - Name of the Request queue. If not provided, a default queue is used.
58
+ * @param options - Request options
59
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
56
60
  * @returns The Request queue object.
57
61
  * @see https://docs.apify.com/api/v2/request-queues-post
58
62
  */
59
- getOrCreate(name?: string): Promise<RequestQueue>;
63
+ getOrCreate(name?: string, options?: TimeoutOptions): Promise<RequestQueueResource>;
60
64
  }
61
- export interface RequestQueueCollectionListOptions extends PaginationOptions {
65
+ export interface RequestQueueCollectionListOptions extends PaginationOptions, TimeoutOptions {
62
66
  unnamed?: boolean;
63
67
  desc?: boolean;
64
68
  /**
@@ -67,6 +71,6 @@ export interface RequestQueueCollectionListOptions extends PaginationOptions {
67
71
  */
68
72
  ownership?: STORAGE_OWNERSHIP_FILTER;
69
73
  }
70
- export type RequestQueueCollectionListResult = PaginatedList<RequestQueue> & {
74
+ export type RequestQueueCollectionListResult = PaginatedList<RequestQueueResource> & {
71
75
  unnamed: boolean;
72
76
  };