apify-client 3.0.0-beta.6 → 3.0.0-beta.60

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/README.md +10 -2
  2. package/dist/apify_api_error.d.ts +11 -26
  3. package/dist/apify_api_error.js +31 -22
  4. package/dist/apify_client.d.ts +125 -10
  5. package/dist/apify_client.js +133 -48
  6. package/dist/base/api_client.d.ts +7 -7
  7. package/dist/base/api_client.js +36 -36
  8. package/dist/base/resource_client.d.ts +21 -8
  9. package/dist/base/resource_client.js +67 -33
  10. package/dist/base/resource_collection_client.d.ts +12 -4
  11. package/dist/base/resource_collection_client.js +34 -11
  12. package/dist/body_parser.d.ts +3 -4
  13. package/dist/body_parser.js +19 -26
  14. package/dist/bundle.js +27 -22
  15. package/dist/bundle.js.map +1 -1
  16. package/dist/generated/api.d.ts +785 -290
  17. package/dist/generated/schemas.d.ts +771 -798
  18. package/dist/generated/schemas.js +183 -179
  19. package/dist/http_clients/axios.d.ts +60 -0
  20. package/dist/http_clients/axios.js +117 -0
  21. package/dist/http_clients/base.d.ts +331 -0
  22. package/dist/http_clients/base.js +563 -0
  23. package/dist/http_clients/index.d.ts +4 -0
  24. package/dist/http_clients/index.js +2 -0
  25. package/dist/http_compressors/base.d.ts +40 -0
  26. package/dist/http_compressors/base.js +9 -0
  27. package/dist/http_compressors/brotli.d.ts +33 -0
  28. package/dist/http_compressors/brotli.js +39 -0
  29. package/dist/http_compressors/gzip.d.ts +33 -0
  30. package/dist/http_compressors/gzip.js +39 -0
  31. package/dist/http_compressors/index.d.ts +6 -0
  32. package/dist/http_compressors/index.js +2 -0
  33. package/dist/http_compressors/resolve.d.ts +23 -0
  34. package/dist/http_compressors/resolve.js +27 -0
  35. package/dist/index.d.ts +4 -1
  36. package/dist/index.js +3 -1
  37. package/dist/invalid_response_body_error.d.ts +13 -0
  38. package/dist/invalid_response_body_error.js +17 -0
  39. package/dist/models.d.ts +162 -194
  40. package/dist/models.js +6 -4
  41. package/dist/resource_clients/actor.d.ts +74 -34
  42. package/dist/resource_clients/actor.js +87 -56
  43. package/dist/resource_clients/actor_collection.d.ts +18 -14
  44. package/dist/resource_clients/actor_collection.js +11 -3
  45. package/dist/resource_clients/actor_env_var.d.ts +11 -4
  46. package/dist/resource_clients/actor_env_var.js +16 -6
  47. package/dist/resource_clients/actor_env_var_collection.d.ts +16 -20
  48. package/dist/resource_clients/actor_env_var_collection.js +17 -11
  49. package/dist/resource_clients/actor_version.d.ts +20 -5
  50. package/dist/resource_clients/actor_version.js +19 -9
  51. package/dist/resource_clients/actor_version_collection.d.ts +18 -19
  52. package/dist/resource_clients/actor_version_collection.js +19 -11
  53. package/dist/resource_clients/build.d.ts +20 -7
  54. package/dist/resource_clients/build.js +35 -15
  55. package/dist/resource_clients/build_collection.d.ts +7 -5
  56. package/dist/resource_clients/build_collection.js +4 -1
  57. package/dist/resource_clients/dataset.d.ts +53 -35
  58. package/dist/resource_clients/dataset.js +105 -80
  59. package/dist/resource_clients/dataset_collection.d.ts +10 -6
  60. package/dist/resource_clients/dataset_collection.js +13 -2
  61. package/dist/resource_clients/key_value_store.d.ts +73 -23
  62. package/dist/resource_clients/key_value_store.js +86 -43
  63. package/dist/resource_clients/key_value_store_collection.d.ts +10 -6
  64. package/dist/resource_clients/key_value_store_collection.js +13 -2
  65. package/dist/resource_clients/log.d.ts +13 -20
  66. package/dist/resource_clients/log.js +141 -59
  67. package/dist/resource_clients/request_queue.d.ts +74 -36
  68. package/dist/resource_clients/request_queue.js +182 -106
  69. package/dist/resource_clients/request_queue_collection.d.ts +9 -5
  70. package/dist/resource_clients/request_queue_collection.js +9 -3
  71. package/dist/resource_clients/run.d.ts +119 -28
  72. package/dist/resource_clients/run.js +190 -49
  73. package/dist/resource_clients/run_collection.d.ts +5 -3
  74. package/dist/resource_clients/run_collection.js +4 -1
  75. package/dist/resource_clients/schedule.d.ts +16 -7
  76. package/dist/resource_clients/schedule.js +30 -22
  77. package/dist/resource_clients/schedule_collection.d.ts +6 -2
  78. package/dist/resource_clients/schedule_collection.js +9 -3
  79. package/dist/resource_clients/store_collection.d.ts +6 -4
  80. package/dist/resource_clients/store_collection.js +4 -1
  81. package/dist/resource_clients/task.d.ts +36 -13
  82. package/dist/resource_clients/task.js +79 -46
  83. package/dist/resource_clients/task_collection.d.ts +9 -5
  84. package/dist/resource_clients/task_collection.js +9 -3
  85. package/dist/resource_clients/user.d.ts +19 -10
  86. package/dist/resource_clients/user.js +42 -38
  87. package/dist/resource_clients/webhook.d.ts +17 -8
  88. package/dist/resource_clients/webhook.js +30 -22
  89. package/dist/resource_clients/webhook_collection.d.ts +8 -4
  90. package/dist/resource_clients/webhook_collection.js +9 -3
  91. package/dist/resource_clients/webhook_dispatch.d.ts +4 -1
  92. package/dist/resource_clients/webhook_dispatch.js +7 -2
  93. package/dist/resource_clients/webhook_dispatch_collection.d.ts +3 -1
  94. package/dist/resource_clients/webhook_dispatch_collection.js +4 -1
  95. package/dist/runtime/node.d.ts +6 -0
  96. package/dist/runtime/node.js +49 -0
  97. package/dist/runtime/types.d.ts +52 -0
  98. package/dist/runtime/types.js +1 -0
  99. package/dist/runtime/web.d.ts +5 -0
  100. package/dist/runtime/web.js +18 -0
  101. package/dist/schemas.d.ts +1 -1
  102. package/dist/schemas.js +1 -1
  103. package/dist/statistics.d.ts +1 -6
  104. package/dist/statistics.js +2 -2
  105. package/dist/timeouts.d.ts +84 -0
  106. package/dist/timeouts.js +37 -0
  107. package/dist/timezones.d.ts +1 -1
  108. package/dist/utils.d.ts +89 -53
  109. package/dist/utils.js +214 -161
  110. package/dist/wait_for_resources.d.ts +8 -0
  111. package/dist/wait_for_resources.js +39 -0
  112. package/package.json +32 -29
  113. package/dist/http_client.d.ts +0 -88
  114. package/dist/http_client.js +0 -255
  115. package/dist/interceptors.d.ts +0 -19
  116. package/dist/interceptors.js +0 -105
@@ -1,25 +1,26 @@
1
1
  import { z } from 'zod';
2
2
  import { MAX_PAYLOAD_SIZE_BYTES, REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION } from '@apify/consts';
3
3
  import log from '@apify/log';
4
- import { MEDIUM_TIMEOUT_MILLIS, ResourceClient, SMALL_TIMEOUT_MILLIS } from '../base/resource_client.js';
4
+ import { ResourceClient } from '../base/resource_client.js';
5
5
  import { ResponseValidationError } from '../response_validation_error.js';
6
6
  import * as schemas from '../schemas.js';
7
- import { anyObjectSchema, cast, catchNotFoundOrThrow, isNonArrayObject, mutuallyExclusive, parseArgument, parseDateFields, parseResponse, RequestQueuePaginationIterator, sliceArrayByByteLength, } from '../utils.js';
7
+ import { optionalSignalSchema, optionalTimeoutSchema, timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
8
+ import { anyObjectSchema, catchNotFoundOrThrow, isNonArrayObject, minForLimitParam, parseArgument, parseResponse, RequestQueuePaginationIterator, sleep, splitIntoJsonArrayBatches, utf8ByteLength, } from '../utils.js';
8
9
  const DEFAULT_PARALLEL_BATCH_ADD_REQUESTS = 5;
9
10
  const DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS = 3;
10
11
  const DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS = 500;
11
12
  const DEFAULT_REQUEST_QUEUE_REQUEST_PAGE_LIMIT = 1000;
12
13
  const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
13
- const listHeadOptionsSchema = z.strictObject({ limit: z.number().min(0).optional() });
14
+ const listHeadOptionsSchema = z.strictObject({ limit: z.number().min(0).optional(), ...timeoutOptionsShape });
14
15
  const listAndLockHeadOptionsSchema = z.strictObject({
15
16
  lockSecs: z.number(),
16
17
  limit: z.number().min(0).optional(),
18
+ ...timeoutOptionsShape,
17
19
  });
18
20
  // Predicates, not `z.looseObject` arms: these run over a whole batch, and an object arm would copy
19
21
  // every key of every request. `id` is assigned by the API, so a new request must not carry one.
20
22
  const newRequestSchema = z.custom((value) => isNonArrayObject(value) && value.id === undefined, 'Expected a request object without an `id`');
21
- const forefrontOptionsSchema = z.strictObject({ forefront: z.boolean().optional() });
22
- const batchAddRequestsSchema = z.array(newRequestSchema).min(1).max(REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION);
23
+ const forefrontOptionsSchema = z.strictObject({ forefront: z.boolean().optional(), ...timeoutOptionsShape });
23
24
  const batchAddRequestsWithRetriesSchema = z.array(newRequestSchema).min(1);
24
25
  const optionalBooleanSchema = z.boolean().optional();
25
26
  const optionalNumberSchema = z.number().optional();
@@ -33,25 +34,23 @@ const existingRequestSchema = z.custom((value) => isNonArrayObject(value) && typ
33
34
  const prolongRequestLockOptionsSchema = z.strictObject({
34
35
  lockSecs: z.number(),
35
36
  forefront: z.boolean().optional(),
37
+ ...timeoutOptionsShape,
36
38
  });
37
39
  const requestFilterSchema = z.array(z.enum(['locked', 'pending'])).min(1);
38
- const listRequestsOptionsSchema = z
39
- .strictObject({
40
+ const listRequestsOptionsSchema = z.strictObject({
40
41
  limit: z.number().min(0).optional(),
41
- exclusiveStartId: z.string().optional(),
42
+ chunkSize: z.number().positive().optional(),
42
43
  cursor: z.string().optional(),
43
44
  filter: requestFilterSchema.optional(),
44
- })
45
- .refine(...mutuallyExclusive('exclusiveStartId', 'cursor'));
46
- const paginateRequestsOptionsSchema = z
47
- .strictObject({
45
+ ...timeoutOptionsShape,
46
+ });
47
+ const paginateRequestsOptionsSchema = z.strictObject({
48
48
  limit: z.number().min(0).optional(),
49
49
  maxPageLimit: z.number().default(DEFAULT_REQUEST_QUEUE_REQUEST_PAGE_LIMIT),
50
- exclusiveStartId: z.string().optional(),
51
50
  cursor: z.string().optional(),
52
51
  filter: requestFilterSchema.optional(),
53
- })
54
- .refine(...mutuallyExclusive('exclusiveStartId', 'cursor'));
52
+ ...timeoutOptionsShape,
53
+ });
55
54
  /**
56
55
  * Client for managing a specific Request queue.
57
56
  *
@@ -83,8 +82,8 @@ const paginateRequestsOptionsSchema = z
83
82
  * @see https://docs.apify.com/platform/storage/request-queue
84
83
  */
85
84
  export class RequestQueueClient extends ResourceClient {
86
- clientKey;
87
- timeoutMillis;
85
+ #clientKey;
86
+ #timeoutSecs;
88
87
  /**
89
88
  * @hidden
90
89
  */
@@ -93,36 +92,57 @@ export class RequestQueueClient extends ResourceClient {
93
92
  resourcePath: 'request-queues',
94
93
  ...options,
95
94
  });
96
- this.clientKey = userOptions.clientKey;
97
- this.timeoutMillis = userOptions.timeoutSecs ? userOptions.timeoutSecs * 1e3 : undefined;
95
+ this.#clientKey = userOptions.clientKey;
96
+ this.#timeoutSecs = userOptions.timeoutSecs;
97
+ }
98
+ /**
99
+ * Picks the timeout of a request: an explicit per-call `timeoutSecs` as is, otherwise the method's default
100
+ * tier, capped at the queue-wide `timeoutSecs` when one was given.
101
+ */
102
+ #resolveTimeout(timeoutSecs, defaultTier) {
103
+ if (timeoutSecs !== undefined)
104
+ return timeoutSecs;
105
+ if (this.#timeoutSecs === undefined)
106
+ return defaultTier;
107
+ const tierSecs = this.httpClient.timeoutMillis[defaultTier] / 1000;
108
+ return tierSecs <= this.#timeoutSecs ? defaultTier : this.#timeoutSecs;
98
109
  }
99
110
  /**
100
111
  * Gets the Request queue object from the Apify API.
101
112
  *
113
+ * @param options - Request options
114
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
102
115
  * @returns The RequestQueue object, or `undefined` if it does not exist
103
116
  * @see https://docs.apify.com/api/v2/request-queue-get
104
117
  */
105
- async get() {
106
- return this._get(schemas.RequestQueue(), {}, SMALL_TIMEOUT_MILLIS);
118
+ async get(options = {}) {
119
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
120
+ return this.getResource(schemas.RequestQueueResource(), {}, this.#resolveTimeout(timeoutSecs, 'short'), signal);
107
121
  }
108
122
  /**
109
123
  * Updates the Request queue with specified fields.
110
124
  *
111
125
  * @param newFields - Fields to update in the Request queue
126
+ * @param options - Request options
127
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
112
128
  * @returns The updated RequestQueue object
113
129
  * @see https://docs.apify.com/api/v2/request-queue-put
114
130
  */
115
- async update(newFields) {
131
+ async update(newFields, options = {}) {
116
132
  parseArgument(newFields, anyObjectSchema);
117
- return this._update(schemas.RequestQueue(), newFields, SMALL_TIMEOUT_MILLIS);
133
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
134
+ return this.updateResource(schemas.RequestQueueResource(), newFields, this.#resolveTimeout(timeoutSecs, 'short'), signal);
118
135
  }
119
136
  /**
120
137
  * Deletes the Request queue.
121
138
  *
139
+ * @param options - Request options
140
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
122
141
  * @see https://docs.apify.com/api/v2/request-queue-delete
123
142
  */
124
- async delete() {
125
- return this._delete(SMALL_TIMEOUT_MILLIS);
143
+ async delete(options = {}) {
144
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
145
+ return this.deleteResource(this.#resolveTimeout(timeoutSecs, 'short'), signal);
126
146
  }
127
147
  /**
128
148
  * Lists requests from the beginning of the queue (head).
@@ -131,18 +151,21 @@ export class RequestQueueClient extends ResourceClient {
131
151
  * inspecting what requests are waiting to be processed.
132
152
  *
133
153
  * @param options - Options for listing (e.g., limit)
154
+ * @param options.limit - Maximum number of requests to return.
155
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
134
156
  * @returns List of requests from the queue head
135
157
  * @see https://docs.apify.com/api/v2/request-queue-head-get
136
158
  */
137
159
  async listHead(options = {}) {
138
160
  const parsed = parseArgument(options, listHeadOptionsSchema, 'RequestQueueClientListHeadOptions');
139
161
  const response = await this.httpClient.call({
140
- url: this._url('head'),
162
+ url: this.buildUrl('head'),
141
163
  method: 'GET',
142
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
143
- params: this._params({
164
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
165
+ signal: parsed.signal,
166
+ params: this.buildParams({
144
167
  limit: parsed.limit,
145
- clientKey: this.clientKey,
168
+ clientKey: this.#clientKey,
146
169
  }),
147
170
  });
148
171
  return parseResponse(response, schemas.RequestQueueHead());
@@ -159,6 +182,7 @@ export class RequestQueueClient extends ResourceClient {
159
182
  * @param options - Lock configuration
160
183
  * @param options.lockSecs - **Required.** Duration in seconds to lock the requests. After this time, the locks expire and requests can be retrieved by other clients.
161
184
  * @param options.limit - Maximum number of requests to return. Default is 25.
185
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
162
186
  * @returns Object containing `items` (locked requests), `queueModifiedAt`, `hadMultipleClients`, and lock information
163
187
  * @see https://docs.apify.com/api/v2/request-queue-head-lock-post
164
188
  *
@@ -183,13 +207,14 @@ export class RequestQueueClient extends ResourceClient {
183
207
  async listAndLockHead(options) {
184
208
  const parsed = parseArgument(options, listAndLockHeadOptionsSchema, 'RequestQueueClientListAndLockHeadOptions');
185
209
  const response = await this.httpClient.call({
186
- url: this._url('head/lock'),
210
+ url: this.buildUrl('head/lock'),
187
211
  method: 'POST',
188
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
189
- params: this._params({
212
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
213
+ signal: parsed.signal,
214
+ params: this.buildParams({
190
215
  limit: parsed.limit,
191
216
  lockSecs: parsed.lockSecs,
192
- clientKey: this.clientKey,
217
+ clientKey: this.#clientKey,
193
218
  }),
194
219
  });
195
220
  return parseResponse(response, schemas.LockedRequestQueueHead());
@@ -210,6 +235,7 @@ export class RequestQueueClient extends ResourceClient {
210
235
  * @param request.payload - HTTP payload for POST/PUT requests (string).
211
236
  * @param options - Additional options
212
237
  * @param options.forefront - If `true`, adds the request to the beginning of the queue. Default is `false` (adds to the end).
238
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
213
239
  * @returns Object with `requestId`, `wasAlreadyPresent`, and `wasAlreadyHandled` flags
214
240
  * @see https://docs.apify.com/api/v2/request-queue-requests-post
215
241
  *
@@ -236,13 +262,14 @@ export class RequestQueueClient extends ResourceClient {
236
262
  parseArgument(request, newRequestSchema);
237
263
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
238
264
  const response = await this.httpClient.call({
239
- url: this._url('requests'),
265
+ url: this.buildUrl('requests'),
240
266
  method: 'POST',
241
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
267
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
268
+ signal: parsed.signal,
242
269
  data: request,
243
- params: this._params({
270
+ params: this.buildParams({
244
271
  forefront: parsed.forefront,
245
- clientKey: this.clientKey,
272
+ clientKey: this.#clientKey,
246
273
  }),
247
274
  });
248
275
  return parseResponse(response, schemas.RequestRegistration());
@@ -252,23 +279,26 @@ export class RequestQueueClient extends ResourceClient {
252
279
  *
253
280
  * @private
254
281
  */
255
- async _batchAddRequests(requests, options = {}) {
256
- parseArgument(requests, batchAddRequestsSchema);
282
+ async addRequestBatch(requests, options = {}) {
257
283
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
258
284
  const response = await this.httpClient.call({
259
- url: this._url('requests/batch'),
285
+ url: this.buildUrl('requests/batch'),
260
286
  method: 'POST',
261
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
262
- data: requests,
263
- params: this._params({
287
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
288
+ signal: parsed.signal,
289
+ // The body is assembled from the requests as `batchAddRequests` serialized them; the explicit content type
290
+ // makes the request interceptor send the string as it is instead of serializing the requests again.
291
+ headers: { 'content-type': 'application/json' },
292
+ data: `[${requests.map(({ json }) => json).join(',')}]`,
293
+ params: this.buildParams({
264
294
  forefront: parsed.forefront,
265
- clientKey: this.clientKey,
295
+ clientKey: this.#clientKey,
266
296
  }),
267
297
  });
268
298
  return parseResponse(response, schemas.BatchAddResult());
269
299
  }
270
- async _batchAddRequestsWithRetries(requests, options = {}) {
271
- const { forefront, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
300
+ async addRequestBatchWithRetries(requests, options = {}) {
301
+ const { forefront, timeoutSecs, signal, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
272
302
  // Keep track of the requests that remain to be processed (in parameter format)
273
303
  let remainingRequests = requests;
274
304
  // Keep track of the requests that have been processed (in api format)
@@ -278,8 +308,10 @@ export class RequestQueueClient extends ResourceClient {
278
308
  let unprocessedRequests = [];
279
309
  for (let i = 0; i < 1 + maxUnprocessedRequestsRetries; i++) {
280
310
  try {
281
- const response = await this._batchAddRequests(remainingRequests, {
311
+ const response = await this.addRequestBatch(remainingRequests, {
282
312
  forefront,
313
+ timeoutSecs,
314
+ signal,
283
315
  });
284
316
  processedRequests.push(...response.processedRequests);
285
317
  unprocessedRequests = response.unprocessedRequests;
@@ -291,13 +323,15 @@ export class RequestQueueClient extends ResourceClient {
291
323
  // Get unique keys of all requests processed so far
292
324
  const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
293
325
  // Requests remaining to be processed are the all that remain
294
- remainingRequests = requests.filter(({ uniqueKey }) => !processedRequestsUniqueKeys.includes(uniqueKey));
326
+ remainingRequests = requests.filter(({ request }) => !processedRequestsUniqueKeys.includes(request.uniqueKey));
295
327
  // Stop if all requests have been processed
296
328
  if (remainingRequests.length === 0) {
297
329
  break;
298
330
  }
299
331
  }
300
332
  catch (err) {
333
+ // An abort rejects the whole call, since the caller asked to stop and not for a partial result.
334
+ signal?.throwIfAborted();
301
335
  // A response the specification does not describe is reported, as everywhere else: the server may well
302
336
  // have added the batch, so calling it unprocessed would hide the mismatch behind a wrong answer.
303
337
  if (err instanceof ResponseValidationError)
@@ -307,18 +341,16 @@ export class RequestQueueClient extends ResourceClient {
307
341
  // This ensures that this method does not throw and keeps the signature.
308
342
  const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
309
343
  unprocessedRequests = requests
310
- .filter(({ uniqueKey }) => !processedRequestsUniqueKeys.includes(uniqueKey))
311
- .map(({ method, uniqueKey, url }) => ({ method, uniqueKey, url }));
344
+ .filter(({ request }) => !processedRequestsUniqueKeys.includes(request.uniqueKey))
345
+ .map(({ request: { method, uniqueKey, url } }) => ({ method, uniqueKey, url }));
312
346
  break;
313
347
  }
314
348
  // Exponential backoff
315
349
  const delayMillis = Math.floor((1 + Math.random()) * 2 ** i * minDelayBetweenUnprocessedRequestsRetriesMillis);
316
- await new Promise((resolve) => {
317
- setTimeout(resolve, delayMillis);
318
- });
350
+ await sleep(delayMillis, signal);
351
+ signal?.throwIfAborted();
319
352
  }
320
- const result = { processedRequests, unprocessedRequests };
321
- return cast(parseDateFields(result));
353
+ return { processedRequests, unprocessedRequests };
322
354
  }
323
355
  /**
324
356
  * Adds multiple requests to the queue in a single operation.
@@ -335,6 +367,7 @@ export class RequestQueueClient extends ResourceClient {
335
367
  * @param options.maxUnprocessedRequestsRetries - Maximum number of retry attempts for rate-limited requests. Default is 3.
336
368
  * @param options.maxParallel - Maximum number of parallel batch API calls. Default is 5.
337
369
  * @param options.minDelayBetweenUnprocessedRequestsRetriesMillis - Minimum delay before retrying rate-limited requests. Default is 500ms.
370
+ * @param options.timeoutSecs - Timeout for each batch API request. Default is `'medium'`.
338
371
  * @returns Object with `processedRequests` (successfully added) and `unprocessedRequests` (failed after all retries)
339
372
  * @see https://docs.apify.com/api/v2/request-queue-requests-batch-post
340
373
  *
@@ -359,21 +392,36 @@ export class RequestQueueClient extends ResourceClient {
359
392
  * @since Added in 2.1.0
360
393
  */
361
394
  async batchAddRequests(requests, options = {}) {
362
- const { forefront, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, maxParallel = DEFAULT_PARALLEL_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
395
+ const { forefront, timeoutSecs, signal, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, maxParallel = DEFAULT_PARALLEL_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
363
396
  parseArgument(requests, batchAddRequestsWithRetriesSchema);
364
397
  parseArgument(forefront, optionalBooleanSchema);
398
+ parseArgument(timeoutSecs, optionalTimeoutSchema);
399
+ parseArgument(signal, optionalSignalSchema);
365
400
  parseArgument(maxUnprocessedRequestsRetries, optionalNumberSchema);
366
401
  parseArgument(maxParallel, optionalNumberSchema);
367
402
  parseArgument(minDelayBetweenUnprocessedRequestsRetriesMillis, optionalNumberSchema);
368
403
  const executingRequests = new Set();
369
404
  const individualResults = [];
370
405
  const payloadSizeLimitBytes = MAX_PAYLOAD_SIZE_BYTES - Math.ceil(MAX_PAYLOAD_SIZE_BYTES * SAFETY_BUFFER_PERCENT);
406
+ // Serialize every request once: the byte lengths decide the batch boundaries, and the same strings are joined
407
+ // into the batch bodies, so nothing is stringified a second time when it is sent.
408
+ const serializedRequests = requests.map((request, index) => {
409
+ const json = JSON.stringify(request);
410
+ const byteLength = utf8ByteLength(json);
411
+ // Two more bytes for the brackets, which even a batch of one request carries.
412
+ if (byteLength + 2 > payloadSizeLimitBytes) {
413
+ throw new Error(`RequestQueueClient.batchAddRequests: The size of the request with index: ${index} ` +
414
+ `exceeds the maximum allowed size (${payloadSizeLimitBytes} bytes).`);
415
+ }
416
+ return { request, json, byteLength };
417
+ });
418
+ const batches = splitIntoJsonArrayBatches(serializedRequests, {
419
+ maxCount: REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION,
420
+ maxByteLength: payloadSizeLimitBytes,
421
+ });
371
422
  // Keep a pool of up to `maxParallel` requests running at once
372
- let i = 0;
373
- while (i < requests.length) {
374
- const slicedRequests = requests.slice(i, i + REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION);
375
- const requestsInBatch = sliceArrayByByteLength(slicedRequests, payloadSizeLimitBytes, i);
376
- const requestPromise = this._batchAddRequestsWithRetries(requestsInBatch, options);
423
+ for (const batch of batches) {
424
+ const requestPromise = this.addRequestBatchWithRetries(batch, options);
377
425
  executingRequests.add(requestPromise);
378
426
  // A rejection reaches the caller through the awaits below; this bookkeeping chain only has to avoid
379
427
  // turning it into an unhandled one of its own.
@@ -384,7 +432,6 @@ export class RequestQueueClient extends ResourceClient {
384
432
  if (executingRequests.size >= maxParallel) {
385
433
  await Promise.race(executingRequests);
386
434
  }
387
- i += requestsInBatch.length;
388
435
  }
389
436
  // Get results from remaining operations
390
437
  await Promise.all(executingRequests);
@@ -405,19 +452,23 @@ export class RequestQueueClient extends ResourceClient {
405
452
  * Requests can be identified by either their ID or unique key.
406
453
  *
407
454
  * @param requests - Array of requests to delete (by id or uniqueKey)
455
+ * @param options - Request options
456
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
408
457
  * @returns Result containing processed and unprocessed requests
409
458
  * @see https://docs.apify.com/api/v2/request-queue-requests-batch-delete
410
459
  * @since Added in 2.3.0
411
460
  */
412
- async batchDeleteRequests(requests) {
461
+ async batchDeleteRequests(requests, options = {}) {
413
462
  parseArgument(requests, batchDeleteRequestsSchema);
463
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
414
464
  const response = await this.httpClient.call({
415
- url: this._url('requests/batch'),
465
+ url: this.buildUrl('requests/batch'),
416
466
  method: 'DELETE',
417
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
467
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
468
+ signal,
418
469
  data: requests,
419
- params: this._params({
420
- clientKey: this.clientKey,
470
+ params: this.buildParams({
471
+ clientKey: this.#clientKey,
421
472
  }),
422
473
  });
423
474
  return parseResponse(response, schemas.BatchDeleteResult());
@@ -426,20 +477,24 @@ export class RequestQueueClient extends ResourceClient {
426
477
  * Gets a specific request from the queue by its ID.
427
478
  *
428
479
  * @param id - Request ID
480
+ * @param options - Request options
481
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
429
482
  * @returns The request object, or `undefined` if not found
430
483
  * @see https://docs.apify.com/api/v2/request-queue-request-get
431
484
  */
432
- async getRequest(id) {
485
+ async getRequest(id, options = {}) {
433
486
  parseArgument(id, requestIdSchema);
487
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
434
488
  const requestOpts = {
435
- url: this._url(['requests', id]),
489
+ url: this.buildUrl(['requests', id]),
436
490
  method: 'GET',
437
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
438
- params: this._params(),
491
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
492
+ signal,
493
+ params: this.buildParams(),
439
494
  };
440
495
  try {
441
496
  const response = await this.httpClient.call(requestOpts);
442
- return parseResponse(response, schemas.Request());
497
+ return parseResponse(response, schemas.RequestResource());
443
498
  }
444
499
  catch (err) {
445
500
  catchNotFoundOrThrow(err);
@@ -451,6 +506,8 @@ export class RequestQueueClient extends ResourceClient {
451
506
  *
452
507
  * @param request - The updated request object (must include id)
453
508
  * @param options - Update options such as whether to move to front
509
+ * @param options.forefront - If `true`, moves the request to the beginning of the queue. Default is `false`.
510
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
454
511
  * @returns Information about the updated request
455
512
  * @see https://docs.apify.com/api/v2/request-queue-request-put
456
513
  */
@@ -458,13 +515,14 @@ export class RequestQueueClient extends ResourceClient {
458
515
  parseArgument(request, existingRequestSchema);
459
516
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
460
517
  const response = await this.httpClient.call({
461
- url: this._url(['requests', request.id]),
518
+ url: this.buildUrl(['requests', request.id]),
462
519
  method: 'PUT',
463
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
520
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
521
+ signal: parsed.signal,
464
522
  data: request,
465
- params: this._params({
523
+ params: this.buildParams({
466
524
  forefront: parsed.forefront,
467
- clientKey: this.clientKey,
525
+ clientKey: this.#clientKey,
468
526
  }),
469
527
  });
470
528
  return parseResponse(response, schemas.RequestRegistration());
@@ -473,15 +531,19 @@ export class RequestQueueClient extends ResourceClient {
473
531
  * Deletes a specific request from the queue.
474
532
  *
475
533
  * @param id - Request ID
534
+ * @param options - Request options
535
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
476
536
  */
477
- async deleteRequest(id) {
537
+ async deleteRequest(id, options = {}) {
478
538
  parseArgument(id, requestIdSchema);
539
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
479
540
  await this.httpClient.call({
480
- url: this._url(['requests', id]),
541
+ url: this.buildUrl(['requests', id]),
481
542
  method: 'DELETE',
482
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
483
- params: this._params({
484
- clientKey: this.clientKey,
543
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
544
+ signal,
545
+ params: this.buildParams({
546
+ clientKey: this.#clientKey,
485
547
  }),
486
548
  });
487
549
  }
@@ -496,6 +558,7 @@ export class RequestQueueClient extends ResourceClient {
496
558
  * @param options - Lock extension options
497
559
  * @param options.lockSecs - **Required.** New lock duration in seconds from now.
498
560
  * @param options.forefront - If `true`, moves the request to the beginning of the queue when the lock expires. Default is `false`.
561
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
499
562
  * @returns Object with new `lockExpiresAt` timestamp
500
563
  * @see https://docs.apify.com/api/v2/request-queue-request-lock-put
501
564
  *
@@ -514,13 +577,14 @@ export class RequestQueueClient extends ResourceClient {
514
577
  parseArgument(id, requestIdSchema);
515
578
  const parsed = parseArgument(options, prolongRequestLockOptionsSchema, 'RequestQueueClientProlongRequestLockOptions');
516
579
  const response = await this.httpClient.call({
517
- url: this._url(['requests', id, 'lock']),
580
+ url: this.buildUrl(['requests', id, 'lock']),
518
581
  method: 'PUT',
519
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
520
- params: this._params({
582
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
583
+ signal: parsed.signal,
584
+ params: this.buildParams({
521
585
  forefront: parsed.forefront,
522
586
  lockSecs: parsed.lockSecs,
523
- clientKey: this.clientKey,
587
+ clientKey: this.#clientKey,
524
588
  }),
525
589
  });
526
590
  return parseResponse(response, schemas.RequestLockInfo());
@@ -533,6 +597,8 @@ export class RequestQueueClient extends ResourceClient {
533
597
  *
534
598
  * @param id - Request ID
535
599
  * @param options - Options such as whether to move to front
600
+ * @param options.forefront - If `true`, moves the request to the beginning of the queue. Default is `false`.
601
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
536
602
  * @see https://docs.apify.com/api/v2/request-queue-request-lock-delete
537
603
  * @since Added in 2.4.1
538
604
  */
@@ -540,12 +606,13 @@ export class RequestQueueClient extends ResourceClient {
540
606
  parseArgument(id, requestIdSchema);
541
607
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientDeleteRequestLockOptions');
542
608
  await this.httpClient.call({
543
- url: this._url(['requests', id, 'lock']),
609
+ url: this.buildUrl(['requests', id, 'lock']),
544
610
  method: 'DELETE',
545
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
546
- params: this._params({
611
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
612
+ signal: parsed.signal,
613
+ params: this.buildParams({
547
614
  forefront: parsed.forefront,
548
- clientKey: this.clientKey,
615
+ clientKey: this.#clientKey,
549
616
  }),
550
617
  });
551
618
  }
@@ -556,26 +623,33 @@ export class RequestQueueClient extends ResourceClient {
556
623
  * queue contents.
557
624
  *
558
625
  * @param options - Pagination options
626
+ * @param options.chunkSize - Maximum number of requests fetched per API call when iterating across pages. `limit`
627
+ * then caps the total across all pages.
628
+ * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
559
629
  * @returns List of requests with pagination information
560
630
  * @see https://docs.apify.com/api/v2/request-queue-requests-get
561
631
  * @since Added in 2.5.1
562
632
  */
563
633
  listRequests(options = {}) {
564
- const parsed = parseArgument(options, listRequestsOptionsSchema, 'RequestQueueClientListRequestsOptions');
634
+ // `chunkSize` only sizes the page requests, and `timeoutSecs` and `signal` apply to every one of them; none of
635
+ // them is an API parameter, so they must not reach the query string.
636
+ const { chunkSize, timeoutSecs, signal, ...parsed } = parseArgument(options, listRequestsOptionsSchema, 'RequestQueueClientListRequestsOptions');
637
+ const pageTimeout = this.#resolveTimeout(timeoutSecs, 'medium');
565
638
  const getPaginatedList = async (rqListOptions = {}) => {
566
639
  const response = await this.httpClient.call({
567
- url: this._url('requests'),
640
+ url: this.buildUrl('requests'),
568
641
  method: 'GET',
569
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
570
- params: this._params({
642
+ timeoutSecs: pageTimeout,
643
+ signal,
644
+ params: this.buildParams({
571
645
  ...rqListOptions,
572
646
  filter: rqListOptions.filter ? rqListOptions.filter.join(',') : undefined,
573
- clientKey: this.clientKey,
647
+ clientKey: this.#clientKey,
574
648
  }),
575
649
  });
576
650
  return parseResponse(response, schemas.ListOfRequests());
577
651
  };
578
- const paginatedListPromise = getPaginatedList(parsed);
652
+ const paginatedListPromise = getPaginatedList({ ...parsed, limit: minForLimitParam(parsed.limit, chunkSize) });
579
653
  async function* asyncGenerator() {
580
654
  let currentPage = await paginatedListPromise;
581
655
  yield* currentPage.items;
@@ -588,9 +662,7 @@ export class RequestQueueClient extends ResourceClient {
588
662
  ) {
589
663
  const newOptions = {
590
664
  ...parsed,
591
- limit: remainingItems,
592
- // remove original exclusiveStartId, if there was any, and use cursor-based pagination
593
- exclusiveStartId: undefined,
665
+ limit: minForLimitParam(remainingItems, chunkSize),
594
666
  cursor: currentPage.nextCursor,
595
667
  };
596
668
  currentPage = await getPaginatedList(newOptions);
@@ -610,17 +682,21 @@ export class RequestQueueClient extends ResourceClient {
610
682
  * This is useful for releasing all locks at once, for example when shutting down
611
683
  * a crawler gracefully.
612
684
  *
685
+ * @param options - Request options
686
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
613
687
  * @returns Number of requests that were unlocked
614
688
  * @see https://docs.apify.com/api/v2/request-queue-requests-unlock-post
615
689
  * @since Added in 2.12.5
616
690
  */
617
- async unlockRequests() {
691
+ async unlockRequests(options = {}) {
692
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
618
693
  const response = await this.httpClient.call({
619
- url: this._url('requests/unlock'),
694
+ url: this.buildUrl('requests/unlock'),
620
695
  method: 'POST',
621
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
622
- params: this._params({
623
- clientKey: this.clientKey,
696
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'long'),
697
+ signal,
698
+ params: this.buildParams({
699
+ clientKey: this.#clientKey,
624
700
  }),
625
701
  });
626
702
  return parseResponse(response, schemas.UnlockRequestsResult());
@@ -632,6 +708,7 @@ export class RequestQueueClient extends ResourceClient {
632
708
  * automatically handling pagination behind the scenes.
633
709
  *
634
710
  * @param options - Pagination options
711
+ * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
635
712
  * @returns An async iterable of request pages
636
713
  * @see https://docs.apify.com/api/v2/request-queue-requests-get
637
714
  *
@@ -644,11 +721,10 @@ export class RequestQueueClient extends ResourceClient {
644
721
  * @since Added in 2.5.1
645
722
  */
646
723
  paginateRequests(options = {}) {
647
- const { limit, exclusiveStartId, cursor, filter, maxPageLimit } = parseArgument(options, paginateRequestsOptionsSchema, 'RequestQueueClientPaginateRequestsOptions');
724
+ const { limit, cursor, filter, maxPageLimit, timeoutSecs, signal } = parseArgument(options, paginateRequestsOptionsSchema, 'RequestQueueClientPaginateRequestsOptions');
648
725
  return new RequestQueuePaginationIterator({
649
- getPage: async (pageOptions) => this.listRequests({ ...pageOptions, filter }),
726
+ getPage: async (pageOptions) => this.listRequests({ ...pageOptions, filter, timeoutSecs, signal }),
650
727
  limit,
651
- exclusiveStartId,
652
728
  cursor,
653
729
  maxPageLimit,
654
730
  });