apify-client 3.0.0-beta.5 → 3.0.0-beta.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/README.md +13 -3
  2. package/dist/apify_api_error.d.ts +62 -21
  3. package/dist/apify_api_error.js +96 -23
  4. package/dist/apify_client.d.ts +125 -10
  5. package/dist/apify_client.js +133 -48
  6. package/dist/base/api_client.d.ts +7 -7
  7. package/dist/base/api_client.js +36 -24
  8. package/dist/base/resource_client.d.ts +21 -8
  9. package/dist/base/resource_client.js +54 -26
  10. package/dist/base/resource_collection_client.d.ts +12 -4
  11. package/dist/base/resource_collection_client.js +34 -11
  12. package/dist/body_parser.d.ts +3 -4
  13. package/dist/body_parser.js +19 -26
  14. package/dist/bundle.js +27 -22
  15. package/dist/bundle.js.map +1 -1
  16. package/dist/generated/api.d.ts +610 -112
  17. package/dist/generated/schemas.d.ts +726 -703
  18. package/dist/generated/schemas.js +92 -85
  19. package/dist/http_clients/axios.d.ts +60 -0
  20. package/dist/http_clients/axios.js +117 -0
  21. package/dist/http_clients/base.d.ts +333 -0
  22. package/dist/http_clients/base.js +563 -0
  23. package/dist/http_clients/index.d.ts +4 -0
  24. package/dist/http_clients/index.js +2 -0
  25. package/dist/http_compressors/base.d.ts +40 -0
  26. package/dist/http_compressors/base.js +9 -0
  27. package/dist/http_compressors/brotli.d.ts +33 -0
  28. package/dist/http_compressors/brotli.js +39 -0
  29. package/dist/http_compressors/gzip.d.ts +33 -0
  30. package/dist/http_compressors/gzip.js +39 -0
  31. package/dist/http_compressors/index.d.ts +6 -0
  32. package/dist/http_compressors/index.js +2 -0
  33. package/dist/http_compressors/resolve.d.ts +23 -0
  34. package/dist/http_compressors/resolve.js +27 -0
  35. package/dist/index.d.ts +4 -1
  36. package/dist/index.js +3 -1
  37. package/dist/invalid_response_body_error.d.ts +13 -0
  38. package/dist/invalid_response_body_error.js +17 -0
  39. package/dist/models.d.ts +19 -22
  40. package/dist/resource_clients/actor.d.ts +72 -32
  41. package/dist/resource_clients/actor.js +87 -56
  42. package/dist/resource_clients/actor_collection.d.ts +9 -5
  43. package/dist/resource_clients/actor_collection.js +11 -3
  44. package/dist/resource_clients/actor_env_var.d.ts +10 -3
  45. package/dist/resource_clients/actor_env_var.js +16 -6
  46. package/dist/resource_clients/actor_env_var_collection.d.ts +14 -18
  47. package/dist/resource_clients/actor_env_var_collection.js +17 -11
  48. package/dist/resource_clients/actor_version.d.ts +18 -3
  49. package/dist/resource_clients/actor_version.js +19 -9
  50. package/dist/resource_clients/actor_version_collection.d.ts +16 -17
  51. package/dist/resource_clients/actor_version_collection.js +19 -11
  52. package/dist/resource_clients/build.d.ts +18 -6
  53. package/dist/resource_clients/build.js +34 -15
  54. package/dist/resource_clients/build_collection.d.ts +3 -1
  55. package/dist/resource_clients/build_collection.js +4 -1
  56. package/dist/resource_clients/dataset.d.ts +51 -33
  57. package/dist/resource_clients/dataset.js +105 -80
  58. package/dist/resource_clients/dataset_collection.d.ts +6 -2
  59. package/dist/resource_clients/dataset_collection.js +13 -2
  60. package/dist/resource_clients/key_value_store.d.ts +58 -16
  61. package/dist/resource_clients/key_value_store.js +78 -40
  62. package/dist/resource_clients/key_value_store_collection.d.ts +6 -2
  63. package/dist/resource_clients/key_value_store_collection.js +13 -2
  64. package/dist/resource_clients/log.d.ts +11 -19
  65. package/dist/resource_clients/log.js +64 -46
  66. package/dist/resource_clients/request_queue.d.ts +56 -26
  67. package/dist/resource_clients/request_queue.js +176 -103
  68. package/dist/resource_clients/request_queue_collection.d.ts +6 -2
  69. package/dist/resource_clients/request_queue_collection.js +9 -3
  70. package/dist/resource_clients/run.d.ts +108 -18
  71. package/dist/resource_clients/run.js +183 -43
  72. package/dist/resource_clients/run_collection.d.ts +3 -1
  73. package/dist/resource_clients/run_collection.js +4 -1
  74. package/dist/resource_clients/schedule.d.ts +14 -5
  75. package/dist/resource_clients/schedule.js +29 -21
  76. package/dist/resource_clients/schedule_collection.d.ts +6 -2
  77. package/dist/resource_clients/schedule_collection.js +9 -3
  78. package/dist/resource_clients/store_collection.d.ts +3 -1
  79. package/dist/resource_clients/store_collection.js +4 -1
  80. package/dist/resource_clients/task.d.ts +33 -10
  81. package/dist/resource_clients/task.js +79 -46
  82. package/dist/resource_clients/task_collection.d.ts +6 -2
  83. package/dist/resource_clients/task_collection.js +9 -3
  84. package/dist/resource_clients/user.d.ts +17 -8
  85. package/dist/resource_clients/user.js +42 -38
  86. package/dist/resource_clients/webhook.d.ts +14 -5
  87. package/dist/resource_clients/webhook.js +30 -22
  88. package/dist/resource_clients/webhook_collection.d.ts +6 -2
  89. package/dist/resource_clients/webhook_collection.js +9 -3
  90. package/dist/resource_clients/webhook_dispatch.d.ts +4 -1
  91. package/dist/resource_clients/webhook_dispatch.js +7 -2
  92. package/dist/resource_clients/webhook_dispatch_collection.d.ts +3 -1
  93. package/dist/resource_clients/webhook_dispatch_collection.js +4 -1
  94. package/dist/runtime/node.d.ts +6 -0
  95. package/dist/runtime/node.js +49 -0
  96. package/dist/runtime/types.d.ts +52 -0
  97. package/dist/runtime/types.js +1 -0
  98. package/dist/runtime/web.d.ts +5 -0
  99. package/dist/runtime/web.js +18 -0
  100. package/dist/schemas.d.ts +1 -1
  101. package/dist/schemas.js +1 -1
  102. package/dist/statistics.d.ts +1 -6
  103. package/dist/statistics.js +2 -2
  104. package/dist/timeouts.d.ts +84 -0
  105. package/dist/timeouts.js +37 -0
  106. package/dist/timezones.d.ts +1 -1
  107. package/dist/utils.d.ts +81 -52
  108. package/dist/utils.js +201 -170
  109. package/dist/wait_for_resources.d.ts +8 -0
  110. package/dist/wait_for_resources.js +39 -0
  111. package/package.json +32 -29
  112. package/dist/http_client.d.ts +0 -88
  113. package/dist/http_client.js +0 -255
  114. package/dist/interceptors.d.ts +0 -19
  115. package/dist/interceptors.js +0 -105
@@ -1,25 +1,26 @@
1
1
  import { z } from 'zod';
2
2
  import { MAX_PAYLOAD_SIZE_BYTES, REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION } from '@apify/consts';
3
3
  import log from '@apify/log';
4
- import { MEDIUM_TIMEOUT_MILLIS, ResourceClient, SMALL_TIMEOUT_MILLIS } from '../base/resource_client.js';
4
+ import { ResourceClient } from '../base/resource_client.js';
5
5
  import { ResponseValidationError } from '../response_validation_error.js';
6
6
  import * as schemas from '../schemas.js';
7
- import { anyObjectSchema, cast, catchNotFoundOrThrow, isNonArrayObject, mutuallyExclusive, parseArgument, parseDateFields, parseResponse, RequestQueuePaginationIterator, sliceArrayByByteLength, } from '../utils.js';
7
+ import { optionalSignalSchema, optionalTimeoutSchema, timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
8
+ import { anyObjectSchema, catchNotFoundOrThrow, isNonArrayObject, parseArgument, parseResponse, RequestQueuePaginationIterator, sleep, splitIntoJsonArrayBatches, utf8ByteLength, } from '../utils.js';
8
9
  const DEFAULT_PARALLEL_BATCH_ADD_REQUESTS = 5;
9
10
  const DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS = 3;
10
11
  const DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS = 500;
11
12
  const DEFAULT_REQUEST_QUEUE_REQUEST_PAGE_LIMIT = 1000;
12
13
  const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
13
- const listHeadOptionsSchema = z.strictObject({ limit: z.number().min(0).optional() });
14
+ const listHeadOptionsSchema = z.strictObject({ limit: z.number().min(0).optional(), ...timeoutOptionsShape });
14
15
  const listAndLockHeadOptionsSchema = z.strictObject({
15
16
  lockSecs: z.number(),
16
17
  limit: z.number().min(0).optional(),
18
+ ...timeoutOptionsShape,
17
19
  });
18
20
  // Predicates, not `z.looseObject` arms: these run over a whole batch, and an object arm would copy
19
21
  // every key of every request. `id` is assigned by the API, so a new request must not carry one.
20
22
  const newRequestSchema = z.custom((value) => isNonArrayObject(value) && value.id === undefined, 'Expected a request object without an `id`');
21
- const forefrontOptionsSchema = z.strictObject({ forefront: z.boolean().optional() });
22
- const batchAddRequestsSchema = z.array(newRequestSchema).min(1).max(REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION);
23
+ const forefrontOptionsSchema = z.strictObject({ forefront: z.boolean().optional(), ...timeoutOptionsShape });
23
24
  const batchAddRequestsWithRetriesSchema = z.array(newRequestSchema).min(1);
24
25
  const optionalBooleanSchema = z.boolean().optional();
25
26
  const optionalNumberSchema = z.number().optional();
@@ -33,25 +34,22 @@ const existingRequestSchema = z.custom((value) => isNonArrayObject(value) && typ
33
34
  const prolongRequestLockOptionsSchema = z.strictObject({
34
35
  lockSecs: z.number(),
35
36
  forefront: z.boolean().optional(),
37
+ ...timeoutOptionsShape,
36
38
  });
37
39
  const requestFilterSchema = z.array(z.enum(['locked', 'pending'])).min(1);
38
- const listRequestsOptionsSchema = z
39
- .strictObject({
40
+ const listRequestsOptionsSchema = z.strictObject({
40
41
  limit: z.number().min(0).optional(),
41
- exclusiveStartId: z.string().optional(),
42
42
  cursor: z.string().optional(),
43
43
  filter: requestFilterSchema.optional(),
44
- })
45
- .refine(...mutuallyExclusive('exclusiveStartId', 'cursor'));
46
- const paginateRequestsOptionsSchema = z
47
- .strictObject({
44
+ ...timeoutOptionsShape,
45
+ });
46
+ const paginateRequestsOptionsSchema = z.strictObject({
48
47
  limit: z.number().min(0).optional(),
49
48
  maxPageLimit: z.number().default(DEFAULT_REQUEST_QUEUE_REQUEST_PAGE_LIMIT),
50
- exclusiveStartId: z.string().optional(),
51
49
  cursor: z.string().optional(),
52
50
  filter: requestFilterSchema.optional(),
53
- })
54
- .refine(...mutuallyExclusive('exclusiveStartId', 'cursor'));
51
+ ...timeoutOptionsShape,
52
+ });
55
53
  /**
56
54
  * Client for managing a specific Request queue.
57
55
  *
@@ -83,8 +81,8 @@ const paginateRequestsOptionsSchema = z
83
81
  * @see https://docs.apify.com/platform/storage/request-queue
84
82
  */
85
83
  export class RequestQueueClient extends ResourceClient {
86
- clientKey;
87
- timeoutMillis;
84
+ #clientKey;
85
+ #timeoutSecs;
88
86
  /**
89
87
  * @hidden
90
88
  */
@@ -93,36 +91,57 @@ export class RequestQueueClient extends ResourceClient {
93
91
  resourcePath: 'request-queues',
94
92
  ...options,
95
93
  });
96
- this.clientKey = userOptions.clientKey;
97
- this.timeoutMillis = userOptions.timeoutSecs ? userOptions.timeoutSecs * 1e3 : undefined;
94
+ this.#clientKey = userOptions.clientKey;
95
+ this.#timeoutSecs = userOptions.timeoutSecs;
96
+ }
97
+ /**
98
+ * Picks the timeout of a request: an explicit per-call `timeoutSecs` as is, otherwise the method's default
99
+ * tier, capped at the queue-wide `timeoutSecs` when one was given.
100
+ */
101
+ #resolveTimeout(timeoutSecs, defaultTier) {
102
+ if (timeoutSecs !== undefined)
103
+ return timeoutSecs;
104
+ if (this.#timeoutSecs === undefined)
105
+ return defaultTier;
106
+ const tierSecs = this.httpClient.timeoutMillis[defaultTier] / 1000;
107
+ return tierSecs <= this.#timeoutSecs ? defaultTier : this.#timeoutSecs;
98
108
  }
99
109
  /**
100
110
  * Gets the Request queue object from the Apify API.
101
111
  *
112
+ * @param options - Request options
113
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
102
114
  * @returns The RequestQueue object, or `undefined` if it does not exist
103
115
  * @see https://docs.apify.com/api/v2/request-queue-get
104
116
  */
105
- async get() {
106
- return this._get(schemas.RequestQueue(), {}, SMALL_TIMEOUT_MILLIS);
117
+ async get(options = {}) {
118
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
119
+ return this.getResource(schemas.RequestQueue(), {}, this.#resolveTimeout(timeoutSecs, 'short'), signal);
107
120
  }
108
121
  /**
109
122
  * Updates the Request queue with specified fields.
110
123
  *
111
124
  * @param newFields - Fields to update in the Request queue
125
+ * @param options - Request options
126
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
112
127
  * @returns The updated RequestQueue object
113
128
  * @see https://docs.apify.com/api/v2/request-queue-put
114
129
  */
115
- async update(newFields) {
130
+ async update(newFields, options = {}) {
116
131
  parseArgument(newFields, anyObjectSchema);
117
- return this._update(schemas.RequestQueue(), newFields, SMALL_TIMEOUT_MILLIS);
132
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
133
+ return this.updateResource(schemas.RequestQueue(), newFields, this.#resolveTimeout(timeoutSecs, 'short'), signal);
118
134
  }
119
135
  /**
120
136
  * Deletes the Request queue.
121
137
  *
138
+ * @param options - Request options
139
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
122
140
  * @see https://docs.apify.com/api/v2/request-queue-delete
123
141
  */
124
- async delete() {
125
- return this._delete(SMALL_TIMEOUT_MILLIS);
142
+ async delete(options = {}) {
143
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
144
+ return this.deleteResource(this.#resolveTimeout(timeoutSecs, 'short'), signal);
126
145
  }
127
146
  /**
128
147
  * Lists requests from the beginning of the queue (head).
@@ -131,18 +150,21 @@ export class RequestQueueClient extends ResourceClient {
131
150
  * inspecting what requests are waiting to be processed.
132
151
  *
133
152
  * @param options - Options for listing (e.g., limit)
153
+ * @param options.limit - Maximum number of requests to return.
154
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
134
155
  * @returns List of requests from the queue head
135
156
  * @see https://docs.apify.com/api/v2/request-queue-head-get
136
157
  */
137
158
  async listHead(options = {}) {
138
159
  const parsed = parseArgument(options, listHeadOptionsSchema, 'RequestQueueClientListHeadOptions');
139
160
  const response = await this.httpClient.call({
140
- url: this._url('head'),
161
+ url: this.buildUrl('head'),
141
162
  method: 'GET',
142
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
143
- params: this._params({
163
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
164
+ signal: parsed.signal,
165
+ params: this.buildParams({
144
166
  limit: parsed.limit,
145
- clientKey: this.clientKey,
167
+ clientKey: this.#clientKey,
146
168
  }),
147
169
  });
148
170
  return parseResponse(response, schemas.RequestQueueHead());
@@ -159,6 +181,7 @@ export class RequestQueueClient extends ResourceClient {
159
181
  * @param options - Lock configuration
160
182
  * @param options.lockSecs - **Required.** Duration in seconds to lock the requests. After this time, the locks expire and requests can be retrieved by other clients.
161
183
  * @param options.limit - Maximum number of requests to return. Default is 25.
184
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
162
185
  * @returns Object containing `items` (locked requests), `queueModifiedAt`, `hadMultipleClients`, and lock information
163
186
  * @see https://docs.apify.com/api/v2/request-queue-head-lock-post
164
187
  *
@@ -183,13 +206,14 @@ export class RequestQueueClient extends ResourceClient {
183
206
  async listAndLockHead(options) {
184
207
  const parsed = parseArgument(options, listAndLockHeadOptionsSchema, 'RequestQueueClientListAndLockHeadOptions');
185
208
  const response = await this.httpClient.call({
186
- url: this._url('head/lock'),
209
+ url: this.buildUrl('head/lock'),
187
210
  method: 'POST',
188
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
189
- params: this._params({
211
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
212
+ signal: parsed.signal,
213
+ params: this.buildParams({
190
214
  limit: parsed.limit,
191
215
  lockSecs: parsed.lockSecs,
192
- clientKey: this.clientKey,
216
+ clientKey: this.#clientKey,
193
217
  }),
194
218
  });
195
219
  return parseResponse(response, schemas.LockedRequestQueueHead());
@@ -210,6 +234,7 @@ export class RequestQueueClient extends ResourceClient {
210
234
  * @param request.payload - HTTP payload for POST/PUT requests (string).
211
235
  * @param options - Additional options
212
236
  * @param options.forefront - If `true`, adds the request to the beginning of the queue. Default is `false` (adds to the end).
237
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
213
238
  * @returns Object with `requestId`, `wasAlreadyPresent`, and `wasAlreadyHandled` flags
214
239
  * @see https://docs.apify.com/api/v2/request-queue-requests-post
215
240
  *
@@ -236,13 +261,14 @@ export class RequestQueueClient extends ResourceClient {
236
261
  parseArgument(request, newRequestSchema);
237
262
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
238
263
  const response = await this.httpClient.call({
239
- url: this._url('requests'),
264
+ url: this.buildUrl('requests'),
240
265
  method: 'POST',
241
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
266
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
267
+ signal: parsed.signal,
242
268
  data: request,
243
- params: this._params({
269
+ params: this.buildParams({
244
270
  forefront: parsed.forefront,
245
- clientKey: this.clientKey,
271
+ clientKey: this.#clientKey,
246
272
  }),
247
273
  });
248
274
  return parseResponse(response, schemas.RequestRegistration());
@@ -252,23 +278,26 @@ export class RequestQueueClient extends ResourceClient {
252
278
  *
253
279
  * @private
254
280
  */
255
- async _batchAddRequests(requests, options = {}) {
256
- parseArgument(requests, batchAddRequestsSchema);
281
+ async addRequestBatch(requests, options = {}) {
257
282
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
258
283
  const response = await this.httpClient.call({
259
- url: this._url('requests/batch'),
284
+ url: this.buildUrl('requests/batch'),
260
285
  method: 'POST',
261
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
262
- data: requests,
263
- params: this._params({
286
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
287
+ signal: parsed.signal,
288
+ // The body is assembled from the requests as `batchAddRequests` serialized them; the explicit content type
289
+ // makes the request interceptor send the string as it is instead of serializing the requests again.
290
+ headers: { 'content-type': 'application/json' },
291
+ data: `[${requests.map(({ json }) => json).join(',')}]`,
292
+ params: this.buildParams({
264
293
  forefront: parsed.forefront,
265
- clientKey: this.clientKey,
294
+ clientKey: this.#clientKey,
266
295
  }),
267
296
  });
268
297
  return parseResponse(response, schemas.BatchAddResult());
269
298
  }
270
- async _batchAddRequestsWithRetries(requests, options = {}) {
271
- const { forefront, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
299
+ async addRequestBatchWithRetries(requests, options = {}) {
300
+ const { forefront, timeoutSecs, signal, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
272
301
  // Keep track of the requests that remain to be processed (in parameter format)
273
302
  let remainingRequests = requests;
274
303
  // Keep track of the requests that have been processed (in api format)
@@ -278,8 +307,10 @@ export class RequestQueueClient extends ResourceClient {
278
307
  let unprocessedRequests = [];
279
308
  for (let i = 0; i < 1 + maxUnprocessedRequestsRetries; i++) {
280
309
  try {
281
- const response = await this._batchAddRequests(remainingRequests, {
310
+ const response = await this.addRequestBatch(remainingRequests, {
282
311
  forefront,
312
+ timeoutSecs,
313
+ signal,
283
314
  });
284
315
  processedRequests.push(...response.processedRequests);
285
316
  unprocessedRequests = response.unprocessedRequests;
@@ -291,13 +322,15 @@ export class RequestQueueClient extends ResourceClient {
291
322
  // Get unique keys of all requests processed so far
292
323
  const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
293
324
  // Requests remaining to be processed are the all that remain
294
- remainingRequests = requests.filter(({ uniqueKey }) => !processedRequestsUniqueKeys.includes(uniqueKey));
325
+ remainingRequests = requests.filter(({ request }) => !processedRequestsUniqueKeys.includes(request.uniqueKey));
295
326
  // Stop if all requests have been processed
296
327
  if (remainingRequests.length === 0) {
297
328
  break;
298
329
  }
299
330
  }
300
331
  catch (err) {
332
+ // An abort rejects the whole call, since the caller asked to stop and not for a partial result.
333
+ signal?.throwIfAborted();
301
334
  // A response the specification does not describe is reported, as everywhere else: the server may well
302
335
  // have added the batch, so calling it unprocessed would hide the mismatch behind a wrong answer.
303
336
  if (err instanceof ResponseValidationError)
@@ -307,18 +340,16 @@ export class RequestQueueClient extends ResourceClient {
307
340
  // This ensures that this method does not throw and keeps the signature.
308
341
  const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
309
342
  unprocessedRequests = requests
310
- .filter(({ uniqueKey }) => !processedRequestsUniqueKeys.includes(uniqueKey))
311
- .map(({ method, uniqueKey, url }) => ({ method, uniqueKey, url }));
343
+ .filter(({ request }) => !processedRequestsUniqueKeys.includes(request.uniqueKey))
344
+ .map(({ request: { method, uniqueKey, url } }) => ({ method, uniqueKey, url }));
312
345
  break;
313
346
  }
314
347
  // Exponential backoff
315
348
  const delayMillis = Math.floor((1 + Math.random()) * 2 ** i * minDelayBetweenUnprocessedRequestsRetriesMillis);
316
- await new Promise((resolve) => {
317
- setTimeout(resolve, delayMillis);
318
- });
349
+ await sleep(delayMillis, signal);
350
+ signal?.throwIfAborted();
319
351
  }
320
- const result = { processedRequests, unprocessedRequests };
321
- return cast(parseDateFields(result));
352
+ return { processedRequests, unprocessedRequests };
322
353
  }
323
354
  /**
324
355
  * Adds multiple requests to the queue in a single operation.
@@ -335,6 +366,7 @@ export class RequestQueueClient extends ResourceClient {
335
366
  * @param options.maxUnprocessedRequestsRetries - Maximum number of retry attempts for rate-limited requests. Default is 3.
336
367
  * @param options.maxParallel - Maximum number of parallel batch API calls. Default is 5.
337
368
  * @param options.minDelayBetweenUnprocessedRequestsRetriesMillis - Minimum delay before retrying rate-limited requests. Default is 500ms.
369
+ * @param options.timeoutSecs - Timeout for each batch API request. Default is `'medium'`.
338
370
  * @returns Object with `processedRequests` (successfully added) and `unprocessedRequests` (failed after all retries)
339
371
  * @see https://docs.apify.com/api/v2/request-queue-requests-batch-post
340
372
  *
@@ -359,21 +391,36 @@ export class RequestQueueClient extends ResourceClient {
359
391
  * @since Added in 2.1.0
360
392
  */
361
393
  async batchAddRequests(requests, options = {}) {
362
- const { forefront, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, maxParallel = DEFAULT_PARALLEL_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
394
+ const { forefront, timeoutSecs, signal, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, maxParallel = DEFAULT_PARALLEL_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
363
395
  parseArgument(requests, batchAddRequestsWithRetriesSchema);
364
396
  parseArgument(forefront, optionalBooleanSchema);
397
+ parseArgument(timeoutSecs, optionalTimeoutSchema);
398
+ parseArgument(signal, optionalSignalSchema);
365
399
  parseArgument(maxUnprocessedRequestsRetries, optionalNumberSchema);
366
400
  parseArgument(maxParallel, optionalNumberSchema);
367
401
  parseArgument(minDelayBetweenUnprocessedRequestsRetriesMillis, optionalNumberSchema);
368
402
  const executingRequests = new Set();
369
403
  const individualResults = [];
370
404
  const payloadSizeLimitBytes = MAX_PAYLOAD_SIZE_BYTES - Math.ceil(MAX_PAYLOAD_SIZE_BYTES * SAFETY_BUFFER_PERCENT);
405
+ // Serialize every request once: the byte lengths decide the batch boundaries, and the same strings are joined
406
+ // into the batch bodies, so nothing is stringified a second time when it is sent.
407
+ const serializedRequests = requests.map((request, index) => {
408
+ const json = JSON.stringify(request);
409
+ const byteLength = utf8ByteLength(json);
410
+ // Two more bytes for the brackets, which even a batch of one request carries.
411
+ if (byteLength + 2 > payloadSizeLimitBytes) {
412
+ throw new Error(`RequestQueueClient.batchAddRequests: The size of the request with index: ${index} ` +
413
+ `exceeds the maximum allowed size (${payloadSizeLimitBytes} bytes).`);
414
+ }
415
+ return { request, json, byteLength };
416
+ });
417
+ const batches = splitIntoJsonArrayBatches(serializedRequests, {
418
+ maxCount: REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION,
419
+ maxByteLength: payloadSizeLimitBytes,
420
+ });
371
421
  // Keep a pool of up to `maxParallel` requests running at once
372
- let i = 0;
373
- while (i < requests.length) {
374
- const slicedRequests = requests.slice(i, i + REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION);
375
- const requestsInBatch = sliceArrayByByteLength(slicedRequests, payloadSizeLimitBytes, i);
376
- const requestPromise = this._batchAddRequestsWithRetries(requestsInBatch, options);
422
+ for (const batch of batches) {
423
+ const requestPromise = this.addRequestBatchWithRetries(batch, options);
377
424
  executingRequests.add(requestPromise);
378
425
  // A rejection reaches the caller through the awaits below; this bookkeeping chain only has to avoid
379
426
  // turning it into an unhandled one of its own.
@@ -384,7 +431,6 @@ export class RequestQueueClient extends ResourceClient {
384
431
  if (executingRequests.size >= maxParallel) {
385
432
  await Promise.race(executingRequests);
386
433
  }
387
- i += requestsInBatch.length;
388
434
  }
389
435
  // Get results from remaining operations
390
436
  await Promise.all(executingRequests);
@@ -405,19 +451,23 @@ export class RequestQueueClient extends ResourceClient {
405
451
  * Requests can be identified by either their ID or unique key.
406
452
  *
407
453
  * @param requests - Array of requests to delete (by id or uniqueKey)
454
+ * @param options - Request options
455
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
408
456
  * @returns Result containing processed and unprocessed requests
409
457
  * @see https://docs.apify.com/api/v2/request-queue-requests-batch-delete
410
458
  * @since Added in 2.3.0
411
459
  */
412
- async batchDeleteRequests(requests) {
460
+ async batchDeleteRequests(requests, options = {}) {
413
461
  parseArgument(requests, batchDeleteRequestsSchema);
462
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
414
463
  const response = await this.httpClient.call({
415
- url: this._url('requests/batch'),
464
+ url: this.buildUrl('requests/batch'),
416
465
  method: 'DELETE',
417
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
466
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
467
+ signal,
418
468
  data: requests,
419
- params: this._params({
420
- clientKey: this.clientKey,
469
+ params: this.buildParams({
470
+ clientKey: this.#clientKey,
421
471
  }),
422
472
  });
423
473
  return parseResponse(response, schemas.BatchDeleteResult());
@@ -426,16 +476,20 @@ export class RequestQueueClient extends ResourceClient {
426
476
  * Gets a specific request from the queue by its ID.
427
477
  *
428
478
  * @param id - Request ID
479
+ * @param options - Request options
480
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
429
481
  * @returns The request object, or `undefined` if not found
430
482
  * @see https://docs.apify.com/api/v2/request-queue-request-get
431
483
  */
432
- async getRequest(id) {
484
+ async getRequest(id, options = {}) {
433
485
  parseArgument(id, requestIdSchema);
486
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
434
487
  const requestOpts = {
435
- url: this._url(['requests', id]),
488
+ url: this.buildUrl(['requests', id]),
436
489
  method: 'GET',
437
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
438
- params: this._params(),
490
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
491
+ signal,
492
+ params: this.buildParams(),
439
493
  };
440
494
  try {
441
495
  const response = await this.httpClient.call(requestOpts);
@@ -451,6 +505,8 @@ export class RequestQueueClient extends ResourceClient {
451
505
  *
452
506
  * @param request - The updated request object (must include id)
453
507
  * @param options - Update options such as whether to move to front
508
+ * @param options.forefront - If `true`, moves the request to the beginning of the queue. Default is `false`.
509
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
454
510
  * @returns Information about the updated request
455
511
  * @see https://docs.apify.com/api/v2/request-queue-request-put
456
512
  */
@@ -458,13 +514,14 @@ export class RequestQueueClient extends ResourceClient {
458
514
  parseArgument(request, existingRequestSchema);
459
515
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
460
516
  const response = await this.httpClient.call({
461
- url: this._url(['requests', request.id]),
517
+ url: this.buildUrl(['requests', request.id]),
462
518
  method: 'PUT',
463
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
519
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
520
+ signal: parsed.signal,
464
521
  data: request,
465
- params: this._params({
522
+ params: this.buildParams({
466
523
  forefront: parsed.forefront,
467
- clientKey: this.clientKey,
524
+ clientKey: this.#clientKey,
468
525
  }),
469
526
  });
470
527
  return parseResponse(response, schemas.RequestRegistration());
@@ -473,15 +530,19 @@ export class RequestQueueClient extends ResourceClient {
473
530
  * Deletes a specific request from the queue.
474
531
  *
475
532
  * @param id - Request ID
533
+ * @param options - Request options
534
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
476
535
  */
477
- async deleteRequest(id) {
536
+ async deleteRequest(id, options = {}) {
478
537
  parseArgument(id, requestIdSchema);
538
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
479
539
  await this.httpClient.call({
480
- url: this._url(['requests', id]),
540
+ url: this.buildUrl(['requests', id]),
481
541
  method: 'DELETE',
482
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
483
- params: this._params({
484
- clientKey: this.clientKey,
542
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
543
+ signal,
544
+ params: this.buildParams({
545
+ clientKey: this.#clientKey,
485
546
  }),
486
547
  });
487
548
  }
@@ -496,6 +557,7 @@ export class RequestQueueClient extends ResourceClient {
496
557
  * @param options - Lock extension options
497
558
  * @param options.lockSecs - **Required.** New lock duration in seconds from now.
498
559
  * @param options.forefront - If `true`, moves the request to the beginning of the queue when the lock expires. Default is `false`.
560
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
499
561
  * @returns Object with new `lockExpiresAt` timestamp
500
562
  * @see https://docs.apify.com/api/v2/request-queue-request-lock-put
501
563
  *
@@ -514,13 +576,14 @@ export class RequestQueueClient extends ResourceClient {
514
576
  parseArgument(id, requestIdSchema);
515
577
  const parsed = parseArgument(options, prolongRequestLockOptionsSchema, 'RequestQueueClientProlongRequestLockOptions');
516
578
  const response = await this.httpClient.call({
517
- url: this._url(['requests', id, 'lock']),
579
+ url: this.buildUrl(['requests', id, 'lock']),
518
580
  method: 'PUT',
519
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
520
- params: this._params({
581
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
582
+ signal: parsed.signal,
583
+ params: this.buildParams({
521
584
  forefront: parsed.forefront,
522
585
  lockSecs: parsed.lockSecs,
523
- clientKey: this.clientKey,
586
+ clientKey: this.#clientKey,
524
587
  }),
525
588
  });
526
589
  return parseResponse(response, schemas.RequestLockInfo());
@@ -533,6 +596,8 @@ export class RequestQueueClient extends ResourceClient {
533
596
  *
534
597
  * @param id - Request ID
535
598
  * @param options - Options such as whether to move to front
599
+ * @param options.forefront - If `true`, moves the request to the beginning of the queue. Default is `false`.
600
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
536
601
  * @see https://docs.apify.com/api/v2/request-queue-request-lock-delete
537
602
  * @since Added in 2.4.1
538
603
  */
@@ -540,12 +605,13 @@ export class RequestQueueClient extends ResourceClient {
540
605
  parseArgument(id, requestIdSchema);
541
606
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientDeleteRequestLockOptions');
542
607
  await this.httpClient.call({
543
- url: this._url(['requests', id, 'lock']),
608
+ url: this.buildUrl(['requests', id, 'lock']),
544
609
  method: 'DELETE',
545
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
546
- params: this._params({
610
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
611
+ signal: parsed.signal,
612
+ params: this.buildParams({
547
613
  forefront: parsed.forefront,
548
- clientKey: this.clientKey,
614
+ clientKey: this.#clientKey,
549
615
  }),
550
616
  });
551
617
  }
@@ -556,21 +622,26 @@ export class RequestQueueClient extends ResourceClient {
556
622
  * queue contents.
557
623
  *
558
624
  * @param options - Pagination options
625
+ * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
559
626
  * @returns List of requests with pagination information
560
627
  * @see https://docs.apify.com/api/v2/request-queue-requests-get
561
628
  * @since Added in 2.5.1
562
629
  */
563
630
  listRequests(options = {}) {
564
- const parsed = parseArgument(options, listRequestsOptionsSchema, 'RequestQueueClientListRequestsOptions');
631
+ // `timeoutSecs` and `signal` apply to every page request; they are not API parameters, so they must not reach
632
+ // the query string.
633
+ const { timeoutSecs, signal, ...parsed } = parseArgument(options, listRequestsOptionsSchema, 'RequestQueueClientListRequestsOptions');
634
+ const pageTimeout = this.#resolveTimeout(timeoutSecs, 'medium');
565
635
  const getPaginatedList = async (rqListOptions = {}) => {
566
636
  const response = await this.httpClient.call({
567
- url: this._url('requests'),
637
+ url: this.buildUrl('requests'),
568
638
  method: 'GET',
569
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
570
- params: this._params({
639
+ timeoutSecs: pageTimeout,
640
+ signal,
641
+ params: this.buildParams({
571
642
  ...rqListOptions,
572
643
  filter: rqListOptions.filter ? rqListOptions.filter.join(',') : undefined,
573
- clientKey: this.clientKey,
644
+ clientKey: this.#clientKey,
574
645
  }),
575
646
  });
576
647
  return parseResponse(response, schemas.ListOfRequests());
@@ -589,8 +660,6 @@ export class RequestQueueClient extends ResourceClient {
589
660
  const newOptions = {
590
661
  ...parsed,
591
662
  limit: remainingItems,
592
- // remove original exclusiveStartId, if there was any, and use cursor-based pagination
593
- exclusiveStartId: undefined,
594
663
  cursor: currentPage.nextCursor,
595
664
  };
596
665
  currentPage = await getPaginatedList(newOptions);
@@ -610,17 +679,21 @@ export class RequestQueueClient extends ResourceClient {
610
679
  * This is useful for releasing all locks at once, for example when shutting down
611
680
  * a crawler gracefully.
612
681
  *
682
+ * @param options - Request options
683
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
613
684
  * @returns Number of requests that were unlocked
614
685
  * @see https://docs.apify.com/api/v2/request-queue-requests-unlock-post
615
686
  * @since Added in 2.12.5
616
687
  */
617
- async unlockRequests() {
688
+ async unlockRequests(options = {}) {
689
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
618
690
  const response = await this.httpClient.call({
619
- url: this._url('requests/unlock'),
691
+ url: this.buildUrl('requests/unlock'),
620
692
  method: 'POST',
621
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
622
- params: this._params({
623
- clientKey: this.clientKey,
693
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'long'),
694
+ signal,
695
+ params: this.buildParams({
696
+ clientKey: this.#clientKey,
624
697
  }),
625
698
  });
626
699
  return parseResponse(response, schemas.UnlockRequestsResult());
@@ -632,6 +705,7 @@ export class RequestQueueClient extends ResourceClient {
632
705
  * automatically handling pagination behind the scenes.
633
706
  *
634
707
  * @param options - Pagination options
708
+ * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
635
709
  * @returns An async iterable of request pages
636
710
  * @see https://docs.apify.com/api/v2/request-queue-requests-get
637
711
  *
@@ -644,11 +718,10 @@ export class RequestQueueClient extends ResourceClient {
644
718
  * @since Added in 2.5.1
645
719
  */
646
720
  paginateRequests(options = {}) {
647
- const { limit, exclusiveStartId, cursor, filter, maxPageLimit } = parseArgument(options, paginateRequestsOptionsSchema, 'RequestQueueClientPaginateRequestsOptions');
721
+ const { limit, cursor, filter, maxPageLimit, timeoutSecs, signal } = parseArgument(options, paginateRequestsOptionsSchema, 'RequestQueueClientPaginateRequestsOptions');
648
722
  return new RequestQueuePaginationIterator({
649
- getPage: async (pageOptions) => this.listRequests({ ...pageOptions, filter }),
723
+ getPage: async (pageOptions) => this.listRequests({ ...pageOptions, filter, timeoutSecs, signal }),
650
724
  limit,
651
- exclusiveStartId,
652
725
  cursor,
653
726
  maxPageLimit,
654
727
  });
@@ -1,6 +1,7 @@
1
1
  import { STORAGE_OWNERSHIP_FILTER } from '@apify/consts';
2
2
  import type { ApiClientSubResourceOptions } from '../base/api_client.js';
3
3
  import { ResourceCollectionClient } from '../base/resource_collection_client.js';
4
+ import type { TimeoutOptions } from '../timeouts.js';
4
5
  import type { PaginatedList, PaginationOptions } from '../utils.js';
5
6
  import type { RequestQueue } from './request_queue.js';
6
7
  /**
@@ -45,6 +46,7 @@ export declare class RequestQueueCollectionClient extends ResourceCollectionClie
45
46
  * ```
46
47
  *
47
48
  * @param options - Pagination options.
49
+ * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
48
50
  * @returns A paginated iterator of Request queues.
49
51
  * @see https://docs.apify.com/api/v2/request-queues-get
50
52
  */
@@ -53,12 +55,14 @@ export declare class RequestQueueCollectionClient extends ResourceCollectionClie
53
55
  * Gets or creates a Request queue with the specified name.
54
56
  *
55
57
  * @param name - Name of the Request queue. If not provided, a default queue is used.
58
+ * @param options - Request options
59
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
56
60
  * @returns The Request queue object.
57
61
  * @see https://docs.apify.com/api/v2/request-queues-post
58
62
  */
59
- getOrCreate(name?: string): Promise<RequestQueue>;
63
+ getOrCreate(name?: string, options?: TimeoutOptions): Promise<RequestQueue>;
60
64
  }
61
- export interface RequestQueueCollectionListOptions extends PaginationOptions {
65
+ export interface RequestQueueCollectionListOptions extends PaginationOptions, TimeoutOptions {
62
66
  unnamed?: boolean;
63
67
  desc?: boolean;
64
68
  /**