apify-client 3.0.0-beta.4 → 3.0.0-beta.41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/README.md +13 -3
  2. package/dist/apify_api_error.d.ts +62 -21
  3. package/dist/apify_api_error.js +96 -23
  4. package/dist/apify_client.d.ts +125 -10
  5. package/dist/apify_client.js +133 -48
  6. package/dist/base/api_client.d.ts +7 -7
  7. package/dist/base/api_client.js +24 -16
  8. package/dist/base/resource_client.d.ts +23 -9
  9. package/dist/base/resource_client.js +57 -29
  10. package/dist/base/resource_collection_client.d.ts +8 -4
  11. package/dist/base/resource_collection_client.js +26 -15
  12. package/dist/body_parser.d.ts +3 -4
  13. package/dist/body_parser.js +19 -26
  14. package/dist/bundle.js +29 -22
  15. package/dist/bundle.js.map +1 -1
  16. package/dist/generated/api.d.ts +887 -322
  17. package/dist/generated/schemas.d.ts +6717 -0
  18. package/dist/generated/schemas.js +1527 -0
  19. package/dist/http_clients/axios.d.ts +60 -0
  20. package/dist/http_clients/axios.js +117 -0
  21. package/dist/http_clients/base.d.ts +333 -0
  22. package/dist/http_clients/base.js +563 -0
  23. package/dist/http_clients/index.d.ts +4 -0
  24. package/dist/http_clients/index.js +2 -0
  25. package/dist/http_compressors/base.d.ts +40 -0
  26. package/dist/http_compressors/base.js +9 -0
  27. package/dist/http_compressors/brotli.d.ts +33 -0
  28. package/dist/http_compressors/brotli.js +39 -0
  29. package/dist/http_compressors/gzip.d.ts +33 -0
  30. package/dist/http_compressors/gzip.js +39 -0
  31. package/dist/http_compressors/index.d.ts +6 -0
  32. package/dist/http_compressors/index.js +2 -0
  33. package/dist/http_compressors/resolve.d.ts +23 -0
  34. package/dist/http_compressors/resolve.js +27 -0
  35. package/dist/index.d.ts +5 -1
  36. package/dist/index.js +4 -1
  37. package/dist/invalid_response_body_error.d.ts +13 -0
  38. package/dist/invalid_response_body_error.js +17 -0
  39. package/dist/lazy_schema.d.ts +8 -0
  40. package/dist/lazy_schema.js +11 -0
  41. package/dist/models.d.ts +33 -61
  42. package/dist/resource_clients/actor.d.ts +52 -31
  43. package/dist/resource_clients/actor.js +81 -58
  44. package/dist/resource_clients/actor_collection.d.ts +9 -5
  45. package/dist/resource_clients/actor_collection.js +12 -3
  46. package/dist/resource_clients/actor_env_var.d.ts +10 -3
  47. package/dist/resource_clients/actor_env_var.js +17 -6
  48. package/dist/resource_clients/actor_env_var_collection.d.ts +14 -18
  49. package/dist/resource_clients/actor_env_var_collection.js +18 -11
  50. package/dist/resource_clients/actor_version.d.ts +18 -3
  51. package/dist/resource_clients/actor_version.js +20 -9
  52. package/dist/resource_clients/actor_version_collection.d.ts +16 -17
  53. package/dist/resource_clients/actor_version_collection.js +20 -11
  54. package/dist/resource_clients/build.d.ts +18 -6
  55. package/dist/resource_clients/build.js +37 -17
  56. package/dist/resource_clients/build_collection.d.ts +3 -1
  57. package/dist/resource_clients/build_collection.js +5 -1
  58. package/dist/resource_clients/dataset.d.ts +49 -31
  59. package/dist/resource_clients/dataset.js +102 -76
  60. package/dist/resource_clients/dataset_collection.d.ts +6 -2
  61. package/dist/resource_clients/dataset_collection.js +14 -2
  62. package/dist/resource_clients/key_value_store.d.ts +58 -16
  63. package/dist/resource_clients/key_value_store.js +80 -41
  64. package/dist/resource_clients/key_value_store_collection.d.ts +6 -2
  65. package/dist/resource_clients/key_value_store_collection.js +14 -2
  66. package/dist/resource_clients/log.d.ts +11 -19
  67. package/dist/resource_clients/log.js +64 -46
  68. package/dist/resource_clients/request_queue.d.ts +56 -26
  69. package/dist/resource_clients/request_queue.js +197 -116
  70. package/dist/resource_clients/request_queue_collection.d.ts +6 -2
  71. package/dist/resource_clients/request_queue_collection.js +10 -3
  72. package/dist/resource_clients/run.d.ts +53 -18
  73. package/dist/resource_clients/run.js +94 -47
  74. package/dist/resource_clients/run_collection.d.ts +3 -1
  75. package/dist/resource_clients/run_collection.js +5 -1
  76. package/dist/resource_clients/schedule.d.ts +16 -7
  77. package/dist/resource_clients/schedule.js +32 -21
  78. package/dist/resource_clients/schedule_collection.d.ts +6 -2
  79. package/dist/resource_clients/schedule_collection.js +10 -3
  80. package/dist/resource_clients/store_collection.d.ts +3 -1
  81. package/dist/resource_clients/store_collection.js +5 -1
  82. package/dist/resource_clients/task.d.ts +28 -10
  83. package/dist/resource_clients/task.js +71 -46
  84. package/dist/resource_clients/task_collection.d.ts +6 -2
  85. package/dist/resource_clients/task_collection.js +10 -3
  86. package/dist/resource_clients/user.d.ts +17 -8
  87. package/dist/resource_clients/user.js +43 -39
  88. package/dist/resource_clients/webhook.d.ts +14 -5
  89. package/dist/resource_clients/webhook.js +31 -22
  90. package/dist/resource_clients/webhook_collection.d.ts +6 -2
  91. package/dist/resource_clients/webhook_collection.js +10 -3
  92. package/dist/resource_clients/webhook_dispatch.d.ts +4 -1
  93. package/dist/resource_clients/webhook_dispatch.js +8 -2
  94. package/dist/resource_clients/webhook_dispatch_collection.d.ts +3 -1
  95. package/dist/resource_clients/webhook_dispatch_collection.js +5 -1
  96. package/dist/response_validation_error.d.ts +26 -0
  97. package/dist/response_validation_error.js +37 -0
  98. package/dist/runtime/node.d.ts +6 -0
  99. package/dist/runtime/node.js +49 -0
  100. package/dist/runtime/types.d.ts +52 -0
  101. package/dist/runtime/types.js +1 -0
  102. package/dist/runtime/web.d.ts +5 -0
  103. package/dist/runtime/web.js +18 -0
  104. package/dist/schemas.d.ts +15 -0
  105. package/dist/schemas.js +15 -0
  106. package/dist/statistics.d.ts +1 -6
  107. package/dist/statistics.js +2 -2
  108. package/dist/timeouts.d.ts +84 -0
  109. package/dist/timeouts.js +37 -0
  110. package/dist/timezones.d.ts +1 -1
  111. package/dist/utils.d.ts +86 -46
  112. package/dist/utils.js +218 -164
  113. package/package.json +32 -29
  114. package/dist/http_client.d.ts +0 -88
  115. package/dist/http_client.js +0 -255
  116. package/dist/interceptors.d.ts +0 -19
  117. package/dist/interceptors.js +0 -105
@@ -1,23 +1,26 @@
1
1
  import { z } from 'zod';
2
2
  import { MAX_PAYLOAD_SIZE_BYTES, REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION } from '@apify/consts';
3
3
  import log from '@apify/log';
4
- import { MEDIUM_TIMEOUT_MILLIS, ResourceClient, SMALL_TIMEOUT_MILLIS } from '../base/resource_client.js';
5
- import { anyObjectSchema, cast, catchNotFoundOrThrow, isNonArrayObject, mutuallyExclusive, parseArgument, parseDateFields, pluckData, RequestQueuePaginationIterator, sliceArrayByByteLength, } from '../utils.js';
4
+ import { ResourceClient } from '../base/resource_client.js';
5
+ import { ResponseValidationError } from '../response_validation_error.js';
6
+ import * as schemas from '../schemas.js';
7
+ import { optionalSignalSchema, optionalTimeoutSchema, timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
8
+ import { anyObjectSchema, catchNotFoundOrThrow, isNonArrayObject, parseArgument, parseResponse, RequestQueuePaginationIterator, sleep, splitIntoJsonArrayBatches, utf8ByteLength, } from '../utils.js';
6
9
  const DEFAULT_PARALLEL_BATCH_ADD_REQUESTS = 5;
7
10
  const DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS = 3;
8
11
  const DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS = 500;
9
12
  const DEFAULT_REQUEST_QUEUE_REQUEST_PAGE_LIMIT = 1000;
10
13
  const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
11
- const listHeadOptionsSchema = z.strictObject({ limit: z.number().min(0).optional() });
14
+ const listHeadOptionsSchema = z.strictObject({ limit: z.number().min(0).optional(), ...timeoutOptionsShape });
12
15
  const listAndLockHeadOptionsSchema = z.strictObject({
13
16
  lockSecs: z.number(),
14
17
  limit: z.number().min(0).optional(),
18
+ ...timeoutOptionsShape,
15
19
  });
16
20
  // Predicates, not `z.looseObject` arms: these run over a whole batch, and an object arm would copy
17
21
  // every key of every request. `id` is assigned by the API, so a new request must not carry one.
18
22
  const newRequestSchema = z.custom((value) => isNonArrayObject(value) && value.id === undefined, 'Expected a request object without an `id`');
19
- const forefrontOptionsSchema = z.strictObject({ forefront: z.boolean().optional() });
20
- const batchAddRequestsSchema = z.array(newRequestSchema).min(1).max(REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION);
23
+ const forefrontOptionsSchema = z.strictObject({ forefront: z.boolean().optional(), ...timeoutOptionsShape });
21
24
  const batchAddRequestsWithRetriesSchema = z.array(newRequestSchema).min(1);
22
25
  const optionalBooleanSchema = z.boolean().optional();
23
26
  const optionalNumberSchema = z.number().optional();
@@ -31,25 +34,22 @@ const existingRequestSchema = z.custom((value) => isNonArrayObject(value) && typ
31
34
  const prolongRequestLockOptionsSchema = z.strictObject({
32
35
  lockSecs: z.number(),
33
36
  forefront: z.boolean().optional(),
37
+ ...timeoutOptionsShape,
34
38
  });
35
39
  const requestFilterSchema = z.array(z.enum(['locked', 'pending'])).min(1);
36
- const listRequestsOptionsSchema = z
37
- .strictObject({
40
+ const listRequestsOptionsSchema = z.strictObject({
38
41
  limit: z.number().min(0).optional(),
39
- exclusiveStartId: z.string().optional(),
40
42
  cursor: z.string().optional(),
41
43
  filter: requestFilterSchema.optional(),
42
- })
43
- .refine(...mutuallyExclusive('exclusiveStartId', 'cursor'));
44
- const paginateRequestsOptionsSchema = z
45
- .strictObject({
44
+ ...timeoutOptionsShape,
45
+ });
46
+ const paginateRequestsOptionsSchema = z.strictObject({
46
47
  limit: z.number().min(0).optional(),
47
48
  maxPageLimit: z.number().default(DEFAULT_REQUEST_QUEUE_REQUEST_PAGE_LIMIT),
48
- exclusiveStartId: z.string().optional(),
49
49
  cursor: z.string().optional(),
50
50
  filter: requestFilterSchema.optional(),
51
- })
52
- .refine(...mutuallyExclusive('exclusiveStartId', 'cursor'));
51
+ ...timeoutOptionsShape,
52
+ });
53
53
  /**
54
54
  * Client for managing a specific Request queue.
55
55
  *
@@ -81,8 +81,8 @@ const paginateRequestsOptionsSchema = z
81
81
  * @see https://docs.apify.com/platform/storage/request-queue
82
82
  */
83
83
  export class RequestQueueClient extends ResourceClient {
84
- clientKey;
85
- timeoutMillis;
84
+ #clientKey;
85
+ #timeoutSecs;
86
86
  /**
87
87
  * @hidden
88
88
  */
@@ -91,36 +91,57 @@ export class RequestQueueClient extends ResourceClient {
91
91
  resourcePath: 'request-queues',
92
92
  ...options,
93
93
  });
94
- this.clientKey = userOptions.clientKey;
95
- this.timeoutMillis = userOptions.timeoutSecs ? userOptions.timeoutSecs * 1e3 : undefined;
94
+ this.#clientKey = userOptions.clientKey;
95
+ this.#timeoutSecs = userOptions.timeoutSecs;
96
+ }
97
+ /**
98
+ * Picks the timeout of a request: an explicit per-call `timeoutSecs` as is, otherwise the method's default
99
+ * tier, capped at the queue-wide `timeoutSecs` when one was given.
100
+ */
101
+ #resolveTimeout(timeoutSecs, defaultTier) {
102
+ if (timeoutSecs !== undefined)
103
+ return timeoutSecs;
104
+ if (this.#timeoutSecs === undefined)
105
+ return defaultTier;
106
+ const tierSecs = this.httpClient.timeoutMillis[defaultTier] / 1000;
107
+ return tierSecs <= this.#timeoutSecs ? defaultTier : this.#timeoutSecs;
96
108
  }
97
109
  /**
98
110
  * Gets the Request queue object from the Apify API.
99
111
  *
112
+ * @param options - Request options
113
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
100
114
  * @returns The RequestQueue object, or `undefined` if it does not exist
101
115
  * @see https://docs.apify.com/api/v2/request-queue-get
102
116
  */
103
- async get() {
104
- return this._get({}, SMALL_TIMEOUT_MILLIS);
117
+ async get(options = {}) {
118
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
119
+ return this.getResource(schemas.RequestQueue(), {}, this.#resolveTimeout(timeoutSecs, 'short'), signal);
105
120
  }
106
121
  /**
107
122
  * Updates the Request queue with specified fields.
108
123
  *
109
124
  * @param newFields - Fields to update in the Request queue
125
+ * @param options - Request options
126
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
110
127
  * @returns The updated RequestQueue object
111
128
  * @see https://docs.apify.com/api/v2/request-queue-put
112
129
  */
113
- async update(newFields) {
130
+ async update(newFields, options = {}) {
114
131
  parseArgument(newFields, anyObjectSchema);
115
- return this._update(newFields, SMALL_TIMEOUT_MILLIS);
132
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
133
+ return this.updateResource(schemas.RequestQueue(), newFields, this.#resolveTimeout(timeoutSecs, 'short'), signal);
116
134
  }
117
135
  /**
118
136
  * Deletes the Request queue.
119
137
  *
138
+ * @param options - Request options
139
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
120
140
  * @see https://docs.apify.com/api/v2/request-queue-delete
121
141
  */
122
- async delete() {
123
- return this._delete(SMALL_TIMEOUT_MILLIS);
142
+ async delete(options = {}) {
143
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
144
+ return this.deleteResource(this.#resolveTimeout(timeoutSecs, 'short'), signal);
124
145
  }
125
146
  /**
126
147
  * Lists requests from the beginning of the queue (head).
@@ -129,21 +150,24 @@ export class RequestQueueClient extends ResourceClient {
129
150
  * inspecting what requests are waiting to be processed.
130
151
  *
131
152
  * @param options - Options for listing (e.g., limit)
153
+ * @param options.limit - Maximum number of requests to return.
154
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
132
155
  * @returns List of requests from the queue head
133
156
  * @see https://docs.apify.com/api/v2/request-queue-head-get
134
157
  */
135
158
  async listHead(options = {}) {
136
159
  const parsed = parseArgument(options, listHeadOptionsSchema, 'RequestQueueClientListHeadOptions');
137
160
  const response = await this.httpClient.call({
138
- url: this._url('head'),
161
+ url: this.buildUrl('head'),
139
162
  method: 'GET',
140
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
141
- params: this._params({
163
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
164
+ signal: parsed.signal,
165
+ params: this.buildParams({
142
166
  limit: parsed.limit,
143
- clientKey: this.clientKey,
167
+ clientKey: this.#clientKey,
144
168
  }),
145
169
  });
146
- return cast(parseDateFields(pluckData(response.data)));
170
+ return parseResponse(response, schemas.RequestQueueHead());
147
171
  }
148
172
  /**
149
173
  * Gets and locks the next requests from the queue head for processing.
@@ -157,6 +181,7 @@ export class RequestQueueClient extends ResourceClient {
157
181
  * @param options - Lock configuration
158
182
  * @param options.lockSecs - **Required.** Duration in seconds to lock the requests. After this time, the locks expire and requests can be retrieved by other clients.
159
183
  * @param options.limit - Maximum number of requests to return. Default is 25.
184
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
160
185
  * @returns Object containing `items` (locked requests), `queueModifiedAt`, `hadMultipleClients`, and lock information
161
186
  * @see https://docs.apify.com/api/v2/request-queue-head-lock-post
162
187
  *
@@ -181,16 +206,17 @@ export class RequestQueueClient extends ResourceClient {
181
206
  async listAndLockHead(options) {
182
207
  const parsed = parseArgument(options, listAndLockHeadOptionsSchema, 'RequestQueueClientListAndLockHeadOptions');
183
208
  const response = await this.httpClient.call({
184
- url: this._url('head/lock'),
209
+ url: this.buildUrl('head/lock'),
185
210
  method: 'POST',
186
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
187
- params: this._params({
211
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
212
+ signal: parsed.signal,
213
+ params: this.buildParams({
188
214
  limit: parsed.limit,
189
215
  lockSecs: parsed.lockSecs,
190
- clientKey: this.clientKey,
216
+ clientKey: this.#clientKey,
191
217
  }),
192
218
  });
193
- return cast(parseDateFields(pluckData(response.data)));
219
+ return parseResponse(response, schemas.LockedRequestQueueHead());
194
220
  }
195
221
  /**
196
222
  * Adds a single request to the queue.
@@ -208,6 +234,7 @@ export class RequestQueueClient extends ResourceClient {
208
234
  * @param request.payload - HTTP payload for POST/PUT requests (string).
209
235
  * @param options - Additional options
210
236
  * @param options.forefront - If `true`, adds the request to the beginning of the queue. Default is `false` (adds to the end).
237
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
211
238
  * @returns Object with `requestId`, `wasAlreadyPresent`, and `wasAlreadyHandled` flags
212
239
  * @see https://docs.apify.com/api/v2/request-queue-requests-post
213
240
  *
@@ -234,39 +261,43 @@ export class RequestQueueClient extends ResourceClient {
234
261
  parseArgument(request, newRequestSchema);
235
262
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
236
263
  const response = await this.httpClient.call({
237
- url: this._url('requests'),
264
+ url: this.buildUrl('requests'),
238
265
  method: 'POST',
239
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
266
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
267
+ signal: parsed.signal,
240
268
  data: request,
241
- params: this._params({
269
+ params: this.buildParams({
242
270
  forefront: parsed.forefront,
243
- clientKey: this.clientKey,
271
+ clientKey: this.#clientKey,
244
272
  }),
245
273
  });
246
- return cast(parseDateFields(pluckData(response.data)));
274
+ return parseResponse(response, schemas.RequestRegistration());
247
275
  }
248
276
  /**
249
277
  * Writes requests to Request queue in batch.
250
278
  *
251
279
  * @private
252
280
  */
253
- async _batchAddRequests(requests, options = {}) {
254
- parseArgument(requests, batchAddRequestsSchema);
281
+ async addRequestBatch(requests, options = {}) {
255
282
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
256
- const { data } = await this.httpClient.call({
257
- url: this._url('requests/batch'),
283
+ const response = await this.httpClient.call({
284
+ url: this.buildUrl('requests/batch'),
258
285
  method: 'POST',
259
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
260
- data: requests,
261
- params: this._params({
286
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
287
+ signal: parsed.signal,
288
+ // The body is assembled from the requests as `batchAddRequests` serialized them; the explicit content type
289
+ // makes the request interceptor send the string as it is instead of serializing the requests again.
290
+ headers: { 'content-type': 'application/json' },
291
+ data: `[${requests.map(({ json }) => json).join(',')}]`,
292
+ params: this.buildParams({
262
293
  forefront: parsed.forefront,
263
- clientKey: this.clientKey,
294
+ clientKey: this.#clientKey,
264
295
  }),
265
296
  });
266
- return cast(parseDateFields(pluckData(data)));
297
+ return parseResponse(response, schemas.BatchAddResult());
267
298
  }
268
- async _batchAddRequestsWithRetries(requests, options = {}) {
269
- const { forefront, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
299
+ async addRequestBatchWithRetries(requests, options = {}) {
300
+ const { forefront, timeoutSecs, signal, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
270
301
  // Keep track of the requests that remain to be processed (in parameter format)
271
302
  let remainingRequests = requests;
272
303
  // Keep track of the requests that have been processed (in api format)
@@ -276,8 +307,10 @@ export class RequestQueueClient extends ResourceClient {
276
307
  let unprocessedRequests = [];
277
308
  for (let i = 0; i < 1 + maxUnprocessedRequestsRetries; i++) {
278
309
  try {
279
- const response = await this._batchAddRequests(remainingRequests, {
310
+ const response = await this.addRequestBatch(remainingRequests, {
280
311
  forefront,
312
+ timeoutSecs,
313
+ signal,
281
314
  });
282
315
  processedRequests.push(...response.processedRequests);
283
316
  unprocessedRequests = response.unprocessedRequests;
@@ -289,30 +322,34 @@ export class RequestQueueClient extends ResourceClient {
289
322
  // Get unique keys of all requests processed so far
290
323
  const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
291
324
  // Requests remaining to be processed are the all that remain
292
- remainingRequests = requests.filter(({ uniqueKey }) => !processedRequestsUniqueKeys.includes(uniqueKey));
325
+ remainingRequests = requests.filter(({ request }) => !processedRequestsUniqueKeys.includes(request.uniqueKey));
293
326
  // Stop if all requests have been processed
294
327
  if (remainingRequests.length === 0) {
295
328
  break;
296
329
  }
297
330
  }
298
331
  catch (err) {
332
+ // An abort rejects the whole call, since the caller asked to stop and not for a partial result.
333
+ signal?.throwIfAborted();
334
+ // A response the specification does not describe is reported, as everywhere else: the server may well
335
+ // have added the batch, so calling it unprocessed would hide the mismatch behind a wrong answer.
336
+ if (err instanceof ResponseValidationError)
337
+ throw err;
299
338
  log.exception(err, 'Request batch insert failed');
300
339
  // When something fails and http client does not retry, the remaining requests are treated as unprocessed.
301
340
  // This ensures that this method does not throw and keeps the signature.
302
341
  const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
303
342
  unprocessedRequests = requests
304
- .filter(({ uniqueKey }) => !processedRequestsUniqueKeys.includes(uniqueKey))
305
- .map(({ method, uniqueKey, url }) => ({ method, uniqueKey, url }));
343
+ .filter(({ request }) => !processedRequestsUniqueKeys.includes(request.uniqueKey))
344
+ .map(({ request: { method, uniqueKey, url } }) => ({ method, uniqueKey, url }));
306
345
  break;
307
346
  }
308
347
  // Exponential backoff
309
348
  const delayMillis = Math.floor((1 + Math.random()) * 2 ** i * minDelayBetweenUnprocessedRequestsRetriesMillis);
310
- await new Promise((resolve) => {
311
- setTimeout(resolve, delayMillis);
312
- });
349
+ await sleep(delayMillis, signal);
350
+ signal?.throwIfAborted();
313
351
  }
314
- const result = { processedRequests, unprocessedRequests };
315
- return cast(parseDateFields(result));
352
+ return { processedRequests, unprocessedRequests };
316
353
  }
317
354
  /**
318
355
  * Adds multiple requests to the queue in a single operation.
@@ -329,6 +366,7 @@ export class RequestQueueClient extends ResourceClient {
329
366
  * @param options.maxUnprocessedRequestsRetries - Maximum number of retry attempts for rate-limited requests. Default is 3.
330
367
  * @param options.maxParallel - Maximum number of parallel batch API calls. Default is 5.
331
368
  * @param options.minDelayBetweenUnprocessedRequestsRetriesMillis - Minimum delay before retrying rate-limited requests. Default is 500ms.
369
+ * @param options.timeoutSecs - Timeout for each batch API request. Default is `'medium'`.
332
370
  * @returns Object with `processedRequests` (successfully added) and `unprocessedRequests` (failed after all retries)
333
371
  * @see https://docs.apify.com/api/v2/request-queue-requests-batch-post
334
372
  *
@@ -353,30 +391,46 @@ export class RequestQueueClient extends ResourceClient {
353
391
  * @since Added in 2.1.0
354
392
  */
355
393
  async batchAddRequests(requests, options = {}) {
356
- const { forefront, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, maxParallel = DEFAULT_PARALLEL_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
394
+ const { forefront, timeoutSecs, signal, maxUnprocessedRequestsRetries = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, maxParallel = DEFAULT_PARALLEL_BATCH_ADD_REQUESTS, minDelayBetweenUnprocessedRequestsRetriesMillis = DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS, } = options;
357
395
  parseArgument(requests, batchAddRequestsWithRetriesSchema);
358
396
  parseArgument(forefront, optionalBooleanSchema);
397
+ parseArgument(timeoutSecs, optionalTimeoutSchema);
398
+ parseArgument(signal, optionalSignalSchema);
359
399
  parseArgument(maxUnprocessedRequestsRetries, optionalNumberSchema);
360
400
  parseArgument(maxParallel, optionalNumberSchema);
361
401
  parseArgument(minDelayBetweenUnprocessedRequestsRetriesMillis, optionalNumberSchema);
362
402
  const executingRequests = new Set();
363
403
  const individualResults = [];
364
404
  const payloadSizeLimitBytes = MAX_PAYLOAD_SIZE_BYTES - Math.ceil(MAX_PAYLOAD_SIZE_BYTES * SAFETY_BUFFER_PERCENT);
405
+ // Serialize every request once: the byte lengths decide the batch boundaries, and the same strings are joined
406
+ // into the batch bodies, so nothing is stringified a second time when it is sent.
407
+ const serializedRequests = requests.map((request, index) => {
408
+ const json = JSON.stringify(request);
409
+ const byteLength = utf8ByteLength(json);
410
+ // Two more bytes for the brackets, which even a batch of one request carries.
411
+ if (byteLength + 2 > payloadSizeLimitBytes) {
412
+ throw new Error(`RequestQueueClient.batchAddRequests: The size of the request with index: ${index} ` +
413
+ `exceeds the maximum allowed size (${payloadSizeLimitBytes} bytes).`);
414
+ }
415
+ return { request, json, byteLength };
416
+ });
417
+ const batches = splitIntoJsonArrayBatches(serializedRequests, {
418
+ maxCount: REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION,
419
+ maxByteLength: payloadSizeLimitBytes,
420
+ });
365
421
  // Keep a pool of up to `maxParallel` requests running at once
366
- let i = 0;
367
- while (i < requests.length) {
368
- const slicedRequests = requests.slice(i, i + REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION);
369
- const requestsInBatch = sliceArrayByByteLength(slicedRequests, payloadSizeLimitBytes, i);
370
- const requestPromise = this._batchAddRequestsWithRetries(requestsInBatch, options);
422
+ for (const batch of batches) {
423
+ const requestPromise = this.addRequestBatchWithRetries(batch, options);
371
424
  executingRequests.add(requestPromise);
425
+ // A rejection reaches the caller through the awaits below; this bookkeeping chain only has to avoid
426
+ // turning it into an unhandled one of its own.
372
427
  void requestPromise.then((batchAddResult) => {
373
428
  executingRequests.delete(requestPromise);
374
429
  individualResults.push(batchAddResult);
375
- });
430
+ }, () => undefined);
376
431
  if (executingRequests.size >= maxParallel) {
377
432
  await Promise.race(executingRequests);
378
433
  }
379
- i += requestsInBatch.length;
380
434
  }
381
435
  // Get results from remaining operations
382
436
  await Promise.all(executingRequests);
@@ -397,41 +451,49 @@ export class RequestQueueClient extends ResourceClient {
397
451
  * Requests can be identified by either their ID or unique key.
398
452
  *
399
453
  * @param requests - Array of requests to delete (by id or uniqueKey)
454
+ * @param options - Request options
455
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
400
456
  * @returns Result containing processed and unprocessed requests
401
457
  * @see https://docs.apify.com/api/v2/request-queue-requests-batch-delete
402
458
  * @since Added in 2.3.0
403
459
  */
404
- async batchDeleteRequests(requests) {
460
+ async batchDeleteRequests(requests, options = {}) {
405
461
  parseArgument(requests, batchDeleteRequestsSchema);
406
- const { data } = await this.httpClient.call({
407
- url: this._url('requests/batch'),
462
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
463
+ const response = await this.httpClient.call({
464
+ url: this.buildUrl('requests/batch'),
408
465
  method: 'DELETE',
409
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
466
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
467
+ signal,
410
468
  data: requests,
411
- params: this._params({
412
- clientKey: this.clientKey,
469
+ params: this.buildParams({
470
+ clientKey: this.#clientKey,
413
471
  }),
414
472
  });
415
- return cast(parseDateFields(pluckData(data)));
473
+ return parseResponse(response, schemas.BatchDeleteResult());
416
474
  }
417
475
  /**
418
476
  * Gets a specific request from the queue by its ID.
419
477
  *
420
478
  * @param id - Request ID
479
+ * @param options - Request options
480
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
421
481
  * @returns The request object, or `undefined` if not found
422
482
  * @see https://docs.apify.com/api/v2/request-queue-request-get
423
483
  */
424
- async getRequest(id) {
484
+ async getRequest(id, options = {}) {
425
485
  parseArgument(id, requestIdSchema);
486
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
426
487
  const requestOpts = {
427
- url: this._url(['requests', id]),
488
+ url: this.buildUrl(['requests', id]),
428
489
  method: 'GET',
429
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
430
- params: this._params(),
490
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
491
+ signal,
492
+ params: this.buildParams(),
431
493
  };
432
494
  try {
433
495
  const response = await this.httpClient.call(requestOpts);
434
- return cast(parseDateFields(pluckData(response.data)));
496
+ return parseResponse(response, schemas.Request());
435
497
  }
436
498
  catch (err) {
437
499
  catchNotFoundOrThrow(err);
@@ -443,6 +505,8 @@ export class RequestQueueClient extends ResourceClient {
443
505
  *
444
506
  * @param request - The updated request object (must include id)
445
507
  * @param options - Update options such as whether to move to front
508
+ * @param options.forefront - If `true`, moves the request to the beginning of the queue. Default is `false`.
509
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
446
510
  * @returns Information about the updated request
447
511
  * @see https://docs.apify.com/api/v2/request-queue-request-put
448
512
  */
@@ -450,30 +514,35 @@ export class RequestQueueClient extends ResourceClient {
450
514
  parseArgument(request, existingRequestSchema);
451
515
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
452
516
  const response = await this.httpClient.call({
453
- url: this._url(['requests', request.id]),
517
+ url: this.buildUrl(['requests', request.id]),
454
518
  method: 'PUT',
455
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
519
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
520
+ signal: parsed.signal,
456
521
  data: request,
457
- params: this._params({
522
+ params: this.buildParams({
458
523
  forefront: parsed.forefront,
459
- clientKey: this.clientKey,
524
+ clientKey: this.#clientKey,
460
525
  }),
461
526
  });
462
- return cast(parseDateFields(pluckData(response.data)));
527
+ return parseResponse(response, schemas.RequestRegistration());
463
528
  }
464
529
  /**
465
530
  * Deletes a specific request from the queue.
466
531
  *
467
532
  * @param id - Request ID
533
+ * @param options - Request options
534
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
468
535
  */
469
- async deleteRequest(id) {
536
+ async deleteRequest(id, options = {}) {
470
537
  parseArgument(id, requestIdSchema);
538
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
471
539
  await this.httpClient.call({
472
- url: this._url(['requests', id]),
540
+ url: this.buildUrl(['requests', id]),
473
541
  method: 'DELETE',
474
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
475
- params: this._params({
476
- clientKey: this.clientKey,
542
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'short'),
543
+ signal,
544
+ params: this.buildParams({
545
+ clientKey: this.#clientKey,
477
546
  }),
478
547
  });
479
548
  }
@@ -488,6 +557,7 @@ export class RequestQueueClient extends ResourceClient {
488
557
  * @param options - Lock extension options
489
558
  * @param options.lockSecs - **Required.** New lock duration in seconds from now.
490
559
  * @param options.forefront - If `true`, moves the request to the beginning of the queue when the lock expires. Default is `false`.
560
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
491
561
  * @returns Object with new `lockExpiresAt` timestamp
492
562
  * @see https://docs.apify.com/api/v2/request-queue-request-lock-put
493
563
  *
@@ -506,16 +576,17 @@ export class RequestQueueClient extends ResourceClient {
506
576
  parseArgument(id, requestIdSchema);
507
577
  const parsed = parseArgument(options, prolongRequestLockOptionsSchema, 'RequestQueueClientProlongRequestLockOptions');
508
578
  const response = await this.httpClient.call({
509
- url: this._url(['requests', id, 'lock']),
579
+ url: this.buildUrl(['requests', id, 'lock']),
510
580
  method: 'PUT',
511
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
512
- params: this._params({
581
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
582
+ signal: parsed.signal,
583
+ params: this.buildParams({
513
584
  forefront: parsed.forefront,
514
585
  lockSecs: parsed.lockSecs,
515
- clientKey: this.clientKey,
586
+ clientKey: this.#clientKey,
516
587
  }),
517
588
  });
518
- return cast(parseDateFields(pluckData(response.data)));
589
+ return parseResponse(response, schemas.RequestLockInfo());
519
590
  }
520
591
  /**
521
592
  * Releases the lock on a request, allowing other clients to process it.
@@ -525,6 +596,8 @@ export class RequestQueueClient extends ResourceClient {
525
596
  *
526
597
  * @param id - Request ID
527
598
  * @param options - Options such as whether to move to front
599
+ * @param options.forefront - If `true`, moves the request to the beginning of the queue. Default is `false`.
600
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
528
601
  * @see https://docs.apify.com/api/v2/request-queue-request-lock-delete
529
602
  * @since Added in 2.4.1
530
603
  */
@@ -532,12 +605,13 @@ export class RequestQueueClient extends ResourceClient {
532
605
  parseArgument(id, requestIdSchema);
533
606
  const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientDeleteRequestLockOptions');
534
607
  await this.httpClient.call({
535
- url: this._url(['requests', id, 'lock']),
608
+ url: this.buildUrl(['requests', id, 'lock']),
536
609
  method: 'DELETE',
537
- timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
538
- params: this._params({
610
+ timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'short'),
611
+ signal: parsed.signal,
612
+ params: this.buildParams({
539
613
  forefront: parsed.forefront,
540
- clientKey: this.clientKey,
614
+ clientKey: this.#clientKey,
541
615
  }),
542
616
  });
543
617
  }
@@ -548,24 +622,29 @@ export class RequestQueueClient extends ResourceClient {
548
622
  * queue contents.
549
623
  *
550
624
  * @param options - Pagination options
625
+ * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
551
626
  * @returns List of requests with pagination information
552
627
  * @see https://docs.apify.com/api/v2/request-queue-requests-get
553
628
  * @since Added in 2.5.1
554
629
  */
555
630
  listRequests(options = {}) {
556
- const parsed = parseArgument(options, listRequestsOptionsSchema, 'RequestQueueClientListRequestsOptions');
631
+ // `timeoutSecs` and `signal` apply to every page request; they are not API parameters, so they must not reach
632
+ // the query string.
633
+ const { timeoutSecs, signal, ...parsed } = parseArgument(options, listRequestsOptionsSchema, 'RequestQueueClientListRequestsOptions');
634
+ const pageTimeout = this.#resolveTimeout(timeoutSecs, 'medium');
557
635
  const getPaginatedList = async (rqListOptions = {}) => {
558
636
  const response = await this.httpClient.call({
559
- url: this._url('requests'),
637
+ url: this.buildUrl('requests'),
560
638
  method: 'GET',
561
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
562
- params: this._params({
639
+ timeoutSecs: pageTimeout,
640
+ signal,
641
+ params: this.buildParams({
563
642
  ...rqListOptions,
564
643
  filter: rqListOptions.filter ? rqListOptions.filter.join(',') : undefined,
565
- clientKey: this.clientKey,
644
+ clientKey: this.#clientKey,
566
645
  }),
567
646
  });
568
- return cast(parseDateFields(pluckData(response.data)));
647
+ return parseResponse(response, schemas.ListOfRequests());
569
648
  };
570
649
  const paginatedListPromise = getPaginatedList(parsed);
571
650
  async function* asyncGenerator() {
@@ -581,8 +660,6 @@ export class RequestQueueClient extends ResourceClient {
581
660
  const newOptions = {
582
661
  ...parsed,
583
662
  limit: remainingItems,
584
- // remove original exclusiveStartId, if there was any, and use cursor-based pagination
585
- exclusiveStartId: undefined,
586
663
  cursor: currentPage.nextCursor,
587
664
  };
588
665
  currentPage = await getPaginatedList(newOptions);
@@ -602,20 +679,24 @@ export class RequestQueueClient extends ResourceClient {
602
679
  * This is useful for releasing all locks at once, for example when shutting down
603
680
  * a crawler gracefully.
604
681
  *
682
+ * @param options - Request options
683
+ * @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
605
684
  * @returns Number of requests that were unlocked
606
685
  * @see https://docs.apify.com/api/v2/request-queue-requests-unlock-post
607
686
  * @since Added in 2.12.5
608
687
  */
609
- async unlockRequests() {
688
+ async unlockRequests(options = {}) {
689
+ const { timeoutSecs, signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
610
690
  const response = await this.httpClient.call({
611
- url: this._url('requests/unlock'),
691
+ url: this.buildUrl('requests/unlock'),
612
692
  method: 'POST',
613
- timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
614
- params: this._params({
615
- clientKey: this.clientKey,
693
+ timeoutSecs: this.#resolveTimeout(timeoutSecs, 'long'),
694
+ signal,
695
+ params: this.buildParams({
696
+ clientKey: this.#clientKey,
616
697
  }),
617
698
  });
618
- return cast(parseDateFields(pluckData(response.data)));
699
+ return parseResponse(response, schemas.UnlockRequestsResult());
619
700
  }
620
701
  /**
621
702
  * Returns an async iterable for paginating through all requests in the queue.
@@ -624,6 +705,7 @@ export class RequestQueueClient extends ResourceClient {
624
705
  * automatically handling pagination behind the scenes.
625
706
  *
626
707
  * @param options - Pagination options
708
+ * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
627
709
  * @returns An async iterable of request pages
628
710
  * @see https://docs.apify.com/api/v2/request-queue-requests-get
629
711
  *
@@ -636,11 +718,10 @@ export class RequestQueueClient extends ResourceClient {
636
718
  * @since Added in 2.5.1
637
719
  */
638
720
  paginateRequests(options = {}) {
639
- const { limit, exclusiveStartId, cursor, filter, maxPageLimit } = parseArgument(options, paginateRequestsOptionsSchema, 'RequestQueueClientPaginateRequestsOptions');
721
+ const { limit, cursor, filter, maxPageLimit, timeoutSecs, signal } = parseArgument(options, paginateRequestsOptionsSchema, 'RequestQueueClientPaginateRequestsOptions');
640
722
  return new RequestQueuePaginationIterator({
641
- getPage: async (pageOptions) => this.listRequests({ ...pageOptions, filter }),
723
+ getPage: async (pageOptions) => this.listRequests({ ...pageOptions, filter, timeoutSecs, signal }),
642
724
  limit,
643
- exclusiveStartId,
644
725
  cursor,
645
726
  maxPageLimit,
646
727
  });