apify-client 3.0.0-beta.5 → 3.0.0-beta.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -3
- package/dist/apify_api_error.d.ts +62 -21
- package/dist/apify_api_error.js +96 -23
- package/dist/apify_client.d.ts +125 -10
- package/dist/apify_client.js +133 -48
- package/dist/base/api_client.d.ts +7 -7
- package/dist/base/api_client.js +36 -24
- package/dist/base/resource_client.d.ts +21 -8
- package/dist/base/resource_client.js +54 -26
- package/dist/base/resource_collection_client.d.ts +12 -4
- package/dist/base/resource_collection_client.js +34 -11
- package/dist/body_parser.d.ts +3 -4
- package/dist/body_parser.js +19 -26
- package/dist/bundle.js +27 -22
- package/dist/bundle.js.map +1 -1
- package/dist/generated/api.d.ts +610 -112
- package/dist/generated/schemas.d.ts +726 -703
- package/dist/generated/schemas.js +92 -85
- package/dist/http_clients/axios.d.ts +60 -0
- package/dist/http_clients/axios.js +117 -0
- package/dist/http_clients/base.d.ts +333 -0
- package/dist/http_clients/base.js +563 -0
- package/dist/http_clients/index.d.ts +4 -0
- package/dist/http_clients/index.js +2 -0
- package/dist/http_compressors/base.d.ts +40 -0
- package/dist/http_compressors/base.js +9 -0
- package/dist/http_compressors/brotli.d.ts +33 -0
- package/dist/http_compressors/brotli.js +39 -0
- package/dist/http_compressors/gzip.d.ts +33 -0
- package/dist/http_compressors/gzip.js +39 -0
- package/dist/http_compressors/index.d.ts +6 -0
- package/dist/http_compressors/index.js +2 -0
- package/dist/http_compressors/resolve.d.ts +23 -0
- package/dist/http_compressors/resolve.js +27 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +3 -1
- package/dist/invalid_response_body_error.d.ts +13 -0
- package/dist/invalid_response_body_error.js +17 -0
- package/dist/models.d.ts +19 -22
- package/dist/resource_clients/actor.d.ts +72 -32
- package/dist/resource_clients/actor.js +87 -56
- package/dist/resource_clients/actor_collection.d.ts +9 -5
- package/dist/resource_clients/actor_collection.js +11 -3
- package/dist/resource_clients/actor_env_var.d.ts +10 -3
- package/dist/resource_clients/actor_env_var.js +16 -6
- package/dist/resource_clients/actor_env_var_collection.d.ts +14 -18
- package/dist/resource_clients/actor_env_var_collection.js +17 -11
- package/dist/resource_clients/actor_version.d.ts +18 -3
- package/dist/resource_clients/actor_version.js +19 -9
- package/dist/resource_clients/actor_version_collection.d.ts +16 -17
- package/dist/resource_clients/actor_version_collection.js +19 -11
- package/dist/resource_clients/build.d.ts +18 -6
- package/dist/resource_clients/build.js +34 -15
- package/dist/resource_clients/build_collection.d.ts +3 -1
- package/dist/resource_clients/build_collection.js +4 -1
- package/dist/resource_clients/dataset.d.ts +51 -33
- package/dist/resource_clients/dataset.js +105 -80
- package/dist/resource_clients/dataset_collection.d.ts +6 -2
- package/dist/resource_clients/dataset_collection.js +13 -2
- package/dist/resource_clients/key_value_store.d.ts +58 -16
- package/dist/resource_clients/key_value_store.js +78 -40
- package/dist/resource_clients/key_value_store_collection.d.ts +6 -2
- package/dist/resource_clients/key_value_store_collection.js +13 -2
- package/dist/resource_clients/log.d.ts +12 -20
- package/dist/resource_clients/log.js +93 -58
- package/dist/resource_clients/request_queue.d.ts +56 -26
- package/dist/resource_clients/request_queue.js +176 -103
- package/dist/resource_clients/request_queue_collection.d.ts +6 -2
- package/dist/resource_clients/request_queue_collection.js +9 -3
- package/dist/resource_clients/run.d.ts +108 -18
- package/dist/resource_clients/run.js +183 -43
- package/dist/resource_clients/run_collection.d.ts +3 -1
- package/dist/resource_clients/run_collection.js +4 -1
- package/dist/resource_clients/schedule.d.ts +14 -5
- package/dist/resource_clients/schedule.js +29 -21
- package/dist/resource_clients/schedule_collection.d.ts +6 -2
- package/dist/resource_clients/schedule_collection.js +9 -3
- package/dist/resource_clients/store_collection.d.ts +3 -1
- package/dist/resource_clients/store_collection.js +4 -1
- package/dist/resource_clients/task.d.ts +33 -10
- package/dist/resource_clients/task.js +79 -46
- package/dist/resource_clients/task_collection.d.ts +6 -2
- package/dist/resource_clients/task_collection.js +9 -3
- package/dist/resource_clients/user.d.ts +17 -8
- package/dist/resource_clients/user.js +42 -38
- package/dist/resource_clients/webhook.d.ts +14 -5
- package/dist/resource_clients/webhook.js +30 -22
- package/dist/resource_clients/webhook_collection.d.ts +6 -2
- package/dist/resource_clients/webhook_collection.js +9 -3
- package/dist/resource_clients/webhook_dispatch.d.ts +4 -1
- package/dist/resource_clients/webhook_dispatch.js +7 -2
- package/dist/resource_clients/webhook_dispatch_collection.d.ts +3 -1
- package/dist/resource_clients/webhook_dispatch_collection.js +4 -1
- package/dist/runtime/node.d.ts +6 -0
- package/dist/runtime/node.js +49 -0
- package/dist/runtime/types.d.ts +52 -0
- package/dist/runtime/types.js +1 -0
- package/dist/runtime/web.d.ts +5 -0
- package/dist/runtime/web.js +18 -0
- package/dist/schemas.d.ts +1 -1
- package/dist/schemas.js +1 -1
- package/dist/statistics.d.ts +1 -6
- package/dist/statistics.js +2 -2
- package/dist/timeouts.d.ts +84 -0
- package/dist/timeouts.js +37 -0
- package/dist/timezones.d.ts +1 -1
- package/dist/utils.d.ts +81 -52
- package/dist/utils.js +201 -170
- package/dist/wait_for_resources.d.ts +8 -0
- package/dist/wait_for_resources.js +39 -0
- package/package.json +32 -29
- package/dist/http_client.d.ts +0 -88
- package/dist/http_client.js +0 -255
- package/dist/interceptors.d.ts +0 -19
- package/dist/interceptors.js +0 -105
package/dist/utils.js
CHANGED
|
@@ -1,12 +1,64 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import { NotFoundError } from './apify_api_error.js';
|
|
2
3
|
import { parseArgument } from '@apify/validations';
|
|
3
4
|
import { ResponseValidationError } from './response_validation_error.js';
|
|
4
5
|
// @ts-ignore if we enable `resolveJsonModule`, we end up with a `src` folder in `dist`
|
|
5
6
|
import packageJson from '../package.json' with { type: 'json' };
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
/**
|
|
8
|
+
* Smallest request body, in bytes, that is worth compressing. A smaller body already fits in a single network
|
|
9
|
+
* packet, so compressing it costs CPU time without saving a round trip, and the compression format itself can
|
|
10
|
+
* make it larger.
|
|
11
|
+
* @internal
|
|
12
|
+
*/
|
|
13
|
+
export const MIN_COMPRESS_BYTES = 1024;
|
|
14
|
+
const textEncoder = new TextEncoder();
|
|
15
|
+
// Only the version, so a bundler can drop the rest of the manifest.
|
|
16
|
+
export const { version } = packageJson;
|
|
17
|
+
/** Media type prefixes whose payloads carry their own compression, so compressing the request body is wasted work. */
|
|
18
|
+
const ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES = ['audio/', 'image/', 'video/'];
|
|
19
|
+
/** Exact media types whose payloads carry their own compression. */
|
|
20
|
+
const ALREADY_COMPRESSED_MEDIA_TYPES = new Set([
|
|
21
|
+
'application/epub+zip',
|
|
22
|
+
'application/gzip',
|
|
23
|
+
'application/java-archive',
|
|
24
|
+
'application/vnd.android.package-archive',
|
|
25
|
+
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
|
26
|
+
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
27
|
+
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
28
|
+
'application/vnd.rar',
|
|
29
|
+
'application/x-7z-compressed',
|
|
30
|
+
'application/x-bzip',
|
|
31
|
+
'application/x-bzip2',
|
|
32
|
+
'application/x-gzip',
|
|
33
|
+
'application/x-rar-compressed',
|
|
34
|
+
'application/x-xz',
|
|
35
|
+
'application/x-zip-compressed',
|
|
36
|
+
'application/zip',
|
|
37
|
+
'application/zstd',
|
|
38
|
+
'font/woff',
|
|
39
|
+
'font/woff2',
|
|
40
|
+
]);
|
|
41
|
+
/** Uncompressed media types that sit under an already-compressed prefix, so compressing them still pays off. */
|
|
42
|
+
const COMPRESSIBLE_MEDIA_TYPES = new Set([
|
|
43
|
+
'audio/aiff',
|
|
44
|
+
'audio/basic',
|
|
45
|
+
'audio/l16',
|
|
46
|
+
'audio/l24',
|
|
47
|
+
'audio/midi',
|
|
48
|
+
'audio/vnd.wave',
|
|
49
|
+
'audio/wav',
|
|
50
|
+
'audio/wave',
|
|
51
|
+
'audio/x-aiff',
|
|
52
|
+
'audio/x-wav',
|
|
53
|
+
'image/bmp',
|
|
54
|
+
'image/tiff',
|
|
55
|
+
'image/vnd.adobe.photoshop',
|
|
56
|
+
'image/vnd.microsoft.icon',
|
|
57
|
+
'image/x-icon',
|
|
58
|
+
'image/x-ms-bmp',
|
|
59
|
+
]);
|
|
60
|
+
/** Structured syntax suffixes marking a media type as text even under an already-compressed prefix (`image/svg+xml`). */
|
|
61
|
+
const COMPRESSIBLE_MEDIA_TYPE_SUFFIXES = ['+json', '+xml'];
|
|
10
62
|
export { parseArgument };
|
|
11
63
|
/**
|
|
12
64
|
* Accepts any non-null, non-array object as a predicate for `z.custom()`.
|
|
@@ -29,16 +81,17 @@ export const anyObjectSchema = z.custom(isNonArrayObject, {
|
|
|
29
81
|
// in per parse keeps them intact without reaching into the zod config the whole process shares.
|
|
30
82
|
const { localeError } = z.locales.en();
|
|
31
83
|
/**
|
|
32
|
-
* Turns a JSON API response into the value a resource method returns: unwraps the `data` envelope
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
* fields and
|
|
84
|
+
* Turns a JSON API response into the value a resource method returns: unwraps the `data` envelope and validates the
|
|
85
|
+
* result against `schema`, one of the schemas generated from the OpenAPI specification. The validated copy is what
|
|
86
|
+
* callers get, so it is the schema's output -- unknown fields and unknown enum values included, since the schemas let
|
|
87
|
+
* both through, date-time fields turned into `Date` objects, and URL fields normalized, since `z.url()` hands back the
|
|
88
|
+
* parsed URL's serialization.
|
|
36
89
|
*
|
|
37
90
|
* Throws {@link ResponseValidationError} when the response does not match the specification.
|
|
38
91
|
* @internal
|
|
39
92
|
*/
|
|
40
|
-
export function parseResponse(response, schema
|
|
41
|
-
const data =
|
|
93
|
+
export function parseResponse(response, schema) {
|
|
94
|
+
const data = pluckData(response.data);
|
|
42
95
|
const result = schema.safeParse(data, { error: localeError });
|
|
43
96
|
if (!result.success) {
|
|
44
97
|
const { method = 'GET', url = '' } = response.config;
|
|
@@ -58,62 +111,23 @@ export function pluckData(obj) {
|
|
|
58
111
|
throw new Error(`Expected response object with a "data" property, but received: ${obj}`);
|
|
59
112
|
}
|
|
60
113
|
/**
|
|
61
|
-
*
|
|
62
|
-
* Otherwise rethrows error.
|
|
114
|
+
* Swallows a 404 Not Found API error and rethrows anything else.
|
|
63
115
|
*/
|
|
64
116
|
export function catchNotFoundOrThrow(err) {
|
|
65
|
-
|
|
66
|
-
const isNotFoundMessage = err.type === RECORD_NOT_FOUND_TYPE || err.type === RECORD_OR_TOKEN_NOT_FOUND_TYPE || err.httpMethod === 'head';
|
|
67
|
-
const isNotFoundError = isNotFoundStatus && isNotFoundMessage;
|
|
68
|
-
if (!isNotFoundError)
|
|
117
|
+
if (!(err instanceof NotFoundError))
|
|
69
118
|
throw err;
|
|
70
119
|
}
|
|
71
120
|
/**
|
|
72
|
-
*
|
|
73
|
-
* "createdAt").
|
|
121
|
+
* Like `catchNotFoundOrThrow()`, but swallows the 404 only when the client names its resource by ID.
|
|
74
122
|
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
* If the field cannot be converted to Date, it is left as is.
|
|
123
|
+
* A chained client without an ID, such as `run.dataset()` or `run.log()`, requests a path where a 404 can mean either
|
|
124
|
+
* the parent or the default sub-resource is missing. The response cannot tell the two apart, so the error propagates.
|
|
125
|
+
* @internal
|
|
80
126
|
*/
|
|
81
|
-
export function
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
// array spend a level.
|
|
86
|
-
//
|
|
87
|
-
// In a list response it also reaches one level into caller-owned blobs the API stores verbatim, so a
|
|
88
|
-
// listed request's `userData.foo.somethingAt` comes back as a `Date` rather than the string it was
|
|
89
|
-
// written as.
|
|
90
|
-
// TODO: Consider removing this limitation. It might came across as an annoying surprise as it's not communicated.
|
|
91
|
-
if (depth > 4) {
|
|
92
|
-
return input;
|
|
93
|
-
}
|
|
94
|
-
if (Array.isArray(input))
|
|
95
|
-
return input.map((child) => parseDateFields(child, shouldParseField, depth + 1));
|
|
96
|
-
if (!input || typeof input !== 'object')
|
|
97
|
-
return input;
|
|
98
|
-
return Object.entries(input).reduce((output, [k, v]) => {
|
|
99
|
-
const isValObject = !!v && typeof v === 'object';
|
|
100
|
-
if (k.endsWith('At') || (shouldParseField && shouldParseField(k))) {
|
|
101
|
-
if (v) {
|
|
102
|
-
const d = new Date(v);
|
|
103
|
-
output[k] = Number.isNaN(d.getTime()) ? v : d;
|
|
104
|
-
}
|
|
105
|
-
else {
|
|
106
|
-
output[k] = v;
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
else if (isValObject || Array.isArray(v)) {
|
|
110
|
-
output[k] = parseDateFields(v, shouldParseField, depth + 1);
|
|
111
|
-
}
|
|
112
|
-
else {
|
|
113
|
-
output[k] = v;
|
|
114
|
-
}
|
|
115
|
-
return output;
|
|
116
|
-
}, {});
|
|
127
|
+
export function catchNotFoundForResourceOrThrow(err, resourceId) {
|
|
128
|
+
if (!resourceId)
|
|
129
|
+
throw err;
|
|
130
|
+
catchNotFoundOrThrow(err);
|
|
117
131
|
}
|
|
118
132
|
/**
|
|
119
133
|
* Helper function that converts array of webhooks to base64 string
|
|
@@ -121,98 +135,105 @@ export function parseDateFields(input, shouldParseField = null, depth = 0) {
|
|
|
121
135
|
export function stringifyWebhooksToBase64(webhooks) {
|
|
122
136
|
if (!webhooks)
|
|
123
137
|
return;
|
|
124
|
-
|
|
125
|
-
if (isNode()) {
|
|
126
|
-
return Buffer.from(webhooksJson, 'utf8').toString('base64');
|
|
127
|
-
}
|
|
128
|
-
const encoder = new TextEncoder();
|
|
129
|
-
const uint8Array = encoder.encode(webhooksJson);
|
|
130
|
-
return btoa(String.fromCharCode(...uint8Array));
|
|
138
|
+
return bytesToBase64(textEncoder.encode(JSON.stringify(webhooks)));
|
|
131
139
|
}
|
|
132
|
-
let brotliCompressPromisified;
|
|
133
140
|
/**
|
|
134
|
-
*
|
|
141
|
+
* Encodes bytes as base64. `btoa()` takes a binary string, and the bytes are turned into one in slices,
|
|
142
|
+
* because spreading them all into a single `String.fromCharCode()` call overflows the argument limit on
|
|
143
|
+
* inputs of a few tens of kilobytes.
|
|
135
144
|
*/
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
const options = { params: { [constants.BROTLI_PARAM_QUALITY]: 6 } };
|
|
142
|
-
brotliCompressPromisified = async (input) => compress(input, options);
|
|
145
|
+
export function bytesToBase64(bytes) {
|
|
146
|
+
const SLICE_LENGTH = 0x8000;
|
|
147
|
+
let binary = '';
|
|
148
|
+
for (let i = 0; i < bytes.length; i += SLICE_LENGTH) {
|
|
149
|
+
binary += String.fromCharCode(...bytes.subarray(i, i + SLICE_LENGTH));
|
|
143
150
|
}
|
|
144
|
-
return
|
|
151
|
+
return btoa(binary);
|
|
145
152
|
}
|
|
146
|
-
let gzipPromisified;
|
|
147
153
|
/**
|
|
148
|
-
*
|
|
154
|
+
* Concatenates byte chunks into one array.
|
|
149
155
|
*/
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
156
|
+
export function concatBytes(chunks) {
|
|
157
|
+
const result = new Uint8Array(chunks.reduce((length, chunk) => length + chunk.length, 0));
|
|
158
|
+
let offset = 0;
|
|
159
|
+
for (const chunk of chunks) {
|
|
160
|
+
result.set(chunk, offset);
|
|
161
|
+
offset += chunk.length;
|
|
155
162
|
}
|
|
156
|
-
return
|
|
163
|
+
return result;
|
|
157
164
|
}
|
|
158
165
|
/**
|
|
159
|
-
*
|
|
160
|
-
*
|
|
166
|
+
* Views a request body as bytes: a string is UTF-8 encoded, binary values are viewed in place. Anything else
|
|
167
|
+
* - a stream, a `Blob`, form data - is `undefined`.
|
|
168
|
+
* @internal
|
|
161
169
|
*/
|
|
162
|
-
export
|
|
163
|
-
if (
|
|
170
|
+
export function toBytes(value) {
|
|
171
|
+
if (typeof value === 'string')
|
|
172
|
+
return textEncoder.encode(value);
|
|
173
|
+
if (!isBuffer(value))
|
|
164
174
|
return undefined;
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
return
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
175
|
+
return ArrayBuffer.isView(value)
|
|
176
|
+
? new Uint8Array(value.buffer, value.byteOffset, value.byteLength)
|
|
177
|
+
: new Uint8Array(value);
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Decides whether a request body with the given content type is worth compressing.
|
|
181
|
+
*
|
|
182
|
+
* Images, audio, video and archives already carry their own compression. Running them through brotli or gzip
|
|
183
|
+
* burns CPU, holds a second full copy of the body in memory, and usually produces output slightly larger than
|
|
184
|
+
* the input. Formats that are raw despite such a media type, for example `image/bmp` or `audio/wav`, are still
|
|
185
|
+
* compressed. A body with no content type is assumed to be compressible.
|
|
186
|
+
* @internal
|
|
187
|
+
*/
|
|
188
|
+
export function isCompressibleContentType(contentType) {
|
|
189
|
+
if (!contentType)
|
|
190
|
+
return true;
|
|
191
|
+
// `Content-Type` is case-insensitive and may carry parameters, for example `text/plain; charset=utf-8`.
|
|
192
|
+
const mediaType = contentType.split(';', 1)[0].trim().toLowerCase();
|
|
193
|
+
if (COMPRESSIBLE_MEDIA_TYPES.has(mediaType))
|
|
194
|
+
return true;
|
|
195
|
+
if (COMPRESSIBLE_MEDIA_TYPE_SUFFIXES.some((suffix) => mediaType.endsWith(suffix)))
|
|
196
|
+
return true;
|
|
197
|
+
if (ALREADY_COMPRESSED_MEDIA_TYPES.has(mediaType))
|
|
198
|
+
return false;
|
|
199
|
+
return !ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES.some((prefix) => mediaType.startsWith(prefix));
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Reads an environment variable, on runtimes that have them.
|
|
203
|
+
*/
|
|
204
|
+
export function getEnv(name) {
|
|
205
|
+
return typeof process !== 'undefined' ? process.env?.[name] : undefined;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Returns the UTF-8 byte length of a string.
|
|
209
|
+
*/
|
|
210
|
+
export function utf8ByteLength(value) {
|
|
211
|
+
return textEncoder.encode(value).byteLength;
|
|
187
212
|
}
|
|
188
213
|
/**
|
|
189
|
-
*
|
|
214
|
+
* Splits JSON-serialized items into consecutive batches of at most `maxCount` items, each of which fits into a JSON
|
|
215
|
+
* array body - the items joined by commas between brackets - of at most `maxByteLength` bytes. The `byteLength` of an
|
|
216
|
+
* item is the UTF-8 byte length of its serialization. An item too large for a body of its own still gets one, so a
|
|
217
|
+
* caller that cannot send such an item has to reject it beforehand.
|
|
190
218
|
*/
|
|
191
|
-
export function
|
|
192
|
-
const
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
throw new Error(`RequestQueueClient.batchAddRequests: The size of the request with index: ${startIndex + i} ` +
|
|
203
|
-
`exceeds the maximum allowed size (${maxByteLength} bytes).`);
|
|
219
|
+
export function splitIntoJsonArrayBatches(items, { maxCount, maxByteLength }) {
|
|
220
|
+
const batches = [];
|
|
221
|
+
let batch = [];
|
|
222
|
+
// One byte for the opening bracket; each item then adds its own bytes plus one for the comma or the closing
|
|
223
|
+
// bracket that follows it.
|
|
224
|
+
let byteLength = 1;
|
|
225
|
+
for (const item of items) {
|
|
226
|
+
if (batch.length > 0 && (batch.length >= maxCount || byteLength + item.byteLength + 1 > maxByteLength)) {
|
|
227
|
+
batches.push(batch);
|
|
228
|
+
batch = [];
|
|
229
|
+
byteLength = 1;
|
|
204
230
|
}
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
byteLength += itemByteSize;
|
|
208
|
-
slicedArray.push(item);
|
|
231
|
+
batch.push(item);
|
|
232
|
+
byteLength += item.byteLength + 1;
|
|
209
233
|
}
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
if (typeof BROWSER_BUILD !== 'undefined')
|
|
214
|
-
return false;
|
|
215
|
-
return !!(typeof process !== 'undefined' && process.versions && process.versions.node);
|
|
234
|
+
if (batch.length > 0)
|
|
235
|
+
batches.push(batch);
|
|
236
|
+
return batches;
|
|
216
237
|
}
|
|
217
238
|
export function isBuffer(value) {
|
|
218
239
|
// Tag checks rather than `instanceof`, to also match buffers from another realm. `isView()`
|
|
@@ -230,41 +251,30 @@ export function isStream(value) {
|
|
|
230
251
|
const { on, pipe } = value;
|
|
231
252
|
return typeof on === 'function' && typeof pipe === 'function';
|
|
232
253
|
}
|
|
233
|
-
export function getVersionData() {
|
|
234
|
-
if (typeof BROWSER_BUILD !== 'undefined') {
|
|
235
|
-
return { version: VERSION };
|
|
236
|
-
}
|
|
237
|
-
return packageJson;
|
|
238
|
-
}
|
|
239
254
|
/**
|
|
240
|
-
* Helper class to create async iterators from paginated list endpoints
|
|
255
|
+
* Helper class to create async iterators from paginated list endpoints.
|
|
241
256
|
*/
|
|
242
257
|
export class RequestQueuePaginationIterator {
|
|
243
|
-
maxPageLimit;
|
|
244
|
-
getPage;
|
|
245
|
-
limit;
|
|
246
|
-
|
|
247
|
-
cursor;
|
|
258
|
+
#maxPageLimit;
|
|
259
|
+
#getPage;
|
|
260
|
+
#limit;
|
|
261
|
+
#cursor;
|
|
248
262
|
constructor(options) {
|
|
249
|
-
this
|
|
250
|
-
this
|
|
251
|
-
this
|
|
252
|
-
this
|
|
253
|
-
this.getPage = options.getPage;
|
|
263
|
+
this.#maxPageLimit = options.maxPageLimit;
|
|
264
|
+
this.#limit = options.limit;
|
|
265
|
+
this.#cursor = options.cursor;
|
|
266
|
+
this.#getPage = options.getPage;
|
|
254
267
|
}
|
|
255
268
|
async *[Symbol.asyncIterator]() {
|
|
256
|
-
let nextCursor = this
|
|
257
|
-
// allow using exclusiveStartId for the first page, but then we'll delete it to avoid using it for any later page
|
|
258
|
-
let nextExclusiveStartId = this.exclusiveStartId;
|
|
269
|
+
let nextCursor = this.#cursor;
|
|
259
270
|
let iterateItemCount = 0;
|
|
260
271
|
while (true) {
|
|
261
|
-
const pageLimit = this
|
|
262
|
-
? Math.min(this
|
|
263
|
-
: this
|
|
264
|
-
const page = await this
|
|
272
|
+
const pageLimit = this.#limit
|
|
273
|
+
? Math.min(this.#maxPageLimit, this.#limit - iterateItemCount)
|
|
274
|
+
: this.#maxPageLimit;
|
|
275
|
+
const page = await this.#getPage({
|
|
265
276
|
limit: pageLimit,
|
|
266
277
|
cursor: nextCursor,
|
|
267
|
-
exclusiveStartId: nextExclusiveStartId,
|
|
268
278
|
});
|
|
269
279
|
// There are no more pages to iterate
|
|
270
280
|
if (page.items.length === 0)
|
|
@@ -272,10 +282,9 @@ export class RequestQueuePaginationIterator {
|
|
|
272
282
|
yield page;
|
|
273
283
|
iterateItemCount += page.items.length;
|
|
274
284
|
// Limit reached stopping to iterate
|
|
275
|
-
if ((this
|
|
285
|
+
if ((this.#limit && iterateItemCount >= this.#limit) || !page.nextCursor)
|
|
276
286
|
return;
|
|
277
287
|
nextCursor = page.nextCursor;
|
|
278
|
-
nextExclusiveStartId = undefined; // see comment above - delete it for any page after the first one, and paginate with cursor
|
|
279
288
|
}
|
|
280
289
|
}
|
|
281
290
|
}
|
|
@@ -289,6 +298,16 @@ export const paginationOptionsShape = {
|
|
|
289
298
|
offset: z.number().min(0).optional(),
|
|
290
299
|
chunkSize: z.number().positive().optional(),
|
|
291
300
|
};
|
|
301
|
+
/**
|
|
302
|
+
* Key under which a page of dataset items carries the number of rows the API scanned to produce it, as a
|
|
303
|
+
* non-enumerable property that stays out of the page's public shape. The API applies `offset` and `limit` to the
|
|
304
|
+
* dataset's rows first and shapes the result afterwards: filters (`clean`, `skipEmpty`, `skipHidden`) drop items and
|
|
305
|
+
* `unwind` splits or drops them, so `items.length` can land on either side of the rows scanned. The
|
|
306
|
+
* `x-apify-pagination-count` header reports the scanned number, and the offset iterator advances and terminates by it
|
|
307
|
+
* alone. A page without the property falls back to `items.length`.
|
|
308
|
+
* @internal
|
|
309
|
+
*/
|
|
310
|
+
export const SCANNED_COUNT = Symbol('scannedCount');
|
|
292
311
|
export function cast(input) {
|
|
293
312
|
return input;
|
|
294
313
|
}
|
|
@@ -315,15 +334,6 @@ export function applyQueryParamsToUrl(url, options) {
|
|
|
315
334
|
}
|
|
316
335
|
return url;
|
|
317
336
|
}
|
|
318
|
-
/**
|
|
319
|
-
* Builds a `[check, message]` pair to spread into `.refine()`, asserting that at most one of `keys`
|
|
320
|
-
* is present. Pass the options interface as `T`, so that a misspelled key is a type error.
|
|
321
|
-
* @internal
|
|
322
|
-
*/
|
|
323
|
-
export const mutuallyExclusive = (...keys) => [
|
|
324
|
-
(value) => keys.filter((key) => typeof value[key] !== 'undefined').length <= 1,
|
|
325
|
-
`At most one of the following fields is allowed: ${keys.join(', ')}`,
|
|
326
|
-
];
|
|
327
337
|
const pathSegmentSchema = z
|
|
328
338
|
.string()
|
|
329
339
|
.nonempty()
|
|
@@ -345,3 +355,24 @@ export function toPathSegment(value) {
|
|
|
345
355
|
export function toPath(path) {
|
|
346
356
|
return Array.isArray(path) ? path.map(toPathSegment).join('/') : path;
|
|
347
357
|
}
|
|
358
|
+
/**
|
|
359
|
+
* Resolves after `millis`, or right away once `signal` aborts.
|
|
360
|
+
* @internal
|
|
361
|
+
*/
|
|
362
|
+
export async function sleep(millis, signal) {
|
|
363
|
+
return new Promise((resolve) => {
|
|
364
|
+
if (signal?.aborted) {
|
|
365
|
+
resolve();
|
|
366
|
+
return;
|
|
367
|
+
}
|
|
368
|
+
const onAbort = () => {
|
|
369
|
+
clearTimeout(timer);
|
|
370
|
+
resolve();
|
|
371
|
+
};
|
|
372
|
+
const timer = setTimeout(() => {
|
|
373
|
+
signal?.removeEventListener('abort', onAbort);
|
|
374
|
+
resolve();
|
|
375
|
+
}, millis);
|
|
376
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
377
|
+
});
|
|
378
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Log } from '@apify/log';
|
|
2
|
+
/**
|
|
3
|
+
* Makes the `start` request, retrying it every {@link WAIT_FOR_RESOURCES_COOLDOWN_MILLIS} while it fails with one of
|
|
4
|
+
* {@link RESOURCE_LIMIT_ERROR_TYPES}. `true` retries until the request succeeds, a number of seconds bounds the
|
|
5
|
+
* retrying, after which the last error is thrown. Any other error is thrown right away.
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
export declare function startWaitingForResources<T>(start: () => Promise<T>, waitForResources: boolean | number | undefined, logger: Log, signal?: AbortSignal): Promise<T>;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { ApifyApiError } from './apify_api_error.js';
|
|
2
|
+
import { sleep } from './utils.js';
|
|
3
|
+
/**
|
|
4
|
+
* Error types the API rejects a run start with while the account has no free memory or concurrent-run slot for it.
|
|
5
|
+
* Both clear as other runs or builds finish.
|
|
6
|
+
*/
|
|
7
|
+
const RESOURCE_LIMIT_ERROR_TYPES = new Set([
|
|
8
|
+
'actor-memory-limit-exceeded',
|
|
9
|
+
'concurrent-runs-limit-exceeded',
|
|
10
|
+
]);
|
|
11
|
+
/** Cooldown between two attempts to start a run that was rejected for lack of resources. */
|
|
12
|
+
const WAIT_FOR_RESOURCES_COOLDOWN_MILLIS = 10_000;
|
|
13
|
+
/**
|
|
14
|
+
* Makes the `start` request, retrying it every {@link WAIT_FOR_RESOURCES_COOLDOWN_MILLIS} while it fails with one of
|
|
15
|
+
* {@link RESOURCE_LIMIT_ERROR_TYPES}. `true` retries until the request succeeds, a number of seconds bounds the
|
|
16
|
+
* retrying, after which the last error is thrown. Any other error is thrown right away.
|
|
17
|
+
* @internal
|
|
18
|
+
*/
|
|
19
|
+
export async function startWaitingForResources(start, waitForResources, logger, signal) {
|
|
20
|
+
if (!waitForResources)
|
|
21
|
+
return start();
|
|
22
|
+
const deadline = waitForResources === true ? Infinity : Date.now() + waitForResources * 1000;
|
|
23
|
+
for (;;) {
|
|
24
|
+
try {
|
|
25
|
+
return await start();
|
|
26
|
+
}
|
|
27
|
+
catch (err) {
|
|
28
|
+
if (!(err instanceof ApifyApiError) || !RESOURCE_LIMIT_ERROR_TYPES.has(err.type ?? ''))
|
|
29
|
+
throw err;
|
|
30
|
+
const remainingMillis = deadline - Date.now();
|
|
31
|
+
if (remainingMillis <= 0)
|
|
32
|
+
throw err;
|
|
33
|
+
const delayMillis = Math.min(WAIT_FOR_RESOURCES_COOLDOWN_MILLIS, remainingMillis);
|
|
34
|
+
logger.info(`Not enough resources to start the run, retrying in ${Number((delayMillis / 1000).toPrecision(3))}s: ${err.message}`);
|
|
35
|
+
await sleep(delayMillis, signal);
|
|
36
|
+
signal?.throwIfAborted();
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
package/package.json
CHANGED
|
@@ -1,15 +1,21 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "apify-client",
|
|
3
|
-
"version": "3.0.0-beta.
|
|
3
|
+
"version": "3.0.0-beta.51",
|
|
4
4
|
"description": "Apify API client for JavaScript",
|
|
5
5
|
"engines": {
|
|
6
|
-
"node": ">=22.
|
|
6
|
+
"node": ">=22.13.0"
|
|
7
7
|
},
|
|
8
8
|
"type": "module",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"types": "dist/index.d.ts",
|
|
11
11
|
"browser": "dist/bundle.js",
|
|
12
12
|
"unpkg": "dist/bundle.js",
|
|
13
|
+
"imports": {
|
|
14
|
+
"#runtime": {
|
|
15
|
+
"node": "./dist/runtime/node.js",
|
|
16
|
+
"default": "./dist/runtime/web.js"
|
|
17
|
+
}
|
|
18
|
+
},
|
|
13
19
|
"exports": {
|
|
14
20
|
"./package.json": "./package.json",
|
|
15
21
|
"./browser": "./dist/bundle.js",
|
|
@@ -53,58 +59,54 @@
|
|
|
53
59
|
],
|
|
54
60
|
"apify": {
|
|
55
61
|
"openapiSpec": {
|
|
56
|
-
"version": "v2-2026-
|
|
62
|
+
"version": "v2-2026-10-01T153946Z"
|
|
57
63
|
}
|
|
58
64
|
},
|
|
59
65
|
"dependencies": {
|
|
60
66
|
"@apify/consts": "^3.0.1",
|
|
61
67
|
"@apify/log": "^3.0.1",
|
|
62
|
-
"@apify/utilities": "^3.0
|
|
63
|
-
"@apify/validations": "^1.1.
|
|
64
|
-
"@crawlee/types": "^3.18.1",
|
|
68
|
+
"@apify/utilities": "^3.1.0",
|
|
69
|
+
"@apify/validations": "^1.1.1",
|
|
65
70
|
"ansi-colors": "^4.1.3",
|
|
66
|
-
"async-retry": "^1.3.3",
|
|
67
71
|
"axios": "^1.20.0",
|
|
68
|
-
"content-type": "^1.
|
|
69
|
-
"proxy-agent": "^
|
|
72
|
+
"content-type": "^3.1.1",
|
|
73
|
+
"proxy-agent": "^8.0.2",
|
|
70
74
|
"tslib": "^2.8.1",
|
|
71
|
-
"type-fest": "^
|
|
72
|
-
"zod": "^4.5
|
|
75
|
+
"type-fest": "^5.10.0",
|
|
76
|
+
"zod": "^4.6.5"
|
|
73
77
|
},
|
|
74
78
|
"devDependencies": {
|
|
75
79
|
"@apify/oxlint-config": "^0.3.0",
|
|
76
80
|
"@apify/tsconfig": "^0.2.0",
|
|
77
|
-
"@crawlee/puppeteer": "4.0.0-
|
|
78
|
-
"@rsbuild/core": "^2.2.
|
|
81
|
+
"@crawlee/puppeteer": "4.0.0-rc.0",
|
|
82
|
+
"@rsbuild/core": "^2.2.10",
|
|
79
83
|
"@rsbuild/plugin-node-polyfill": "^1.4.6",
|
|
80
|
-
"@rspack/cli": "^2.2.
|
|
81
|
-
"@rspack/core": "^2.2.
|
|
82
|
-
"@types/async-retry": "^1.4.9",
|
|
84
|
+
"@rspack/cli": "^2.2.8",
|
|
85
|
+
"@rspack/core": "^2.2.8",
|
|
83
86
|
"@types/compression": "^1.8.1",
|
|
84
|
-
"@types/content-type": "^1.1.9",
|
|
85
87
|
"@types/express": "^5.0.6",
|
|
86
|
-
"@types/node": "^24.
|
|
88
|
+
"@types/node": "^24.19.0",
|
|
87
89
|
"ajv": "^8.20.0",
|
|
88
90
|
"body-parser": "^2.3.0",
|
|
89
|
-
"compression": "^1.8.
|
|
91
|
+
"compression": "^1.8.2",
|
|
90
92
|
"esbuild": "0.28.2",
|
|
91
93
|
"express": "^5.2.1",
|
|
92
|
-
"
|
|
93
|
-
"
|
|
94
|
-
"oxlint": "
|
|
95
|
-
"
|
|
96
|
-
"puppeteer": "^25.10.0",
|
|
94
|
+
"oxfmt": "0.72.0",
|
|
95
|
+
"oxlint": "1.86.0",
|
|
96
|
+
"oxlint-tsgolint": "7.0.2003",
|
|
97
|
+
"puppeteer": "^25.12.0",
|
|
97
98
|
"rimraf": "^6.1.3",
|
|
98
|
-
"rolldown": "^1.2.
|
|
99
|
-
"typescript": "^
|
|
100
|
-
"
|
|
101
|
-
"
|
|
99
|
+
"rolldown": "^1.2.11",
|
|
100
|
+
"typescript": "^7.0.2",
|
|
101
|
+
"typescript-v6": "npm:typescript@^6.0.3",
|
|
102
|
+
"vitest": "^5.0.2",
|
|
103
|
+
"webpack": "^5.111.1",
|
|
102
104
|
"webpack-cli": "^7.2.3"
|
|
103
105
|
},
|
|
104
106
|
"devEngines": {
|
|
105
107
|
"packageManager": {
|
|
106
108
|
"name": "pnpm",
|
|
107
|
-
"version": "
|
|
109
|
+
"version": "12.8.1",
|
|
108
110
|
"onFail": "warn"
|
|
109
111
|
}
|
|
110
112
|
},
|
|
@@ -119,6 +121,7 @@
|
|
|
119
121
|
"lint:fix": "oxlint --type-aware --fix",
|
|
120
122
|
"tsc-check-tests": "tsc --noEmit --project test/tsconfig.json",
|
|
121
123
|
"tsc-check-scripts": "tsc --noEmit --project tsconfig.scripts.json",
|
|
124
|
+
"tsc-check-docs": "tsc --noEmit --project docs/tsconfig.json",
|
|
122
125
|
"format": "oxfmt",
|
|
123
126
|
"format:check": "oxfmt --check",
|
|
124
127
|
"api:extract": "pnpm build && pnpm dlx github:apify/api-extractor-report --packages=. --extract-command='pnpm api:extract'",
|