apify-client 3.0.0-beta.2 → 3.0.0-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/apify_api_error.d.ts +60 -19
- package/dist/apify_api_error.js +88 -22
- package/dist/apify_client.d.ts +22 -3
- package/dist/apify_client.js +37 -31
- package/dist/base/api_client.d.ts +6 -6
- package/dist/base/api_client.js +24 -16
- package/dist/base/resource_client.d.ts +23 -9
- package/dist/base/resource_client.js +49 -25
- package/dist/base/resource_collection_client.d.ts +7 -4
- package/dist/base/resource_collection_client.js +21 -15
- package/dist/body_parser.d.ts +3 -4
- package/dist/body_parser.js +17 -24
- package/dist/bundle.js +38 -48
- package/dist/bundle.js.map +1 -1
- package/dist/generated/api.d.ts +287 -214
- package/dist/generated/schemas.d.ts +6699 -0
- package/dist/generated/schemas.js +1521 -0
- package/dist/http_client.d.ts +18 -51
- package/dist/http_client.js +90 -85
- package/dist/index.d.ts +3 -1
- package/dist/index.js +2 -1
- package/dist/interceptors.js +24 -6
- package/dist/lazy_schema.d.ts +8 -0
- package/dist/lazy_schema.js +11 -0
- package/dist/models.d.ts +32 -52
- package/dist/resource_clients/actor.d.ts +46 -31
- package/dist/resource_clients/actor.js +71 -53
- package/dist/resource_clients/actor_collection.d.ts +9 -5
- package/dist/resource_clients/actor_collection.js +12 -3
- package/dist/resource_clients/actor_env_var.d.ts +10 -3
- package/dist/resource_clients/actor_env_var.js +17 -6
- package/dist/resource_clients/actor_env_var_collection.d.ts +14 -18
- package/dist/resource_clients/actor_env_var_collection.js +18 -11
- package/dist/resource_clients/actor_version.d.ts +18 -3
- package/dist/resource_clients/actor_version.js +20 -9
- package/dist/resource_clients/actor_version_collection.d.ts +16 -17
- package/dist/resource_clients/actor_version_collection.js +20 -11
- package/dist/resource_clients/build.d.ts +18 -6
- package/dist/resource_clients/build.js +35 -17
- package/dist/resource_clients/build_collection.d.ts +3 -1
- package/dist/resource_clients/build_collection.js +5 -1
- package/dist/resource_clients/dataset.d.ts +23 -9
- package/dist/resource_clients/dataset.js +70 -48
- package/dist/resource_clients/dataset_collection.d.ts +6 -2
- package/dist/resource_clients/dataset_collection.js +13 -2
- package/dist/resource_clients/key_value_store.d.ts +48 -15
- package/dist/resource_clients/key_value_store.js +71 -40
- package/dist/resource_clients/key_value_store_collection.d.ts +6 -2
- package/dist/resource_clients/key_value_store_collection.js +13 -2
- package/dist/resource_clients/log.d.ts +9 -19
- package/dist/resource_clients/log.js +57 -45
- package/dist/resource_clients/request_queue.d.ts +56 -26
- package/dist/resource_clients/request_queue.js +178 -113
- package/dist/resource_clients/request_queue_collection.d.ts +6 -2
- package/dist/resource_clients/request_queue_collection.js +10 -3
- package/dist/resource_clients/run.d.ts +51 -17
- package/dist/resource_clients/run.js +85 -44
- package/dist/resource_clients/run_collection.d.ts +3 -1
- package/dist/resource_clients/run_collection.js +5 -1
- package/dist/resource_clients/schedule.d.ts +16 -7
- package/dist/resource_clients/schedule.js +31 -21
- package/dist/resource_clients/schedule_collection.d.ts +6 -2
- package/dist/resource_clients/schedule_collection.js +10 -3
- package/dist/resource_clients/store_collection.d.ts +3 -1
- package/dist/resource_clients/store_collection.js +5 -1
- package/dist/resource_clients/task.d.ts +28 -10
- package/dist/resource_clients/task.js +67 -44
- package/dist/resource_clients/task_collection.d.ts +6 -2
- package/dist/resource_clients/task_collection.js +10 -3
- package/dist/resource_clients/user.d.ts +17 -8
- package/dist/resource_clients/user.js +40 -39
- package/dist/resource_clients/webhook.d.ts +14 -5
- package/dist/resource_clients/webhook.js +30 -22
- package/dist/resource_clients/webhook_collection.d.ts +6 -2
- package/dist/resource_clients/webhook_collection.js +10 -3
- package/dist/resource_clients/webhook_dispatch.d.ts +4 -1
- package/dist/resource_clients/webhook_dispatch.js +8 -2
- package/dist/resource_clients/webhook_dispatch_collection.d.ts +3 -1
- package/dist/resource_clients/webhook_dispatch_collection.js +5 -1
- package/dist/response_validation_error.d.ts +26 -0
- package/dist/response_validation_error.js +37 -0
- package/dist/runtime/node.d.ts +6 -0
- package/dist/runtime/node.js +58 -0
- package/dist/runtime/types.d.ts +50 -0
- package/dist/runtime/types.js +1 -0
- package/dist/runtime/web.d.ts +5 -0
- package/dist/runtime/web.js +17 -0
- package/dist/schemas.d.ts +15 -0
- package/dist/schemas.js +15 -0
- package/dist/statistics.d.ts +1 -6
- package/dist/statistics.js +2 -2
- package/dist/timeouts.d.ts +72 -0
- package/dist/timeouts.js +31 -0
- package/dist/utils.d.ts +75 -55
- package/dist/utils.js +201 -180
- package/package.json +42 -35
- package/dist/argument_validation_error.d.ts +0 -17
- package/dist/argument_validation_error.js +0 -153
package/dist/utils.js
CHANGED
|
@@ -1,30 +1,60 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import {
|
|
2
|
+
import { NotFoundError } from './apify_api_error.js';
|
|
3
|
+
import { parseArgument } from '@apify/validations';
|
|
4
|
+
import { runtime } from '#runtime';
|
|
5
|
+
import { ResponseValidationError } from './response_validation_error.js';
|
|
3
6
|
// @ts-ignore if we enable `resolveJsonModule`, we end up with a `src` folder in `dist`
|
|
4
7
|
import packageJson from '../package.json' with { type: 'json' };
|
|
5
|
-
const NOT_FOUND_STATUS_CODE = 404;
|
|
6
|
-
const RECORD_NOT_FOUND_TYPE = 'record-not-found';
|
|
7
|
-
const RECORD_OR_TOKEN_NOT_FOUND_TYPE = 'record-or-token-not-found';
|
|
8
8
|
const MIN_COMPRESS_BYTES = 1024;
|
|
9
|
-
|
|
10
|
-
//
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
9
|
+
const textEncoder = new TextEncoder();
|
|
10
|
+
// Only the version, so a bundler can drop the rest of the manifest.
|
|
11
|
+
export const { version } = packageJson;
|
|
12
|
+
/** Media type prefixes whose payloads carry their own compression, so compressing the request body is wasted work. */
|
|
13
|
+
const ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES = ['audio/', 'image/', 'video/'];
|
|
14
|
+
/** Exact media types whose payloads carry their own compression. */
|
|
15
|
+
const ALREADY_COMPRESSED_MEDIA_TYPES = new Set([
|
|
16
|
+
'application/epub+zip',
|
|
17
|
+
'application/gzip',
|
|
18
|
+
'application/java-archive',
|
|
19
|
+
'application/vnd.android.package-archive',
|
|
20
|
+
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
|
21
|
+
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
22
|
+
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
23
|
+
'application/vnd.rar',
|
|
24
|
+
'application/x-7z-compressed',
|
|
25
|
+
'application/x-bzip',
|
|
26
|
+
'application/x-bzip2',
|
|
27
|
+
'application/x-gzip',
|
|
28
|
+
'application/x-rar-compressed',
|
|
29
|
+
'application/x-xz',
|
|
30
|
+
'application/x-zip-compressed',
|
|
31
|
+
'application/zip',
|
|
32
|
+
'application/zstd',
|
|
33
|
+
'font/woff',
|
|
34
|
+
'font/woff2',
|
|
35
|
+
]);
|
|
36
|
+
/** Uncompressed media types that sit under an already-compressed prefix, so compressing them still pays off. */
|
|
37
|
+
const COMPRESSIBLE_MEDIA_TYPES = new Set([
|
|
38
|
+
'audio/aiff',
|
|
39
|
+
'audio/basic',
|
|
40
|
+
'audio/l16',
|
|
41
|
+
'audio/l24',
|
|
42
|
+
'audio/midi',
|
|
43
|
+
'audio/vnd.wave',
|
|
44
|
+
'audio/wav',
|
|
45
|
+
'audio/wave',
|
|
46
|
+
'audio/x-aiff',
|
|
47
|
+
'audio/x-wav',
|
|
48
|
+
'image/bmp',
|
|
49
|
+
'image/tiff',
|
|
50
|
+
'image/vnd.adobe.photoshop',
|
|
51
|
+
'image/vnd.microsoft.icon',
|
|
52
|
+
'image/x-icon',
|
|
53
|
+
'image/x-ms-bmp',
|
|
54
|
+
]);
|
|
55
|
+
/** Structured syntax suffixes marking a media type as text even under an already-compressed prefix (`image/svg+xml`). */
|
|
56
|
+
const COMPRESSIBLE_MEDIA_TYPE_SUFFIXES = ['+json', '+xml'];
|
|
57
|
+
export { parseArgument };
|
|
28
58
|
/**
|
|
29
59
|
* Accepts any non-null, non-array object as a predicate for `z.custom()`.
|
|
30
60
|
* @internal
|
|
@@ -41,6 +71,29 @@ export function isNonArrayObject(value) {
|
|
|
41
71
|
export const anyObjectSchema = z.custom(isNonArrayObject, {
|
|
42
72
|
error: 'Invalid input: expected an object',
|
|
43
73
|
});
|
|
74
|
+
// Zod installs its English locale as a module-level side effect but ships `"sideEffects": false`, so
|
|
75
|
+
// any tree-shaking bundler drops it and every message degrades to a bare "Invalid input". Passing it
|
|
76
|
+
// in per parse keeps them intact without reaching into the zod config the whole process shares.
|
|
77
|
+
const { localeError } = z.locales.en();
|
|
78
|
+
/**
|
|
79
|
+
* Turns a JSON API response into the value a resource method returns: unwraps the `data` envelope and validates the
|
|
80
|
+
* result against `schema`, one of the schemas generated from the OpenAPI specification. The validated copy is what
|
|
81
|
+
* callers get, so it is the schema's output -- unknown fields and unknown enum values included, since the schemas let
|
|
82
|
+
* both through, date-time fields turned into `Date` objects, and URL fields normalized, since `z.url()` hands back the
|
|
83
|
+
* parsed URL's serialization.
|
|
84
|
+
*
|
|
85
|
+
* Throws {@link ResponseValidationError} when the response does not match the specification.
|
|
86
|
+
* @internal
|
|
87
|
+
*/
|
|
88
|
+
export function parseResponse(response, schema) {
|
|
89
|
+
const data = pluckData(response.data);
|
|
90
|
+
const result = schema.safeParse(data, { error: localeError });
|
|
91
|
+
if (!result.success) {
|
|
92
|
+
const { method = 'GET', url = '' } = response.config;
|
|
93
|
+
throw new ResponseValidationError(result.error, data, { method, url });
|
|
94
|
+
}
|
|
95
|
+
return result.data;
|
|
96
|
+
}
|
|
44
97
|
/**
|
|
45
98
|
* Returns object's 'data' property or throws if parameter is not an object,
|
|
46
99
|
* or an object without a 'data' property.
|
|
@@ -53,62 +106,23 @@ export function pluckData(obj) {
|
|
|
53
106
|
throw new Error(`Expected response object with a "data" property, but received: ${obj}`);
|
|
54
107
|
}
|
|
55
108
|
/**
|
|
56
|
-
*
|
|
57
|
-
* Otherwise rethrows error.
|
|
109
|
+
* Swallows a 404 Not Found API error and rethrows anything else.
|
|
58
110
|
*/
|
|
59
111
|
export function catchNotFoundOrThrow(err) {
|
|
60
|
-
|
|
61
|
-
const isNotFoundMessage = err.type === RECORD_NOT_FOUND_TYPE || err.type === RECORD_OR_TOKEN_NOT_FOUND_TYPE || err.httpMethod === 'head';
|
|
62
|
-
const isNotFoundError = isNotFoundStatus && isNotFoundMessage;
|
|
63
|
-
if (!isNotFoundError)
|
|
112
|
+
if (!(err instanceof NotFoundError))
|
|
64
113
|
throw err;
|
|
65
114
|
}
|
|
66
115
|
/**
|
|
67
|
-
*
|
|
68
|
-
* "createdAt").
|
|
69
|
-
*
|
|
70
|
-
* If you want parse other fields as well, you can provide a custom matcher function shouldParseField(). This
|
|
71
|
-
* admittedly awkward approach allows this function to be reused for various purposes without introducing potential
|
|
72
|
-
* breaking changes.
|
|
116
|
+
* Like `catchNotFoundOrThrow()`, but swallows the 404 only when the client names its resource by ID.
|
|
73
117
|
*
|
|
74
|
-
*
|
|
118
|
+
* A chained client without an ID, such as `run.dataset()` or `run.log()`, requests a path where a 404 can mean either
|
|
119
|
+
* the parent or the default sub-resource is missing. The response cannot tell the two apart, so the error propagates.
|
|
120
|
+
* @internal
|
|
75
121
|
*/
|
|
76
|
-
export function
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
// array spend a level.
|
|
81
|
-
//
|
|
82
|
-
// In a list response it also reaches one level into caller-owned blobs the API stores verbatim, so a
|
|
83
|
-
// listed request's `userData.foo.somethingAt` comes back as a `Date` rather than the string it was
|
|
84
|
-
// written as.
|
|
85
|
-
// TODO: Consider removing this limitation. It might came across as an annoying surprise as it's not communicated.
|
|
86
|
-
if (depth > 4) {
|
|
87
|
-
return input;
|
|
88
|
-
}
|
|
89
|
-
if (Array.isArray(input))
|
|
90
|
-
return input.map((child) => parseDateFields(child, shouldParseField, depth + 1));
|
|
91
|
-
if (!input || typeof input !== 'object')
|
|
92
|
-
return input;
|
|
93
|
-
return Object.entries(input).reduce((output, [k, v]) => {
|
|
94
|
-
const isValObject = !!v && typeof v === 'object';
|
|
95
|
-
if (k.endsWith('At') || (shouldParseField && shouldParseField(k))) {
|
|
96
|
-
if (v) {
|
|
97
|
-
const d = new Date(v);
|
|
98
|
-
output[k] = Number.isNaN(d.getTime()) ? v : d;
|
|
99
|
-
}
|
|
100
|
-
else {
|
|
101
|
-
output[k] = v;
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
else if (isValObject || Array.isArray(v)) {
|
|
105
|
-
output[k] = parseDateFields(v, shouldParseField, depth + 1);
|
|
106
|
-
}
|
|
107
|
-
else {
|
|
108
|
-
output[k] = v;
|
|
109
|
-
}
|
|
110
|
-
return output;
|
|
111
|
-
}, {});
|
|
122
|
+
export function catchNotFoundForResourceOrThrow(err, resourceId) {
|
|
123
|
+
if (!resourceId)
|
|
124
|
+
throw err;
|
|
125
|
+
catchNotFoundOrThrow(err);
|
|
112
126
|
}
|
|
113
127
|
/**
|
|
114
128
|
* Helper function that converts array of webhooks to base64 string
|
|
@@ -116,98 +130,116 @@ export function parseDateFields(input, shouldParseField = null, depth = 0) {
|
|
|
116
130
|
export function stringifyWebhooksToBase64(webhooks) {
|
|
117
131
|
if (!webhooks)
|
|
118
132
|
return;
|
|
119
|
-
|
|
120
|
-
if (isNode()) {
|
|
121
|
-
return Buffer.from(webhooksJson, 'utf8').toString('base64');
|
|
122
|
-
}
|
|
123
|
-
const encoder = new TextEncoder();
|
|
124
|
-
const uint8Array = encoder.encode(webhooksJson);
|
|
125
|
-
return btoa(String.fromCharCode(...uint8Array));
|
|
133
|
+
return bytesToBase64(textEncoder.encode(JSON.stringify(webhooks)));
|
|
126
134
|
}
|
|
127
|
-
let brotliCompressPromisified;
|
|
128
135
|
/**
|
|
129
|
-
*
|
|
136
|
+
* Encodes bytes as base64. `btoa()` takes a binary string, and the bytes are turned into one in slices,
|
|
137
|
+
* because spreading them all into a single `String.fromCharCode()` call overflows the argument limit on
|
|
138
|
+
* inputs of a few tens of kilobytes.
|
|
130
139
|
*/
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
const options = { params: { [constants.BROTLI_PARAM_QUALITY]: 6 } };
|
|
137
|
-
brotliCompressPromisified = async (input) => compress(input, options);
|
|
140
|
+
export function bytesToBase64(bytes) {
|
|
141
|
+
const SLICE_LENGTH = 0x8000;
|
|
142
|
+
let binary = '';
|
|
143
|
+
for (let i = 0; i < bytes.length; i += SLICE_LENGTH) {
|
|
144
|
+
binary += String.fromCharCode(...bytes.subarray(i, i + SLICE_LENGTH));
|
|
138
145
|
}
|
|
139
|
-
return
|
|
146
|
+
return btoa(binary);
|
|
140
147
|
}
|
|
141
|
-
let gzipPromisified;
|
|
142
148
|
/**
|
|
143
|
-
*
|
|
149
|
+
* Concatenates byte chunks into one array.
|
|
144
150
|
*/
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
151
|
+
export function concatBytes(chunks) {
|
|
152
|
+
const result = new Uint8Array(chunks.reduce((length, chunk) => length + chunk.length, 0));
|
|
153
|
+
let offset = 0;
|
|
154
|
+
for (const chunk of chunks) {
|
|
155
|
+
result.set(chunk, offset);
|
|
156
|
+
offset += chunk.length;
|
|
150
157
|
}
|
|
151
|
-
return
|
|
158
|
+
return result;
|
|
152
159
|
}
|
|
153
160
|
/**
|
|
154
|
-
*
|
|
155
|
-
*
|
|
161
|
+
* Views a request body as bytes: a string is UTF-8 encoded, binary values are viewed in place. Anything else
|
|
162
|
+
* - a stream, a `Blob`, form data - is `undefined`.
|
|
156
163
|
*/
|
|
157
|
-
|
|
158
|
-
if (
|
|
164
|
+
function toBytes(value) {
|
|
165
|
+
if (typeof value === 'string')
|
|
166
|
+
return textEncoder.encode(value);
|
|
167
|
+
if (!isBuffer(value))
|
|
159
168
|
return undefined;
|
|
169
|
+
return ArrayBuffer.isView(value)
|
|
170
|
+
? new Uint8Array(value.buffer, value.byteOffset, value.byteLength)
|
|
171
|
+
: new Uint8Array(value);
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Decides whether a request body with the given content type is worth compressing.
|
|
175
|
+
*
|
|
176
|
+
* Images, audio, video and archives already carry their own compression. Running them through brotli or gzip
|
|
177
|
+
* burns CPU, holds a second full copy of the body in memory, and usually produces output slightly larger than
|
|
178
|
+
* the input. Formats that are raw despite such a media type, for example `image/bmp` or `audio/wav`, are still
|
|
179
|
+
* compressed. A body with no content type is assumed to be compressible.
|
|
180
|
+
* @internal
|
|
181
|
+
*/
|
|
182
|
+
export function isCompressibleContentType(contentType) {
|
|
183
|
+
if (!contentType)
|
|
184
|
+
return true;
|
|
185
|
+
// `Content-Type` is case-insensitive and may carry parameters, for example `text/plain; charset=utf-8`.
|
|
186
|
+
const mediaType = contentType.split(';', 1)[0].trim().toLowerCase();
|
|
187
|
+
if (COMPRESSIBLE_MEDIA_TYPES.has(mediaType))
|
|
188
|
+
return true;
|
|
189
|
+
if (COMPRESSIBLE_MEDIA_TYPE_SUFFIXES.some((suffix) => mediaType.endsWith(suffix)))
|
|
190
|
+
return true;
|
|
191
|
+
if (ALREADY_COMPRESSED_MEDIA_TYPES.has(mediaType))
|
|
192
|
+
return false;
|
|
193
|
+
return !ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES.some((prefix) => mediaType.startsWith(prefix));
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Compresses the passed value with the runtime's best available algorithm. Returns `undefined` if the data
|
|
197
|
+
* is too small, is not a string or binary value, or if the runtime does not offer compression.
|
|
198
|
+
*/
|
|
199
|
+
export async function maybeCompressValue(value) {
|
|
160
200
|
// Request compression is not that important so let's
|
|
161
201
|
// skip it instead of throwing for unsupported types.
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
const areDataLargeEnough = Buffer.byteLength(value) >= MIN_COMPRESS_BYTES;
|
|
165
|
-
if (!areDataLargeEnough)
|
|
166
|
-
return undefined;
|
|
167
|
-
try {
|
|
168
|
-
return { data: await brotliValue(value), encoding: 'br' };
|
|
169
|
-
}
|
|
170
|
-
catch {
|
|
171
|
-
// Runtimes that only provide a partial `node:zlib` (bundler polyfills, edge runtimes with
|
|
172
|
-
// Node compatibility shims) may not implement brotli, but usually do implement gzip.
|
|
173
|
-
}
|
|
174
|
-
try {
|
|
175
|
-
return { data: await gzipValue(value), encoding: 'gzip' };
|
|
176
|
-
}
|
|
177
|
-
catch {
|
|
178
|
-
// Same reasoning as above: compression is a best-effort optimization, so skip it instead
|
|
179
|
-
// of failing the request.
|
|
202
|
+
const bytes = toBytes(value);
|
|
203
|
+
if (!bytes || bytes.byteLength < MIN_COMPRESS_BYTES)
|
|
180
204
|
return undefined;
|
|
181
|
-
|
|
205
|
+
return runtime.compress(bytes);
|
|
182
206
|
}
|
|
183
207
|
/**
|
|
184
|
-
*
|
|
208
|
+
* Reads an environment variable, on runtimes that have them.
|
|
185
209
|
*/
|
|
186
|
-
export function
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
210
|
+
export function getEnv(name) {
|
|
211
|
+
return typeof process !== 'undefined' ? process.env?.[name] : undefined;
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Returns the UTF-8 byte length of a string.
|
|
215
|
+
*/
|
|
216
|
+
export function utf8ByteLength(value) {
|
|
217
|
+
return textEncoder.encode(value).byteLength;
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Splits JSON-serialized items into consecutive batches of at most `maxCount` items, each of which fits into a JSON
|
|
221
|
+
* array body - the items joined by commas between brackets - of at most `maxByteLength` bytes. The `byteLength` of an
|
|
222
|
+
* item is the UTF-8 byte length of its serialization. An item too large for a body of its own still gets one, so a
|
|
223
|
+
* caller that cannot send such an item has to reject it beforehand.
|
|
224
|
+
*/
|
|
225
|
+
export function splitIntoJsonArrayBatches(items, { maxCount, maxByteLength }) {
|
|
226
|
+
const batches = [];
|
|
227
|
+
let batch = [];
|
|
228
|
+
// One byte for the opening bracket; each item then adds its own bytes plus one for the comma or the closing
|
|
229
|
+
// bracket that follows it.
|
|
230
|
+
let byteLength = 1;
|
|
231
|
+
for (const item of items) {
|
|
232
|
+
if (batch.length > 0 && (batch.length >= maxCount || byteLength + item.byteLength + 1 > maxByteLength)) {
|
|
233
|
+
batches.push(batch);
|
|
234
|
+
batch = [];
|
|
235
|
+
byteLength = 1;
|
|
199
236
|
}
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
byteLength += itemByteSize;
|
|
203
|
-
slicedArray.push(item);
|
|
237
|
+
batch.push(item);
|
|
238
|
+
byteLength += item.byteLength + 1;
|
|
204
239
|
}
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
if (typeof BROWSER_BUILD !== 'undefined')
|
|
209
|
-
return false;
|
|
210
|
-
return !!(typeof process !== 'undefined' && process.versions && process.versions.node);
|
|
240
|
+
if (batch.length > 0)
|
|
241
|
+
batches.push(batch);
|
|
242
|
+
return batches;
|
|
211
243
|
}
|
|
212
244
|
export function isBuffer(value) {
|
|
213
245
|
// Tag checks rather than `instanceof`, to also match buffers from another realm. `isView()`
|
|
@@ -225,41 +257,30 @@ export function isStream(value) {
|
|
|
225
257
|
const { on, pipe } = value;
|
|
226
258
|
return typeof on === 'function' && typeof pipe === 'function';
|
|
227
259
|
}
|
|
228
|
-
export function getVersionData() {
|
|
229
|
-
if (typeof BROWSER_BUILD !== 'undefined') {
|
|
230
|
-
return { version: VERSION };
|
|
231
|
-
}
|
|
232
|
-
return packageJson;
|
|
233
|
-
}
|
|
234
260
|
/**
|
|
235
|
-
* Helper class to create async iterators from paginated list endpoints
|
|
261
|
+
* Helper class to create async iterators from paginated list endpoints.
|
|
236
262
|
*/
|
|
237
263
|
export class RequestQueuePaginationIterator {
|
|
238
|
-
maxPageLimit;
|
|
239
|
-
getPage;
|
|
240
|
-
limit;
|
|
241
|
-
|
|
242
|
-
cursor;
|
|
264
|
+
#maxPageLimit;
|
|
265
|
+
#getPage;
|
|
266
|
+
#limit;
|
|
267
|
+
#cursor;
|
|
243
268
|
constructor(options) {
|
|
244
|
-
this
|
|
245
|
-
this
|
|
246
|
-
this
|
|
247
|
-
this
|
|
248
|
-
this.getPage = options.getPage;
|
|
269
|
+
this.#maxPageLimit = options.maxPageLimit;
|
|
270
|
+
this.#limit = options.limit;
|
|
271
|
+
this.#cursor = options.cursor;
|
|
272
|
+
this.#getPage = options.getPage;
|
|
249
273
|
}
|
|
250
274
|
async *[Symbol.asyncIterator]() {
|
|
251
|
-
let nextCursor = this
|
|
252
|
-
// allow using exclusiveStartId for the first page, but then we'll delete it to avoid using it for any later page
|
|
253
|
-
let nextExclusiveStartId = this.exclusiveStartId;
|
|
275
|
+
let nextCursor = this.#cursor;
|
|
254
276
|
let iterateItemCount = 0;
|
|
255
277
|
while (true) {
|
|
256
|
-
const pageLimit = this
|
|
257
|
-
? Math.min(this
|
|
258
|
-
: this
|
|
259
|
-
const page = await this
|
|
278
|
+
const pageLimit = this.#limit
|
|
279
|
+
? Math.min(this.#maxPageLimit, this.#limit - iterateItemCount)
|
|
280
|
+
: this.#maxPageLimit;
|
|
281
|
+
const page = await this.#getPage({
|
|
260
282
|
limit: pageLimit,
|
|
261
283
|
cursor: nextCursor,
|
|
262
|
-
exclusiveStartId: nextExclusiveStartId,
|
|
263
284
|
});
|
|
264
285
|
// There are no more pages to iterate
|
|
265
286
|
if (page.items.length === 0)
|
|
@@ -267,10 +288,9 @@ export class RequestQueuePaginationIterator {
|
|
|
267
288
|
yield page;
|
|
268
289
|
iterateItemCount += page.items.length;
|
|
269
290
|
// Limit reached stopping to iterate
|
|
270
|
-
if ((this
|
|
291
|
+
if ((this.#limit && iterateItemCount >= this.#limit) || !page.nextCursor)
|
|
271
292
|
return;
|
|
272
293
|
nextCursor = page.nextCursor;
|
|
273
|
-
nextExclusiveStartId = undefined; // see comment above - delete it for any page after the first one, and paginate with cursor
|
|
274
294
|
}
|
|
275
295
|
}
|
|
276
296
|
}
|
|
@@ -284,6 +304,16 @@ export const paginationOptionsShape = {
|
|
|
284
304
|
offset: z.number().min(0).optional(),
|
|
285
305
|
chunkSize: z.number().positive().optional(),
|
|
286
306
|
};
|
|
307
|
+
/**
|
|
308
|
+
* Key under which a page of dataset items carries the number of rows the API scanned to produce it, as a
|
|
309
|
+
* non-enumerable property that stays out of the page's public shape. The API applies `offset` and `limit` to the
|
|
310
|
+
* dataset's rows first and shapes the result afterwards: filters (`clean`, `skipEmpty`, `skipHidden`) drop items and
|
|
311
|
+
* `unwind` splits or drops them, so `items.length` can land on either side of the rows scanned. The
|
|
312
|
+
* `x-apify-pagination-count` header reports the scanned number, and the offset iterator advances and terminates by it
|
|
313
|
+
* alone. A page without the property falls back to `items.length`.
|
|
314
|
+
* @internal
|
|
315
|
+
*/
|
|
316
|
+
export const SCANNED_COUNT = Symbol('scannedCount');
|
|
287
317
|
export function cast(input) {
|
|
288
318
|
return input;
|
|
289
319
|
}
|
|
@@ -310,15 +340,6 @@ export function applyQueryParamsToUrl(url, options) {
|
|
|
310
340
|
}
|
|
311
341
|
return url;
|
|
312
342
|
}
|
|
313
|
-
/**
|
|
314
|
-
* Builds a `[check, message]` pair to spread into `.refine()`, asserting that at most one of `keys`
|
|
315
|
-
* is present. Pass the options interface as `T`, so that a misspelled key is a type error.
|
|
316
|
-
* @internal
|
|
317
|
-
*/
|
|
318
|
-
export const mutuallyExclusive = (...keys) => [
|
|
319
|
-
(value) => keys.filter((key) => typeof value[key] !== 'undefined').length <= 1,
|
|
320
|
-
`At most one of the following fields is allowed: ${keys.join(', ')}`,
|
|
321
|
-
];
|
|
322
343
|
const pathSegmentSchema = z
|
|
323
344
|
.string()
|
|
324
345
|
.nonempty()
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "apify-client",
|
|
3
|
-
"version": "3.0.0-beta.
|
|
3
|
+
"version": "3.0.0-beta.21",
|
|
4
4
|
"description": "Apify API client for JavaScript",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -10,6 +10,12 @@
|
|
|
10
10
|
"types": "dist/index.d.ts",
|
|
11
11
|
"browser": "dist/bundle.js",
|
|
12
12
|
"unpkg": "dist/bundle.js",
|
|
13
|
+
"imports": {
|
|
14
|
+
"#runtime": {
|
|
15
|
+
"node": "./dist/runtime/node.js",
|
|
16
|
+
"default": "./dist/runtime/web.js"
|
|
17
|
+
}
|
|
18
|
+
},
|
|
13
19
|
"exports": {
|
|
14
20
|
"./package.json": "./package.json",
|
|
15
21
|
"./browser": "./dist/bundle.js",
|
|
@@ -53,52 +59,53 @@
|
|
|
53
59
|
],
|
|
54
60
|
"apify": {
|
|
55
61
|
"openapiSpec": {
|
|
56
|
-
"version": "v2-2026-08-
|
|
62
|
+
"version": "v2-2026-08-31T125154Z"
|
|
57
63
|
}
|
|
58
64
|
},
|
|
59
65
|
"dependencies": {
|
|
60
|
-
"@apify/consts": "^
|
|
61
|
-
"@apify/log": "^
|
|
62
|
-
"@apify/utilities": "^
|
|
63
|
-
"@
|
|
64
|
-
"
|
|
66
|
+
"@apify/consts": "^3.0.1",
|
|
67
|
+
"@apify/log": "^3.0.1",
|
|
68
|
+
"@apify/utilities": "^3.0.1",
|
|
69
|
+
"@apify/validations": "^1.1.0",
|
|
70
|
+
"@crawlee/types": "^3.18.1",
|
|
71
|
+
"ansi-colors": "^4.1.3",
|
|
65
72
|
"async-retry": "^1.3.3",
|
|
66
|
-
"axios": "^1.
|
|
73
|
+
"axios": "^1.20.0",
|
|
67
74
|
"content-type": "^1.0.5",
|
|
68
|
-
"proxy-agent": "^
|
|
69
|
-
"tslib": "^2.
|
|
70
|
-
"type-fest": "^4.
|
|
71
|
-
"zod": "^4.
|
|
75
|
+
"proxy-agent": "^8.0.2",
|
|
76
|
+
"tslib": "^2.8.1",
|
|
77
|
+
"type-fest": "^4.41.0",
|
|
78
|
+
"zod": "^4.5.4"
|
|
72
79
|
},
|
|
73
80
|
"devDependencies": {
|
|
74
81
|
"@apify/oxlint-config": "^0.3.0",
|
|
75
82
|
"@apify/tsconfig": "^0.2.0",
|
|
76
|
-
"@crawlee/puppeteer": "
|
|
77
|
-
"@rsbuild/core": "^2.
|
|
78
|
-
"@rsbuild/plugin-node-polyfill": "^1.
|
|
79
|
-
"@rspack/cli": "^2.
|
|
80
|
-
"@rspack/core": "^2.
|
|
81
|
-
"@types/async-retry": "^1.4.
|
|
83
|
+
"@crawlee/puppeteer": "4.0.0-beta.165",
|
|
84
|
+
"@rsbuild/core": "^2.2.3",
|
|
85
|
+
"@rsbuild/plugin-node-polyfill": "^1.4.6",
|
|
86
|
+
"@rspack/cli": "^2.2.2",
|
|
87
|
+
"@rspack/core": "^2.2.2",
|
|
88
|
+
"@types/async-retry": "^1.4.9",
|
|
82
89
|
"@types/compression": "^1.8.1",
|
|
83
|
-
"@types/content-type": "^1.1.
|
|
84
|
-
"@types/express": "^5.0.
|
|
85
|
-
"@types/node": "^24.
|
|
86
|
-
"ajv": "^8.
|
|
87
|
-
"body-parser": "^2.
|
|
88
|
-
"compression": "^1.
|
|
90
|
+
"@types/content-type": "^1.1.9",
|
|
91
|
+
"@types/express": "^5.0.6",
|
|
92
|
+
"@types/node": "^24.13.3",
|
|
93
|
+
"ajv": "^8.20.0",
|
|
94
|
+
"body-parser": "^2.3.0",
|
|
95
|
+
"compression": "^1.8.1",
|
|
89
96
|
"esbuild": "0.28.2",
|
|
90
|
-
"express": "^5.
|
|
97
|
+
"express": "^5.2.1",
|
|
91
98
|
"openapi-typescript": "7.13.0",
|
|
92
|
-
"oxfmt": "0.
|
|
93
|
-
"oxlint": "1.
|
|
99
|
+
"oxfmt": "0.66.0",
|
|
100
|
+
"oxlint": "1.81.0",
|
|
94
101
|
"oxlint-tsgolint": "7.0.2001",
|
|
95
|
-
"puppeteer": "^25.
|
|
96
|
-
"rimraf": "^6.
|
|
97
|
-
"rolldown": "^1.
|
|
98
|
-
"typescript": "^6.0.
|
|
99
|
-
"vitest": "^
|
|
100
|
-
"webpack": "^5.
|
|
101
|
-
"webpack-cli": "^7.
|
|
102
|
+
"puppeteer": "^25.10.0",
|
|
103
|
+
"rimraf": "^6.1.3",
|
|
104
|
+
"rolldown": "^1.2.7",
|
|
105
|
+
"typescript": "^6.0.3",
|
|
106
|
+
"vitest": "^5.0.0",
|
|
107
|
+
"webpack": "^5.110.3",
|
|
108
|
+
"webpack-cli": "^7.2.3"
|
|
102
109
|
},
|
|
103
110
|
"devEngines": {
|
|
104
111
|
"packageManager": {
|
|
@@ -125,6 +132,6 @@
|
|
|
125
132
|
"build:node": "tsc",
|
|
126
133
|
"build:browser": "rsbuild build",
|
|
127
134
|
"spec:fetch": "node scripts/openapi_spec.mts fetch",
|
|
128
|
-
"generate:
|
|
135
|
+
"generate:models": "node scripts/openapi_spec.mts fetch && node scripts/generate_types.mts && node scripts/generate_schemas.mts && node scripts/openapi_spec.mts record-version"
|
|
129
136
|
}
|
|
130
137
|
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import type { z } from 'zod';
|
|
2
|
-
/**
|
|
3
|
-
* Thrown when an argument fails schema validation.
|
|
4
|
-
*
|
|
5
|
-
* Its `message` is a human-readable sentence naming the offending field and the
|
|
6
|
-
* value it received (rather than a raw JSON dump). The structured
|
|
7
|
-
* {@link https://zod.dev | zod} issues are available on `issues`, and the
|
|
8
|
-
* original `ZodError` on `cause`, for programmatic inspection.
|
|
9
|
-
*
|
|
10
|
-
* `apify-client` sits below `@crawlee/core` and the Apify SDK in the dependency
|
|
11
|
-
* graph, so it defines its own error type rather than importing one from them.
|
|
12
|
-
*/
|
|
13
|
-
export declare class ArgumentValidationError extends Error {
|
|
14
|
-
/** Structured issues from the underlying schema check. */
|
|
15
|
-
readonly issues: z.ZodError['issues'];
|
|
16
|
-
constructor(error: z.ZodError, value: unknown, label?: string);
|
|
17
|
-
}
|