apify-client 3.0.0-beta.5 → 3.0.0-beta.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -3
- package/dist/apify_api_error.d.ts +62 -21
- package/dist/apify_api_error.js +96 -23
- package/dist/apify_client.d.ts +125 -10
- package/dist/apify_client.js +133 -48
- package/dist/base/api_client.d.ts +7 -7
- package/dist/base/api_client.js +36 -24
- package/dist/base/resource_client.d.ts +21 -8
- package/dist/base/resource_client.js +54 -26
- package/dist/base/resource_collection_client.d.ts +12 -4
- package/dist/base/resource_collection_client.js +34 -11
- package/dist/body_parser.d.ts +3 -4
- package/dist/body_parser.js +19 -26
- package/dist/bundle.js +27 -22
- package/dist/bundle.js.map +1 -1
- package/dist/generated/api.d.ts +610 -112
- package/dist/generated/schemas.d.ts +726 -703
- package/dist/generated/schemas.js +92 -85
- package/dist/http_clients/axios.d.ts +60 -0
- package/dist/http_clients/axios.js +117 -0
- package/dist/http_clients/base.d.ts +333 -0
- package/dist/http_clients/base.js +563 -0
- package/dist/http_clients/index.d.ts +4 -0
- package/dist/http_clients/index.js +2 -0
- package/dist/http_compressors/base.d.ts +40 -0
- package/dist/http_compressors/base.js +9 -0
- package/dist/http_compressors/brotli.d.ts +33 -0
- package/dist/http_compressors/brotli.js +39 -0
- package/dist/http_compressors/gzip.d.ts +33 -0
- package/dist/http_compressors/gzip.js +39 -0
- package/dist/http_compressors/index.d.ts +6 -0
- package/dist/http_compressors/index.js +2 -0
- package/dist/http_compressors/resolve.d.ts +23 -0
- package/dist/http_compressors/resolve.js +27 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +3 -1
- package/dist/invalid_response_body_error.d.ts +13 -0
- package/dist/invalid_response_body_error.js +17 -0
- package/dist/models.d.ts +19 -22
- package/dist/resource_clients/actor.d.ts +72 -32
- package/dist/resource_clients/actor.js +87 -56
- package/dist/resource_clients/actor_collection.d.ts +9 -5
- package/dist/resource_clients/actor_collection.js +11 -3
- package/dist/resource_clients/actor_env_var.d.ts +10 -3
- package/dist/resource_clients/actor_env_var.js +16 -6
- package/dist/resource_clients/actor_env_var_collection.d.ts +14 -18
- package/dist/resource_clients/actor_env_var_collection.js +17 -11
- package/dist/resource_clients/actor_version.d.ts +18 -3
- package/dist/resource_clients/actor_version.js +19 -9
- package/dist/resource_clients/actor_version_collection.d.ts +16 -17
- package/dist/resource_clients/actor_version_collection.js +19 -11
- package/dist/resource_clients/build.d.ts +18 -6
- package/dist/resource_clients/build.js +34 -15
- package/dist/resource_clients/build_collection.d.ts +3 -1
- package/dist/resource_clients/build_collection.js +4 -1
- package/dist/resource_clients/dataset.d.ts +51 -33
- package/dist/resource_clients/dataset.js +105 -80
- package/dist/resource_clients/dataset_collection.d.ts +6 -2
- package/dist/resource_clients/dataset_collection.js +13 -2
- package/dist/resource_clients/key_value_store.d.ts +58 -16
- package/dist/resource_clients/key_value_store.js +78 -40
- package/dist/resource_clients/key_value_store_collection.d.ts +6 -2
- package/dist/resource_clients/key_value_store_collection.js +13 -2
- package/dist/resource_clients/log.d.ts +11 -19
- package/dist/resource_clients/log.js +64 -46
- package/dist/resource_clients/request_queue.d.ts +56 -26
- package/dist/resource_clients/request_queue.js +176 -103
- package/dist/resource_clients/request_queue_collection.d.ts +6 -2
- package/dist/resource_clients/request_queue_collection.js +9 -3
- package/dist/resource_clients/run.d.ts +108 -18
- package/dist/resource_clients/run.js +183 -43
- package/dist/resource_clients/run_collection.d.ts +3 -1
- package/dist/resource_clients/run_collection.js +4 -1
- package/dist/resource_clients/schedule.d.ts +14 -5
- package/dist/resource_clients/schedule.js +29 -21
- package/dist/resource_clients/schedule_collection.d.ts +6 -2
- package/dist/resource_clients/schedule_collection.js +9 -3
- package/dist/resource_clients/store_collection.d.ts +3 -1
- package/dist/resource_clients/store_collection.js +4 -1
- package/dist/resource_clients/task.d.ts +33 -10
- package/dist/resource_clients/task.js +79 -46
- package/dist/resource_clients/task_collection.d.ts +6 -2
- package/dist/resource_clients/task_collection.js +9 -3
- package/dist/resource_clients/user.d.ts +17 -8
- package/dist/resource_clients/user.js +42 -38
- package/dist/resource_clients/webhook.d.ts +14 -5
- package/dist/resource_clients/webhook.js +30 -22
- package/dist/resource_clients/webhook_collection.d.ts +6 -2
- package/dist/resource_clients/webhook_collection.js +9 -3
- package/dist/resource_clients/webhook_dispatch.d.ts +4 -1
- package/dist/resource_clients/webhook_dispatch.js +7 -2
- package/dist/resource_clients/webhook_dispatch_collection.d.ts +3 -1
- package/dist/resource_clients/webhook_dispatch_collection.js +4 -1
- package/dist/runtime/node.d.ts +6 -0
- package/dist/runtime/node.js +49 -0
- package/dist/runtime/types.d.ts +52 -0
- package/dist/runtime/types.js +1 -0
- package/dist/runtime/web.d.ts +5 -0
- package/dist/runtime/web.js +18 -0
- package/dist/schemas.d.ts +1 -1
- package/dist/schemas.js +1 -1
- package/dist/statistics.d.ts +1 -6
- package/dist/statistics.js +2 -2
- package/dist/timeouts.d.ts +84 -0
- package/dist/timeouts.js +37 -0
- package/dist/timezones.d.ts +1 -1
- package/dist/utils.d.ts +81 -52
- package/dist/utils.js +201 -170
- package/dist/wait_for_resources.d.ts +8 -0
- package/dist/wait_for_resources.js +39 -0
- package/package.json +32 -29
- package/dist/http_client.d.ts +0 -88
- package/dist/http_client.js +0 -255
- package/dist/interceptors.d.ts +0 -19
- package/dist/interceptors.js +0 -105
package/dist/apify_client.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { ACTOR_ENV_VARS, ME_USER_NAME_PLACEHOLDER } from '@apify/consts';
|
|
3
3
|
import logger from '@apify/log';
|
|
4
|
-
import {
|
|
4
|
+
import { AxiosHttpClient } from './http_clients/axios.js';
|
|
5
|
+
import { DEFAULT_MAX_RETRIES, DEFAULT_MIN_DELAY_BETWEEN_RETRIES_MILLIS, HttpClient, httpClientOptionsShape, } from './http_clients/base.js';
|
|
6
|
+
import { compressionSchema } from './http_compressors/resolve.js';
|
|
5
7
|
import { ActorClient } from './resource_clients/actor.js';
|
|
6
8
|
import { ActorCollectionClient } from './resource_clients/actor_collection.js';
|
|
7
9
|
import { BuildClient } from './resource_clients/build.js';
|
|
@@ -26,22 +28,40 @@ import { WebhookCollectionClient } from './resource_clients/webhook_collection.j
|
|
|
26
28
|
import { WebhookDispatchClient } from './resource_clients/webhook_dispatch.js';
|
|
27
29
|
import { WebhookDispatchCollectionClient } from './resource_clients/webhook_dispatch_collection.js';
|
|
28
30
|
import { Statistics } from './statistics.js';
|
|
29
|
-
import {
|
|
30
|
-
|
|
31
|
+
import { DEFAULT_TIMEOUT_LONG_SECS, DEFAULT_TIMEOUT_MAX_SECS, DEFAULT_TIMEOUT_MEDIUM_SECS, DEFAULT_TIMEOUT_SHORT_SECS, } from './timeouts.js';
|
|
32
|
+
import { getEnv, parseArgument } from './utils.js';
|
|
33
|
+
const DEFAULT_API_URL = 'https://api.apify.com';
|
|
34
|
+
const API_VERSION_PATH = '/v2';
|
|
35
|
+
function toApiBaseUrl(url) {
|
|
36
|
+
const parsed = new URL(url);
|
|
37
|
+
const path = parsed.pathname.replace(/\/+$/, '');
|
|
38
|
+
parsed.pathname = path.endsWith(API_VERSION_PATH) ? path : `${path}${API_VERSION_PATH}`;
|
|
39
|
+
return parsed.href;
|
|
40
|
+
}
|
|
31
41
|
const clientOptionsSchema = z.strictObject({
|
|
32
|
-
baseUrl: z.string().default(
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
42
|
+
baseUrl: z.string().default(DEFAULT_API_URL),
|
|
43
|
+
compression: compressionSchema.default('brotli'),
|
|
44
|
+
headers: z.record(z.string(), z.string()).optional(),
|
|
45
|
+
publicBaseUrl: z.string().default(DEFAULT_API_URL),
|
|
46
|
+
maxRetries: httpClientOptionsShape.maxRetries.default(DEFAULT_MAX_RETRIES),
|
|
47
|
+
minDelayBetweenRetriesMillis: httpClientOptionsShape.minDelayBetweenRetriesMillis.default(DEFAULT_MIN_DELAY_BETWEEN_RETRIES_MILLIS),
|
|
48
|
+
timeoutShortSecs: httpClientOptionsShape.timeoutShortSecs.default(DEFAULT_TIMEOUT_SHORT_SECS),
|
|
49
|
+
timeoutMediumSecs: httpClientOptionsShape.timeoutMediumSecs.default(DEFAULT_TIMEOUT_MEDIUM_SECS),
|
|
50
|
+
timeoutLongSecs: httpClientOptionsShape.timeoutLongSecs.default(DEFAULT_TIMEOUT_LONG_SECS),
|
|
51
|
+
timeoutMaxSecs: httpClientOptionsShape.timeoutMaxSecs.default(DEFAULT_TIMEOUT_MAX_SECS),
|
|
38
52
|
token: z.string().optional(),
|
|
39
53
|
userAgentSuffix: z.union([z.string(), z.array(z.string())]).optional(),
|
|
40
54
|
});
|
|
55
|
+
const customHttpClientOptionsSchema = z.strictObject({
|
|
56
|
+
baseUrl: z.string().default(DEFAULT_API_URL),
|
|
57
|
+
publicBaseUrl: z.string().default(DEFAULT_API_URL),
|
|
58
|
+
token: z.string().optional(),
|
|
59
|
+
httpClient: z.instanceof(HttpClient),
|
|
60
|
+
});
|
|
41
61
|
const resourceIdSchema = z.string().min(1);
|
|
42
62
|
const requestQueueOptionsSchema = z.strictObject({
|
|
43
63
|
clientKey: z.string().min(1).optional(),
|
|
44
|
-
timeoutSecs: z.number().optional(),
|
|
64
|
+
timeoutSecs: z.number().positive().optional(),
|
|
45
65
|
});
|
|
46
66
|
/**
|
|
47
67
|
* The official JavaScript client for the Apify API.
|
|
@@ -71,33 +91,98 @@ export class ApifyClient {
|
|
|
71
91
|
*/
|
|
72
92
|
publicBaseUrl;
|
|
73
93
|
token;
|
|
94
|
+
/**
|
|
95
|
+
* Statistics of the API calls made through the client. With a custom HTTP client, this is the client's own
|
|
96
|
+
* `stats` object, so it only reflects what that client records.
|
|
97
|
+
*/
|
|
74
98
|
stats;
|
|
75
99
|
logger;
|
|
76
|
-
httpClient;
|
|
100
|
+
#httpClient;
|
|
101
|
+
/** Configuration of the default HTTP client, applied when no custom one is set. */
|
|
102
|
+
#httpClientOptions;
|
|
103
|
+
/**
|
|
104
|
+
* Creates a client that sends its requests through the built-in {@link AxiosHttpClient}, configured from
|
|
105
|
+
* `options`: the API token, retries, the timeout tiers, request compression and default headers. The HTTP
|
|
106
|
+
* client itself is created when {@link httpClient} is first read. To send the requests through a custom HTTP
|
|
107
|
+
* client, use {@link ApifyClient.withCustomHttpClient} instead.
|
|
108
|
+
*
|
|
109
|
+
* @param options - Configuration of the client and of its HTTP client.
|
|
110
|
+
* @throws {ArgumentValidationError} When an option is unknown or has an invalid value.
|
|
111
|
+
*/
|
|
77
112
|
constructor(options = {}) {
|
|
78
113
|
const parsed = parseArgument(options, clientOptionsSchema, 'ApifyClientOptions');
|
|
79
|
-
const { baseUrl, publicBaseUrl, maxRetries, minDelayBetweenRetriesMillis,
|
|
80
|
-
|
|
81
|
-
this.
|
|
82
|
-
const tempPublicBaseUrl = publicBaseUrl.endsWith('/')
|
|
83
|
-
? publicBaseUrl.slice(0, publicBaseUrl.length - 1)
|
|
84
|
-
: publicBaseUrl;
|
|
85
|
-
this.publicBaseUrl = `${tempPublicBaseUrl}/v2`;
|
|
114
|
+
const { baseUrl, compression, headers, publicBaseUrl, maxRetries, minDelayBetweenRetriesMillis, timeoutShortSecs, timeoutMediumSecs, timeoutLongSecs, timeoutMaxSecs, token, } = parsed;
|
|
115
|
+
this.baseUrl = toApiBaseUrl(baseUrl);
|
|
116
|
+
this.publicBaseUrl = toApiBaseUrl(publicBaseUrl);
|
|
86
117
|
this.token = token;
|
|
87
118
|
this.stats = new Statistics();
|
|
88
119
|
this.logger = logger.child({ prefix: 'ApifyClient' });
|
|
89
|
-
this
|
|
90
|
-
|
|
120
|
+
this.#httpClientOptions = {
|
|
121
|
+
stats: this.stats,
|
|
91
122
|
maxRetries,
|
|
92
123
|
minDelayBetweenRetriesMillis,
|
|
93
|
-
|
|
94
|
-
|
|
124
|
+
compression,
|
|
125
|
+
headers,
|
|
126
|
+
timeoutShortSecs,
|
|
127
|
+
timeoutMediumSecs,
|
|
128
|
+
timeoutLongSecs,
|
|
129
|
+
timeoutMaxSecs,
|
|
95
130
|
logger: this.logger,
|
|
96
131
|
token: this.token,
|
|
97
132
|
userAgentSuffix: parsed.userAgentSuffix,
|
|
98
|
-
}
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Creates an `ApifyClient` that sends its requests through a custom HTTP client.
|
|
137
|
+
*
|
|
138
|
+
* The custom client owns its transport configuration: retries, timeouts, default headers and whatever its
|
|
139
|
+
* transport needs are set on the client itself. This is also how the built-in {@link AxiosHttpClient} is
|
|
140
|
+
* configured beyond what the constructor options offer, for example with axios request interceptors. The
|
|
141
|
+
* token is set as the client's `Authorization` header, unless the client already has one configured.
|
|
142
|
+
*
|
|
143
|
+
* @param options - The token, the API URLs and the HTTP client to use.
|
|
144
|
+
* @returns A client whose {@link httpClient} is the given one.
|
|
145
|
+
*
|
|
146
|
+
* @example
|
|
147
|
+
* ```javascript
|
|
148
|
+
* import { ApifyClient, HttpClient } from 'apify-client';
|
|
149
|
+
*
|
|
150
|
+
* class MyHttpClient extends HttpClient {
|
|
151
|
+
* async sendRequest({ method, url, headers, body, timeoutMillis, stream, signal }) {
|
|
152
|
+
* // Send the request with the HTTP library of your choice and return { status, headers, body }.
|
|
153
|
+
* }
|
|
154
|
+
* }
|
|
155
|
+
*
|
|
156
|
+
* const client = ApifyClient.withCustomHttpClient({ token: 'my-token', httpClient: new MyHttpClient() });
|
|
157
|
+
* ```
|
|
158
|
+
*/
|
|
159
|
+
static withCustomHttpClient(options) {
|
|
160
|
+
const { httpClient, ...rest } = parseArgument(options, customHttpClientOptionsSchema, 'ApifyClientCustomHttpClientOptions');
|
|
161
|
+
const client = new ApifyClient(rest);
|
|
162
|
+
client.httpClient = httpClient;
|
|
163
|
+
return client;
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* The HTTP client the requests go through: the custom one when set through
|
|
167
|
+
* {@link ApifyClient.withCustomHttpClient}, and otherwise an {@link AxiosHttpClient} configured from the
|
|
168
|
+
* constructor options, created on first access. Assigning a client applies {@link token} to it through
|
|
169
|
+
* {@link HttpClient.setDefaultAuthorization} and points {@link stats} at its statistics, so the counters keep
|
|
170
|
+
* tracking the calls the client actually makes. Resource clients hold on to the HTTP client they were created
|
|
171
|
+
* with, so assign before reaching for them. Assigning does not close the client it replaces. A default client
|
|
172
|
+
* that already sent a request keeps its keep-alive agents open until {@link HttpClient.close} is called on it,
|
|
173
|
+
* so close it yourself, or assign before the first request.
|
|
174
|
+
*/
|
|
175
|
+
get httpClient() {
|
|
176
|
+
this.#httpClient ??= new AxiosHttpClient(this.#httpClientOptions);
|
|
177
|
+
return this.#httpClient;
|
|
178
|
+
}
|
|
179
|
+
set httpClient(httpClient) {
|
|
180
|
+
if (this.token)
|
|
181
|
+
httpClient.setDefaultAuthorization(this.token);
|
|
182
|
+
this.#httpClient = httpClient;
|
|
183
|
+
this.stats = httpClient.stats;
|
|
99
184
|
}
|
|
100
|
-
|
|
185
|
+
#subClientOptions() {
|
|
101
186
|
return {
|
|
102
187
|
baseUrl: this.baseUrl,
|
|
103
188
|
publicBaseUrl: this.publicBaseUrl,
|
|
@@ -114,7 +199,7 @@ export class ApifyClient {
|
|
|
114
199
|
* @see https://docs.apify.com/api/v2/acts-get
|
|
115
200
|
*/
|
|
116
201
|
actors() {
|
|
117
|
-
return new ActorCollectionClient(this
|
|
202
|
+
return new ActorCollectionClient(this.#subClientOptions());
|
|
118
203
|
}
|
|
119
204
|
/**
|
|
120
205
|
* Returns a client for a specific Actor.
|
|
@@ -136,7 +221,7 @@ export class ApifyClient {
|
|
|
136
221
|
parseArgument(id, resourceIdSchema);
|
|
137
222
|
return new ActorClient({
|
|
138
223
|
id,
|
|
139
|
-
...this
|
|
224
|
+
...this.#subClientOptions(),
|
|
140
225
|
});
|
|
141
226
|
}
|
|
142
227
|
/**
|
|
@@ -148,7 +233,7 @@ export class ApifyClient {
|
|
|
148
233
|
* @see https://docs.apify.com/api/v2/actor-builds-get
|
|
149
234
|
*/
|
|
150
235
|
builds() {
|
|
151
|
-
return new BuildCollectionClient(this
|
|
236
|
+
return new BuildCollectionClient(this.#subClientOptions());
|
|
152
237
|
}
|
|
153
238
|
/**
|
|
154
239
|
* Returns a client for a specific Actor build.
|
|
@@ -163,7 +248,7 @@ export class ApifyClient {
|
|
|
163
248
|
parseArgument(id, resourceIdSchema);
|
|
164
249
|
return new BuildClient({
|
|
165
250
|
id,
|
|
166
|
-
...this
|
|
251
|
+
...this.#subClientOptions(),
|
|
167
252
|
});
|
|
168
253
|
}
|
|
169
254
|
/**
|
|
@@ -175,7 +260,7 @@ export class ApifyClient {
|
|
|
175
260
|
* @see https://docs.apify.com/api/v2/datasets-get
|
|
176
261
|
*/
|
|
177
262
|
datasets() {
|
|
178
|
-
return new DatasetCollectionClient(this
|
|
263
|
+
return new DatasetCollectionClient(this.#subClientOptions());
|
|
179
264
|
}
|
|
180
265
|
/**
|
|
181
266
|
* Returns a client for a specific dataset.
|
|
@@ -204,7 +289,7 @@ export class ApifyClient {
|
|
|
204
289
|
parseArgument(id, resourceIdSchema);
|
|
205
290
|
return new DatasetClient({
|
|
206
291
|
id,
|
|
207
|
-
...this
|
|
292
|
+
...this.#subClientOptions(),
|
|
208
293
|
});
|
|
209
294
|
}
|
|
210
295
|
/**
|
|
@@ -216,7 +301,7 @@ export class ApifyClient {
|
|
|
216
301
|
* @see https://docs.apify.com/api/v2/key-value-stores-get
|
|
217
302
|
*/
|
|
218
303
|
keyValueStores() {
|
|
219
|
-
return new KeyValueStoreCollectionClient(this
|
|
304
|
+
return new KeyValueStoreCollectionClient(this.#subClientOptions());
|
|
220
305
|
}
|
|
221
306
|
/**
|
|
222
307
|
* Returns a client for a specific key-value store.
|
|
@@ -241,7 +326,7 @@ export class ApifyClient {
|
|
|
241
326
|
parseArgument(id, resourceIdSchema);
|
|
242
327
|
return new KeyValueStoreClient({
|
|
243
328
|
id,
|
|
244
|
-
...this
|
|
329
|
+
...this.#subClientOptions(),
|
|
245
330
|
});
|
|
246
331
|
}
|
|
247
332
|
/**
|
|
@@ -255,7 +340,7 @@ export class ApifyClient {
|
|
|
255
340
|
parseArgument(buildOrRunId, resourceIdSchema);
|
|
256
341
|
return new LogClient({
|
|
257
342
|
id: buildOrRunId,
|
|
258
|
-
...this
|
|
343
|
+
...this.#subClientOptions(),
|
|
259
344
|
});
|
|
260
345
|
}
|
|
261
346
|
/**
|
|
@@ -267,7 +352,7 @@ export class ApifyClient {
|
|
|
267
352
|
* @see https://docs.apify.com/api/v2/request-queues-get
|
|
268
353
|
*/
|
|
269
354
|
requestQueues() {
|
|
270
|
-
return new RequestQueueCollectionClient(this
|
|
355
|
+
return new RequestQueueCollectionClient(this.#subClientOptions());
|
|
271
356
|
}
|
|
272
357
|
/**
|
|
273
358
|
* Returns a client for a specific request queue.
|
|
@@ -295,7 +380,7 @@ export class ApifyClient {
|
|
|
295
380
|
const parsed = parseArgument(options, requestQueueOptionsSchema, 'RequestQueueUserOptions');
|
|
296
381
|
const apiClientOptions = {
|
|
297
382
|
id,
|
|
298
|
-
...this
|
|
383
|
+
...this.#subClientOptions(),
|
|
299
384
|
};
|
|
300
385
|
return new RequestQueueClient(apiClientOptions, parsed);
|
|
301
386
|
}
|
|
@@ -309,7 +394,7 @@ export class ApifyClient {
|
|
|
309
394
|
*/
|
|
310
395
|
runs() {
|
|
311
396
|
return new RunCollectionClient({
|
|
312
|
-
...this
|
|
397
|
+
...this.#subClientOptions(),
|
|
313
398
|
resourcePath: 'actor-runs',
|
|
314
399
|
});
|
|
315
400
|
}
|
|
@@ -336,7 +421,7 @@ export class ApifyClient {
|
|
|
336
421
|
parseArgument(id, resourceIdSchema);
|
|
337
422
|
return new RunClient({
|
|
338
423
|
id,
|
|
339
|
-
...this
|
|
424
|
+
...this.#subClientOptions(),
|
|
340
425
|
});
|
|
341
426
|
}
|
|
342
427
|
/**
|
|
@@ -348,7 +433,7 @@ export class ApifyClient {
|
|
|
348
433
|
* @see https://docs.apify.com/api/v2/actor-tasks-get
|
|
349
434
|
*/
|
|
350
435
|
tasks() {
|
|
351
|
-
return new TaskCollectionClient(this
|
|
436
|
+
return new TaskCollectionClient(this.#subClientOptions());
|
|
352
437
|
}
|
|
353
438
|
/**
|
|
354
439
|
* Returns a client for a specific Actor task.
|
|
@@ -369,7 +454,7 @@ export class ApifyClient {
|
|
|
369
454
|
parseArgument(id, resourceIdSchema);
|
|
370
455
|
return new TaskClient({
|
|
371
456
|
id,
|
|
372
|
-
...this
|
|
457
|
+
...this.#subClientOptions(),
|
|
373
458
|
});
|
|
374
459
|
}
|
|
375
460
|
/**
|
|
@@ -381,7 +466,7 @@ export class ApifyClient {
|
|
|
381
466
|
* @see https://docs.apify.com/api/v2/schedules-get
|
|
382
467
|
*/
|
|
383
468
|
schedules() {
|
|
384
|
-
return new ScheduleCollectionClient(this
|
|
469
|
+
return new ScheduleCollectionClient(this.#subClientOptions());
|
|
385
470
|
}
|
|
386
471
|
/**
|
|
387
472
|
* Returns a client for a specific schedule.
|
|
@@ -396,7 +481,7 @@ export class ApifyClient {
|
|
|
396
481
|
parseArgument(id, resourceIdSchema);
|
|
397
482
|
return new ScheduleClient({
|
|
398
483
|
id,
|
|
399
|
-
...this
|
|
484
|
+
...this.#subClientOptions(),
|
|
400
485
|
});
|
|
401
486
|
}
|
|
402
487
|
/**
|
|
@@ -412,7 +497,7 @@ export class ApifyClient {
|
|
|
412
497
|
parseArgument(id, resourceIdSchema);
|
|
413
498
|
return new UserClient({
|
|
414
499
|
id,
|
|
415
|
-
...this
|
|
500
|
+
...this.#subClientOptions(),
|
|
416
501
|
});
|
|
417
502
|
}
|
|
418
503
|
/**
|
|
@@ -424,7 +509,7 @@ export class ApifyClient {
|
|
|
424
509
|
* @see https://docs.apify.com/api/v2/webhooks-get
|
|
425
510
|
*/
|
|
426
511
|
webhooks() {
|
|
427
|
-
return new WebhookCollectionClient(this
|
|
512
|
+
return new WebhookCollectionClient(this.#subClientOptions());
|
|
428
513
|
}
|
|
429
514
|
/**
|
|
430
515
|
* Returns a client for a specific webhook.
|
|
@@ -439,7 +524,7 @@ export class ApifyClient {
|
|
|
439
524
|
parseArgument(id, resourceIdSchema);
|
|
440
525
|
return new WebhookClient({
|
|
441
526
|
id,
|
|
442
|
-
...this
|
|
527
|
+
...this.#subClientOptions(),
|
|
443
528
|
});
|
|
444
529
|
}
|
|
445
530
|
/**
|
|
@@ -451,7 +536,7 @@ export class ApifyClient {
|
|
|
451
536
|
* @see https://docs.apify.com/api/v2/webhook-dispatches-get
|
|
452
537
|
*/
|
|
453
538
|
webhookDispatches() {
|
|
454
|
-
return new WebhookDispatchCollectionClient(this
|
|
539
|
+
return new WebhookDispatchCollectionClient(this.#subClientOptions());
|
|
455
540
|
}
|
|
456
541
|
/**
|
|
457
542
|
* Returns a client for a specific webhook dispatch.
|
|
@@ -464,7 +549,7 @@ export class ApifyClient {
|
|
|
464
549
|
parseArgument(id, resourceIdSchema);
|
|
465
550
|
return new WebhookDispatchClient({
|
|
466
551
|
id,
|
|
467
|
-
...this
|
|
552
|
+
...this.#subClientOptions(),
|
|
468
553
|
});
|
|
469
554
|
}
|
|
470
555
|
/**
|
|
@@ -476,7 +561,7 @@ export class ApifyClient {
|
|
|
476
561
|
* @see https://docs.apify.com/api/v2/store-get
|
|
477
562
|
*/
|
|
478
563
|
store() {
|
|
479
|
-
return new StoreCollectionClient(this
|
|
564
|
+
return new StoreCollectionClient(this.#subClientOptions());
|
|
480
565
|
}
|
|
481
566
|
/**
|
|
482
567
|
* Sets a status message for the current Actor run.
|
|
@@ -490,7 +575,7 @@ export class ApifyClient {
|
|
|
490
575
|
* @since Added in 2.7.0
|
|
491
576
|
*/
|
|
492
577
|
async setStatusMessage(message, options) {
|
|
493
|
-
const runId =
|
|
578
|
+
const runId = getEnv(ACTOR_ENV_VARS.RUN_ID);
|
|
494
579
|
if (!runId) {
|
|
495
580
|
throw new Error(`Environment variable ${ACTOR_ENV_VARS.RUN_ID} is not set!`);
|
|
496
581
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ApifyClient } from '../apify_client.js';
|
|
2
|
-
import type { HttpClient } from '../
|
|
2
|
+
import type { HttpClient } from '../http_clients/index.js';
|
|
3
3
|
import type { PaginatedResponse, PaginationOptions } from '../utils.js';
|
|
4
4
|
/** @private */
|
|
5
5
|
export interface ApiClientOptions {
|
|
@@ -30,15 +30,15 @@ export declare abstract class ApiClient {
|
|
|
30
30
|
httpClient: HttpClient;
|
|
31
31
|
params?: Record<string, unknown>;
|
|
32
32
|
constructor(options: ApiClientOptions);
|
|
33
|
-
protected
|
|
34
|
-
protected
|
|
35
|
-
protected
|
|
36
|
-
protected
|
|
37
|
-
protected
|
|
33
|
+
protected subResourceOptions<T>(moreOptions?: T): BaseOptions & T;
|
|
34
|
+
protected buildUrl(path?: string | string[]): string;
|
|
35
|
+
protected buildPublicUrl(path?: string | string[]): string;
|
|
36
|
+
protected buildParams<T>(endpointParams?: T): Record<string, unknown>;
|
|
37
|
+
protected toSafeId(id: string): string;
|
|
38
38
|
/**
|
|
39
39
|
* Returns async iterator to iterate through all items and Promise that can be awaited to get first page of results.
|
|
40
40
|
*/
|
|
41
|
-
protected
|
|
41
|
+
protected listPaginatedFromCallback<T extends PaginationOptions, Data, R extends PaginatedResponse<Data>>(getPaginatedList: (options?: T) => Promise<R>, options?: T): AsyncIterable<Data> & Promise<R>;
|
|
42
42
|
}
|
|
43
43
|
export interface BaseOptions {
|
|
44
44
|
baseUrl: string;
|
package/dist/base/api_client.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { toPath, toPathSegment } from '../utils.js';
|
|
1
|
+
import { SCANNED_COUNT, toPath, toPathSegment } from '../utils.js';
|
|
2
2
|
/** @private */
|
|
3
3
|
export class ApiClient {
|
|
4
4
|
id;
|
|
@@ -16,7 +16,7 @@ export class ApiClient {
|
|
|
16
16
|
constructor(options) {
|
|
17
17
|
const { baseUrl, publicBaseUrl, apifyClient, httpClient, resourcePath, id, params = {} } = options;
|
|
18
18
|
this.id = id;
|
|
19
|
-
this.safeId = id && this.
|
|
19
|
+
this.safeId = id && this.toSafeId(id);
|
|
20
20
|
this.baseUrl = baseUrl;
|
|
21
21
|
this.publicBaseUrl = publicBaseUrl;
|
|
22
22
|
this.resourcePath = resourcePath;
|
|
@@ -25,35 +25,35 @@ export class ApiClient {
|
|
|
25
25
|
this.httpClient = httpClient;
|
|
26
26
|
this.params = params;
|
|
27
27
|
}
|
|
28
|
-
|
|
28
|
+
subResourceOptions(moreOptions) {
|
|
29
29
|
const baseOptions = {
|
|
30
|
-
baseUrl: this.
|
|
30
|
+
baseUrl: this.buildUrl(),
|
|
31
31
|
publicBaseUrl: this.publicBaseUrl,
|
|
32
32
|
apifyClient: this.apifyClient,
|
|
33
33
|
httpClient: this.httpClient,
|
|
34
|
-
params: this.
|
|
34
|
+
params: this.buildParams(),
|
|
35
35
|
};
|
|
36
36
|
return { ...baseOptions, ...moreOptions };
|
|
37
37
|
}
|
|
38
|
-
|
|
38
|
+
buildUrl(path) {
|
|
39
39
|
return path ? `${this.url}/${toPath(path)}` : this.url;
|
|
40
40
|
}
|
|
41
|
-
|
|
41
|
+
buildPublicUrl(path) {
|
|
42
42
|
const url = this.id
|
|
43
43
|
? `${this.publicBaseUrl}/${this.resourcePath}/${toPathSegment(this.safeId)}`
|
|
44
44
|
: `${this.publicBaseUrl}/${this.resourcePath}`;
|
|
45
45
|
return path ? `${url}/${toPath(path)}` : url;
|
|
46
46
|
}
|
|
47
|
-
|
|
47
|
+
buildParams(endpointParams) {
|
|
48
48
|
return { ...this.params, ...endpointParams };
|
|
49
49
|
}
|
|
50
|
-
|
|
50
|
+
toSafeId(id) {
|
|
51
51
|
return id.replaceAll('/', '~');
|
|
52
52
|
}
|
|
53
53
|
/**
|
|
54
54
|
* Returns async iterator to iterate through all items and Promise that can be awaited to get first page of results.
|
|
55
55
|
*/
|
|
56
|
-
|
|
56
|
+
listPaginatedFromCallback(getPaginatedList, options = {}) {
|
|
57
57
|
const minForLimitParam = (a, b) => {
|
|
58
58
|
// API treats 0 as undefined for limit parameter
|
|
59
59
|
if (a === 0)
|
|
@@ -67,30 +67,42 @@ export class ApiClient {
|
|
|
67
67
|
return Math.min(a, b);
|
|
68
68
|
};
|
|
69
69
|
// `chunkSize` only sizes this loop's requests; it is not an API parameter, so it must not reach
|
|
70
|
-
// `
|
|
70
|
+
// `buildParams()` and the query string. The same goes for `timeoutSecs` and `signal`, which callers take out
|
|
71
|
+
// before calling this, since they also apply to every page request.
|
|
71
72
|
const { chunkSize, ...listOptions } = options;
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
73
|
+
const firstPageLimit = minForLimitParam(options.limit, chunkSize);
|
|
74
|
+
const paginatedListPromise = getPaginatedList({ ...listOptions, limit: firstPageLimit });
|
|
75
|
+
// A page can return more or fewer items than the rows it scanned (see `SCANNED_COUNT`). The next offset and
|
|
76
|
+
// the stop condition follow the scanned rows: advancing by `items.length` would re-scan rows after a filter
|
|
77
|
+
// dropped some and skip rows after `unwind` multiplied them, and stopping at an empty page would end the
|
|
78
|
+
// iteration in front of rows a filter hid. The scanned number is derived from a dataset's item count, which
|
|
79
|
+
// lags a fresh push, so the larger of it and `items.length` is taken, capped at the rows the request asked for
|
|
80
|
+
// so that an unwound page does not advance past rows the next request would then never read.
|
|
81
|
+
const scannedRows = (page, requestedLimit) => {
|
|
82
|
+
const scanned = Math.max(page[SCANNED_COUNT] ?? 0, page.items.length);
|
|
83
|
+
return requestedLimit ? Math.min(scanned, requestedLimit) : scanned;
|
|
84
|
+
};
|
|
85
|
+
// `total` is not consulted, as it changes when the listed resource grows during the iteration.
|
|
76
86
|
async function* asyncGenerator() {
|
|
77
87
|
let currentPage = await paginatedListPromise;
|
|
78
88
|
yield* currentPage.items;
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
let
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
remainingItems
|
|
89
|
+
let pageScanned = scannedRows(currentPage, firstPageLimit);
|
|
90
|
+
let currentOffset = (options.offset ?? 0) + pageScanned;
|
|
91
|
+
let remainingItems = options.limit ? options.limit - pageScanned : undefined;
|
|
92
|
+
while (pageScanned > 0 && // Continue only if the last page scanned some rows.
|
|
93
|
+
(remainingItems === undefined || remainingItems > 0)) {
|
|
94
|
+
const requestedLimit = minForLimitParam(remainingItems, chunkSize);
|
|
85
95
|
const newOptions = {
|
|
86
96
|
...listOptions,
|
|
87
|
-
limit:
|
|
97
|
+
limit: requestedLimit,
|
|
88
98
|
offset: currentOffset,
|
|
89
99
|
};
|
|
90
100
|
currentPage = await getPaginatedList(newOptions);
|
|
91
101
|
yield* currentPage.items;
|
|
92
|
-
|
|
93
|
-
|
|
102
|
+
pageScanned = scannedRows(currentPage, requestedLimit);
|
|
103
|
+
currentOffset += pageScanned;
|
|
104
|
+
if (remainingItems !== undefined)
|
|
105
|
+
remainingItems -= pageScanned;
|
|
94
106
|
}
|
|
95
107
|
}
|
|
96
108
|
return Object.defineProperty(paginatedListPromise, Symbol.asyncIterator, {
|
|
@@ -1,25 +1,38 @@
|
|
|
1
1
|
import type { ACT_JOB_STATUSES } from '@apify/consts';
|
|
2
2
|
import type { z } from 'zod';
|
|
3
|
+
import type { Timeout, TimeoutOptions, TimeoutTier } from '../timeouts.js';
|
|
3
4
|
import { ApiClient } from './api_client.js';
|
|
4
|
-
export declare const SMALL_TIMEOUT_MILLIS: number;
|
|
5
|
-
export declare const MEDIUM_TIMEOUT_MILLIS: number;
|
|
6
|
-
export declare const DEFAULT_TIMEOUT_MILLIS: number;
|
|
7
5
|
/**
|
|
8
6
|
* Resource client.
|
|
9
7
|
* @private
|
|
10
8
|
*/
|
|
11
9
|
export declare class ResourceClient extends ApiClient {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Picks the timeout of a request that asks the API to hold its response with `waitForFinish`. The request
|
|
12
|
+
* gets the hold the caller asked for plus the round trip its tier allows, so the client does not abort a
|
|
13
|
+
* request while the API is still holding it - which for `start()` and `build()` would retry a call that
|
|
14
|
+
* creates a resource. An explicit per-call `timeoutSecs` is used as given.
|
|
15
|
+
*/
|
|
16
|
+
protected timeoutForWaitForFinish(timeoutSecs: Timeout | undefined, tier: TimeoutTier, waitForFinishSecs: number | undefined): Timeout;
|
|
17
|
+
/**
|
|
18
|
+
* A 404 resolves to `undefined` only when the client names its resource by ID. A chained client without one, such
|
|
19
|
+
* as `run.dataset()`, throws it instead (see `catchNotFoundForResourceOrThrow()`).
|
|
20
|
+
*/
|
|
21
|
+
protected getResource<T, R>(schema: z.ZodType, options: T, timeoutSecs: Timeout, signal?: AbortSignal): Promise<R | undefined>;
|
|
22
|
+
protected updateResource<T, R>(schema: z.ZodType, newFields: T, timeoutSecs: Timeout, signal?: AbortSignal): Promise<R>;
|
|
23
|
+
/**
|
|
24
|
+
* A 404 is swallowed, keeping the DELETE idempotent, only when the client names its resource by ID. A chained client
|
|
25
|
+
* without one throws it instead (see `catchNotFoundForResourceOrThrow()`).
|
|
26
|
+
*/
|
|
27
|
+
protected deleteResource(timeoutSecs: Timeout, signal?: AbortSignal): Promise<void>;
|
|
15
28
|
/**
|
|
16
29
|
* This function is used in Build and Run endpoints so it's kept
|
|
17
30
|
* here to stay DRY.
|
|
18
31
|
*/
|
|
19
|
-
protected
|
|
32
|
+
protected waitForJobFinish<R extends {
|
|
20
33
|
status: (typeof ACT_JOB_STATUSES)[keyof typeof ACT_JOB_STATUSES];
|
|
21
34
|
}>(schema: z.ZodType, options?: WaitForFinishOptions): Promise<R>;
|
|
22
35
|
}
|
|
23
|
-
export interface WaitForFinishOptions {
|
|
36
|
+
export interface WaitForFinishOptions extends TimeoutOptions {
|
|
24
37
|
waitSecs?: number;
|
|
25
38
|
}
|