apify-client 3.0.0-beta.9 → 3.0.0-beta.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -2
- package/dist/apify_api_error.d.ts +13 -27
- package/dist/apify_api_error.js +33 -23
- package/dist/apify_client.d.ts +125 -10
- package/dist/apify_client.js +133 -48
- package/dist/apify_client_error.d.ts +10 -0
- package/dist/apify_client_error.js +10 -0
- package/dist/base/api_client.d.ts +7 -7
- package/dist/base/api_client.js +36 -36
- package/dist/base/resource_client.d.ts +21 -8
- package/dist/base/resource_client.js +70 -36
- package/dist/base/resource_collection_client.d.ts +12 -4
- package/dist/base/resource_collection_client.js +34 -11
- package/dist/body_parser.d.ts +3 -4
- package/dist/body_parser.js +19 -26
- package/dist/bundle.js +27 -22
- package/dist/bundle.js.map +1 -1
- package/dist/generated/api.d.ts +3040 -971
- package/dist/generated/schemas.d.ts +2369 -934
- package/dist/generated/schemas.js +570 -229
- package/dist/http_clients/axios.d.ts +61 -0
- package/dist/http_clients/axios.js +119 -0
- package/dist/http_clients/base.d.ts +386 -0
- package/dist/http_clients/base.js +650 -0
- package/dist/http_clients/index.d.ts +4 -0
- package/dist/http_clients/index.js +2 -0
- package/dist/http_compressors/base.d.ts +40 -0
- package/dist/http_compressors/base.js +9 -0
- package/dist/http_compressors/brotli.d.ts +33 -0
- package/dist/http_compressors/brotli.js +39 -0
- package/dist/http_compressors/gzip.d.ts +33 -0
- package/dist/http_compressors/gzip.js +39 -0
- package/dist/http_compressors/index.d.ts +6 -0
- package/dist/http_compressors/index.js +2 -0
- package/dist/http_compressors/resolve.d.ts +23 -0
- package/dist/http_compressors/resolve.js +27 -0
- package/dist/index.d.ts +6 -1
- package/dist/index.js +5 -1
- package/dist/invalid_response_body_error.d.ts +14 -0
- package/dist/invalid_response_body_error.js +18 -0
- package/dist/models.d.ts +166 -243
- package/dist/models.js +6 -4
- package/dist/resource_clients/actor.d.ts +104 -34
- package/dist/resource_clients/actor.js +130 -56
- package/dist/resource_clients/actor_collection.d.ts +18 -12
- package/dist/resource_clients/actor_collection.js +11 -3
- package/dist/resource_clients/actor_env_var.d.ts +11 -4
- package/dist/resource_clients/actor_env_var.js +16 -6
- package/dist/resource_clients/actor_env_var_collection.d.ts +9 -4
- package/dist/resource_clients/actor_env_var_collection.js +11 -4
- package/dist/resource_clients/actor_server.d.ts +141 -0
- package/dist/resource_clients/actor_server.js +195 -0
- package/dist/resource_clients/actor_version.d.ts +20 -5
- package/dist/resource_clients/actor_version.js +19 -9
- package/dist/resource_clients/actor_version_collection.d.ts +11 -4
- package/dist/resource_clients/actor_version_collection.js +13 -4
- package/dist/resource_clients/build.d.ts +20 -7
- package/dist/resource_clients/build.js +35 -15
- package/dist/resource_clients/build_collection.d.ts +7 -5
- package/dist/resource_clients/build_collection.js +4 -1
- package/dist/resource_clients/dataset.d.ts +83 -37
- package/dist/resource_clients/dataset.js +107 -121
- package/dist/resource_clients/dataset_collection.d.ts +10 -6
- package/dist/resource_clients/dataset_collection.js +13 -2
- package/dist/resource_clients/key_value_store.d.ts +108 -26
- package/dist/resource_clients/key_value_store.js +134 -52
- package/dist/resource_clients/key_value_store_collection.d.ts +10 -6
- package/dist/resource_clients/key_value_store_collection.js +13 -2
- package/dist/resource_clients/log.d.ts +40 -21
- package/dist/resource_clients/log.js +144 -66
- package/dist/resource_clients/request_queue.d.ts +76 -31
- package/dist/resource_clients/request_queue.js +178 -93
- package/dist/resource_clients/request_queue_collection.d.ts +9 -5
- package/dist/resource_clients/request_queue_collection.js +9 -3
- package/dist/resource_clients/run.d.ts +126 -28
- package/dist/resource_clients/run.js +193 -49
- package/dist/resource_clients/run_collection.d.ts +5 -3
- package/dist/resource_clients/run_collection.js +4 -1
- package/dist/resource_clients/schedule.d.ts +16 -7
- package/dist/resource_clients/schedule.js +30 -22
- package/dist/resource_clients/schedule_collection.d.ts +6 -2
- package/dist/resource_clients/schedule_collection.js +9 -3
- package/dist/resource_clients/store_collection.d.ts +6 -4
- package/dist/resource_clients/store_collection.js +4 -1
- package/dist/resource_clients/task.d.ts +36 -13
- package/dist/resource_clients/task.js +79 -46
- package/dist/resource_clients/task_collection.d.ts +9 -5
- package/dist/resource_clients/task_collection.js +9 -3
- package/dist/resource_clients/user.d.ts +19 -10
- package/dist/resource_clients/user.js +42 -38
- package/dist/resource_clients/webhook.d.ts +17 -8
- package/dist/resource_clients/webhook.js +30 -22
- package/dist/resource_clients/webhook_collection.d.ts +8 -4
- package/dist/resource_clients/webhook_collection.js +9 -3
- package/dist/resource_clients/webhook_dispatch.d.ts +4 -1
- package/dist/resource_clients/webhook_dispatch.js +7 -2
- package/dist/resource_clients/webhook_dispatch_collection.d.ts +3 -1
- package/dist/resource_clients/webhook_dispatch_collection.js +4 -1
- package/dist/response_validation_error.d.ts +2 -1
- package/dist/response_validation_error.js +2 -1
- package/dist/runtime/node.d.ts +6 -0
- package/dist/runtime/node.js +80 -0
- package/dist/runtime/types.d.ts +60 -0
- package/dist/runtime/types.js +1 -0
- package/dist/runtime/web.d.ts +5 -0
- package/dist/runtime/web.js +21 -0
- package/dist/schemas.d.ts +368 -4
- package/dist/schemas.js +26 -4
- package/dist/statistics.d.ts +1 -6
- package/dist/statistics.js +2 -2
- package/dist/timeouts.d.ts +84 -0
- package/dist/timeouts.js +37 -0
- package/dist/timezones.d.ts +1 -1
- package/dist/utils.d.ts +89 -59
- package/dist/utils.js +216 -158
- package/dist/wait_for_resources.d.ts +8 -0
- package/dist/wait_for_resources.js +39 -0
- package/package.json +37 -34
- package/dist/http_client.d.ts +0 -88
- package/dist/http_client.js +0 -255
- package/dist/interceptors.d.ts +0 -19
- package/dist/interceptors.js +0 -105
|
@@ -1,24 +1,30 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import log from '@apify/log';
|
|
3
3
|
import { createHmacSignatureAsync, createStorageContentSignatureAsync } from '@apify/utilities';
|
|
4
|
-
import {
|
|
4
|
+
import { ResourceClient } from '../base/resource_client.js';
|
|
5
5
|
import * as schemas from '../schemas.js';
|
|
6
|
-
import {
|
|
6
|
+
import { timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
|
|
7
|
+
import { runtime } from '#runtime';
|
|
8
|
+
import { anyObjectSchema, applyQueryParamsToUrl, catchNotFoundOrThrow, isBlob, isBuffer, isStream, minForLimitParam, parseArgument, parseResponse, } from '../utils.js';
|
|
7
9
|
const listKeysOptionsSchema = z.strictObject({
|
|
8
10
|
limit: z.number().min(0).optional(),
|
|
11
|
+
chunkSize: z.number().positive().optional(),
|
|
9
12
|
exclusiveStartKey: z.string().optional(),
|
|
10
13
|
collection: z.string().optional(),
|
|
11
14
|
prefix: z.string().optional(),
|
|
12
15
|
signature: z.string().optional(),
|
|
16
|
+
...timeoutOptionsShape,
|
|
13
17
|
});
|
|
14
18
|
const nonEmptyKeySchema = z.string().min(1);
|
|
15
|
-
// `
|
|
19
|
+
// `chunkSize` (client-side only) and `signature` (which this method produces) are left out. The options type omits
|
|
20
|
+
// both to match.
|
|
16
21
|
const createKeysPublicUrlOptionsSchema = z.strictObject({
|
|
17
22
|
limit: z.number().min(0).optional(),
|
|
18
23
|
exclusiveStartKey: z.string().optional(),
|
|
19
24
|
collection: z.string().optional(),
|
|
20
25
|
prefix: z.string().optional(),
|
|
21
|
-
expiresInSecs: z.number().optional(),
|
|
26
|
+
expiresInSecs: z.number().positive().optional(),
|
|
27
|
+
...timeoutOptionsShape,
|
|
22
28
|
});
|
|
23
29
|
const keySchema = z.string();
|
|
24
30
|
const getRecordOptionsSchema = z.strictObject({
|
|
@@ -26,6 +32,7 @@ const getRecordOptionsSchema = z.strictObject({
|
|
|
26
32
|
stream: z.boolean().optional(),
|
|
27
33
|
disableRedirect: z.boolean().optional(),
|
|
28
34
|
signature: z.string().optional(),
|
|
35
|
+
...timeoutOptionsShape,
|
|
29
36
|
});
|
|
30
37
|
const recordSchema = z.strictObject({
|
|
31
38
|
key: z.string(),
|
|
@@ -37,11 +44,12 @@ const recordSchema = z.strictObject({
|
|
|
37
44
|
typeof value !== 'symbol' &&
|
|
38
45
|
typeof value !== 'bigint' &&
|
|
39
46
|
typeof value !== 'function' &&
|
|
40
|
-
(typeof value !== 'number' || Number.isFinite(value)), 'Expected a
|
|
47
|
+
(typeof value !== 'number' || Number.isFinite(value)), 'Expected a JSON-serializable value, binary data, or a stream'),
|
|
41
48
|
contentType: z.string().min(1).optional(),
|
|
49
|
+
contentEncoding: z.string().trim().min(1).optional(),
|
|
42
50
|
});
|
|
43
51
|
const recordOptionsSchema = z.strictObject({
|
|
44
|
-
|
|
52
|
+
...timeoutOptionsShape,
|
|
45
53
|
doNotRetryTimeouts: z.boolean().optional(),
|
|
46
54
|
});
|
|
47
55
|
/**
|
|
@@ -85,11 +93,14 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
85
93
|
/**
|
|
86
94
|
* Gets the key-value store object from the Apify API.
|
|
87
95
|
*
|
|
96
|
+
* @param options - Request options
|
|
97
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
88
98
|
* @returns The KeyValueStore object, or `undefined` if it does not exist
|
|
89
99
|
* @see https://docs.apify.com/api/v2/key-value-store-get
|
|
90
100
|
*/
|
|
91
|
-
async get() {
|
|
92
|
-
|
|
101
|
+
async get(options = {}) {
|
|
102
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
103
|
+
return this.getResource(schemas.KeyValueStoreResource(), {}, timeoutSecs, signal);
|
|
93
104
|
}
|
|
94
105
|
/**
|
|
95
106
|
* Updates the key-value store with specified fields.
|
|
@@ -98,20 +109,26 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
98
109
|
* @param newFields.name - New name for the store
|
|
99
110
|
* @param newFields.title - New title for the store
|
|
100
111
|
* @param newFields.generalAccess - General resource access level ('FOLLOW_USER_SETTING', 'ANYONE_WITH_ID_CAN_READ' or 'RESTRICTED')
|
|
112
|
+
* @param options - Request options
|
|
113
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
101
114
|
* @returns The updated KeyValueStore object
|
|
102
115
|
* @see https://docs.apify.com/api/v2/key-value-store-put
|
|
103
116
|
*/
|
|
104
|
-
async update(newFields) {
|
|
117
|
+
async update(newFields, options = {}) {
|
|
105
118
|
parseArgument(newFields, anyObjectSchema);
|
|
106
|
-
|
|
119
|
+
const { timeoutSecs = 'long', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
120
|
+
return this.updateResource(schemas.KeyValueStoreResource(), newFields, timeoutSecs, signal);
|
|
107
121
|
}
|
|
108
122
|
/**
|
|
109
123
|
* Deletes the key-value store.
|
|
110
124
|
*
|
|
125
|
+
* @param options - Request options
|
|
126
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
111
127
|
* @see https://docs.apify.com/api/v2/key-value-store-delete
|
|
112
128
|
*/
|
|
113
|
-
async delete() {
|
|
114
|
-
|
|
129
|
+
async delete(options = {}) {
|
|
130
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
131
|
+
return this.deleteResource(timeoutSecs, signal);
|
|
115
132
|
}
|
|
116
133
|
/**
|
|
117
134
|
* Lists all keys in the key-value store.
|
|
@@ -121,9 +138,12 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
121
138
|
*
|
|
122
139
|
* @param options - Listing options
|
|
123
140
|
* @param options.limit - Maximum number of keys to return. Default is 1000.
|
|
141
|
+
* @param options.chunkSize - Maximum number of keys requested per API call when iterating across pages. `limit`
|
|
142
|
+
* then caps the total across all pages.
|
|
124
143
|
* @param options.exclusiveStartKey - Key to start listing from (for pagination). The listing starts with the next key after this one.
|
|
125
144
|
* @param options.collection - Filter keys by collection name.
|
|
126
145
|
* @param options.prefix - Filter keys that start with this prefix.
|
|
146
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
|
|
127
147
|
* @returns Object containing `items` array of key metadata, pagination info (`count`, `limit`, `isTruncated`, `nextExclusiveStartKey`)
|
|
128
148
|
* @see https://docs.apify.com/api/v2/key-value-store-keys-get
|
|
129
149
|
*
|
|
@@ -149,17 +169,19 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
149
169
|
* ```
|
|
150
170
|
*/
|
|
151
171
|
listKeys(options = {}) {
|
|
152
|
-
|
|
172
|
+
// `chunkSize` only sizes the page requests; it is not an API parameter, so it must not reach the query string.
|
|
173
|
+
const { chunkSize, timeoutSecs = 'medium', signal, ...parsed } = parseArgument(options, listKeysOptionsSchema, 'KeyValueClientListKeysOptions');
|
|
153
174
|
const getPaginatedList = async (kvsListOptions = {}) => {
|
|
154
175
|
const response = await this.httpClient.call({
|
|
155
|
-
url: this.
|
|
176
|
+
url: this.buildUrl('keys'),
|
|
156
177
|
method: 'GET',
|
|
157
|
-
params: this.
|
|
158
|
-
|
|
178
|
+
params: this.buildParams(kvsListOptions),
|
|
179
|
+
timeoutSecs,
|
|
180
|
+
signal,
|
|
159
181
|
});
|
|
160
182
|
return parseResponse(response, schemas.ListOfKeys());
|
|
161
183
|
};
|
|
162
|
-
const paginatedListPromise = getPaginatedList(parsed);
|
|
184
|
+
const paginatedListPromise = getPaginatedList({ ...parsed, limit: minForLimitParam(parsed.limit, chunkSize) });
|
|
163
185
|
async function* asyncGenerator() {
|
|
164
186
|
let currentPage = await paginatedListPromise;
|
|
165
187
|
yield* currentPage.items;
|
|
@@ -172,7 +194,7 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
172
194
|
) {
|
|
173
195
|
const newOptions = {
|
|
174
196
|
...parsed,
|
|
175
|
-
limit: remainingItems,
|
|
197
|
+
limit: minForLimitParam(remainingItems, chunkSize),
|
|
176
198
|
exclusiveStartKey: currentPage.nextExclusiveStartKey,
|
|
177
199
|
};
|
|
178
200
|
currentPage = await getPaginatedList(newOptions);
|
|
@@ -194,6 +216,8 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
194
216
|
* requiring an API token.
|
|
195
217
|
*
|
|
196
218
|
* @param key - The record key
|
|
219
|
+
* @param options - Request options
|
|
220
|
+
* @param options.timeoutSecs - Timeout for the API request that fetches the store. Default is `'long'`.
|
|
197
221
|
* @returns A public URL string for accessing the record
|
|
198
222
|
*
|
|
199
223
|
* @example
|
|
@@ -204,10 +228,11 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
204
228
|
* ```
|
|
205
229
|
* @since Added in 2.14.0
|
|
206
230
|
*/
|
|
207
|
-
async getRecordPublicUrl(key) {
|
|
231
|
+
async getRecordPublicUrl(key, options = {}) {
|
|
208
232
|
parseArgument(key, nonEmptyKeySchema);
|
|
209
|
-
const
|
|
210
|
-
const
|
|
233
|
+
const { timeoutSecs = 'long', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
234
|
+
const store = await this.get({ timeoutSecs, signal });
|
|
235
|
+
const recordPublicUrl = new URL(this.buildPublicUrl(['records', key]));
|
|
211
236
|
if (store?.urlSigningSecretKey) {
|
|
212
237
|
const signature = await createHmacSignatureAsync(store.urlSigningSecretKey, key);
|
|
213
238
|
recordPublicUrl.searchParams.append('signature', signature);
|
|
@@ -221,9 +246,11 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
221
246
|
* the URL will include a cryptographic signature which allows access without authentication.
|
|
222
247
|
*
|
|
223
248
|
* @param options - URL generation options (extends all options from {@link listKeys})
|
|
224
|
-
* @param options.expiresInSecs - Number of seconds until the signed URL expires. If omitted, the URL
|
|
249
|
+
* @param options.expiresInSecs - Number of seconds until the signed URL expires. Must be positive. If omitted, the URL
|
|
250
|
+
* never expires.
|
|
225
251
|
* @param options.limit - Maximum number of keys to return.
|
|
226
252
|
* @param options.prefix - Filter keys by prefix.
|
|
253
|
+
* @param options.timeoutSecs - Timeout for the API request that fetches the store. Default is `'long'`.
|
|
227
254
|
* @returns A public URL string for accessing the keys list
|
|
228
255
|
*
|
|
229
256
|
* @example
|
|
@@ -238,10 +265,9 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
238
265
|
* @since Added in 2.13.0
|
|
239
266
|
*/
|
|
240
267
|
async createKeysPublicUrl(options = {}) {
|
|
241
|
-
const
|
|
242
|
-
const store = await this.get();
|
|
243
|
-
|
|
244
|
-
let createdPublicKeysUrl = new URL(this._publicUrl('keys'));
|
|
268
|
+
const { timeoutSecs = 'long', signal, expiresInSecs, ...queryOptions } = parseArgument(options, createKeysPublicUrlOptionsSchema, 'KeyValueClientCreateKeysUrlOptions');
|
|
269
|
+
const store = await this.get({ timeoutSecs, signal });
|
|
270
|
+
let createdPublicKeysUrl = new URL(this.buildPublicUrl('keys'));
|
|
245
271
|
if (store?.urlSigningSecretKey) {
|
|
246
272
|
const signature = await createStorageContentSignatureAsync({
|
|
247
273
|
resourceId: store.id,
|
|
@@ -259,6 +285,8 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
259
285
|
* This is more efficient than {@link getRecord} when you only need to check for existence.
|
|
260
286
|
*
|
|
261
287
|
* @param key - The record key to check
|
|
288
|
+
* @param options - Request options
|
|
289
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
262
290
|
* @returns `true` if the record exists, `false` if it does not
|
|
263
291
|
* @see https://docs.apify.com/api/v2/key-value-store-record-get
|
|
264
292
|
*
|
|
@@ -271,11 +299,14 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
271
299
|
* ```
|
|
272
300
|
* @since Added in 2.9.0
|
|
273
301
|
*/
|
|
274
|
-
async recordExists(key) {
|
|
302
|
+
async recordExists(key, options = {}) {
|
|
303
|
+
const { timeoutSecs = 'long', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
275
304
|
const requestOpts = {
|
|
276
|
-
url: this.
|
|
305
|
+
url: this.buildUrl(['records', key]),
|
|
277
306
|
method: 'HEAD',
|
|
278
|
-
params: this.
|
|
307
|
+
params: this.buildParams(),
|
|
308
|
+
timeoutSecs,
|
|
309
|
+
signal,
|
|
279
310
|
};
|
|
280
311
|
try {
|
|
281
312
|
await this.httpClient.call(requestOpts);
|
|
@@ -289,7 +320,7 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
289
320
|
async getRecord(key, options = {}) {
|
|
290
321
|
parseArgument(key, keySchema);
|
|
291
322
|
const parsed = parseArgument(options, getRecordOptionsSchema, 'KeyValueClientGetRecordOptions');
|
|
292
|
-
if (parsed.stream && !isNode
|
|
323
|
+
if (parsed.stream && !runtime.isNode) {
|
|
293
324
|
throw new Error('The stream option can only be used in Node.js environment.');
|
|
294
325
|
}
|
|
295
326
|
if ('disableRedirect' in parsed) {
|
|
@@ -300,13 +331,14 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
300
331
|
if (parsed.signature)
|
|
301
332
|
queryParams.signature = parsed.signature;
|
|
302
333
|
const requestOpts = {
|
|
303
|
-
url: this.
|
|
334
|
+
url: this.buildUrl(['records', key]),
|
|
304
335
|
method: 'GET',
|
|
305
|
-
params: this.
|
|
306
|
-
|
|
336
|
+
params: this.buildParams(queryParams),
|
|
337
|
+
timeoutSecs: parsed.timeoutSecs ?? 'long',
|
|
338
|
+
signal: parsed.signal,
|
|
307
339
|
};
|
|
308
340
|
if (parsed.buffer)
|
|
309
|
-
requestOpts.
|
|
341
|
+
requestOpts.responseType = 'buffer';
|
|
310
342
|
if (parsed.stream)
|
|
311
343
|
requestOpts.responseType = 'stream';
|
|
312
344
|
try {
|
|
@@ -329,19 +361,39 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
329
361
|
* The content type is automatically determined based on the value type, but can be
|
|
330
362
|
* overridden using the `contentType` property.
|
|
331
363
|
*
|
|
364
|
+
* A string is sent as it is, even under `application/json`, so a string stored under a JSON content type has
|
|
365
|
+
* to be valid JSON already, or {@link getRecord} fails to parse it back.
|
|
366
|
+
*
|
|
332
367
|
* **Note about streams:** If the value is a stream object (has `.pipe` and `.on` methods),
|
|
333
|
-
* the upload cannot be retried on failure or follow redirects. For reliable uploads,
|
|
334
|
-
*
|
|
368
|
+
* the upload cannot be retried on failure or follow redirects. For reliable uploads of a file,
|
|
369
|
+
* pass a `Blob` from `fs.openAsBlob()` instead: it is read again for every attempt and never held
|
|
370
|
+
* in memory whole.
|
|
335
371
|
*
|
|
336
372
|
* @param record - The record to store
|
|
337
373
|
* @param record.key - Record key (unique identifier)
|
|
338
|
-
* @param record.value - Record value (
|
|
374
|
+
* @param record.value - Record value (a JSON-serializable value, Buffer, ArrayBuffer, typed array, Blob, or
|
|
375
|
+
* Readable)
|
|
339
376
|
* @param record.contentType - Optional MIME type. Auto-detected if not provided:
|
|
340
377
|
* - Objects: `'application/json; charset=utf-8'`
|
|
341
378
|
* - Strings: `'text/plain; charset=utf-8'`
|
|
342
|
-
* -
|
|
379
|
+
* - Blobs: their `type`, or `'application/octet-stream'` when it is empty
|
|
380
|
+
* - Binary values and streams: `'application/octet-stream'`
|
|
381
|
+
*
|
|
382
|
+
* Worth setting for media and archives: the client skips compressing a body
|
|
383
|
+
* whose content type already carries its own compression. The
|
|
384
|
+
* `'application/octet-stream'` fallback is treated as compressible, because
|
|
385
|
+
* it could hold any binary data.
|
|
386
|
+
* @param record.contentEncoding - Optional encoding already applied to `value`, sent as the `Content-Encoding`
|
|
387
|
+
* header. Set it to upload a pre-compressed value: the client then sends the
|
|
388
|
+
* bytes as they are and does not compress them again. The API accepts `gzip`,
|
|
389
|
+
* `br`, `deflate`, and `identity`, and stores the record exactly as uploaded, so
|
|
390
|
+
* this is also the encoding the record is served with. Only binary values and
|
|
391
|
+
* streams can carry a compression, so any encoding other than `identity` throws
|
|
392
|
+
* a `TypeError` when `value` is a string, or a plain object, array, or other
|
|
393
|
+
* value that the client serializes to JSON. Node.js only, since browsers cannot
|
|
394
|
+
* send the header to the API.
|
|
343
395
|
* @param options - Storage options
|
|
344
|
-
* @param options.timeoutSecs - Timeout for the
|
|
396
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
345
397
|
* @param options.doNotRetryTimeouts - If `true`, don't retry on timeout errors. Default is `false`.
|
|
346
398
|
* @see https://docs.apify.com/api/v2/key-value-store-record-put
|
|
347
399
|
*
|
|
@@ -367,18 +419,40 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
367
419
|
* value: imageBuffer,
|
|
368
420
|
* contentType: 'image/png'
|
|
369
421
|
* });
|
|
422
|
+
*
|
|
423
|
+
* // Store a gzipped file without compressing it again
|
|
424
|
+
* const gzippedReport = await readFile('report.json.gz');
|
|
425
|
+
* await client.keyValueStore('my-store').setRecord({
|
|
426
|
+
* key: 'report.json',
|
|
427
|
+
* value: gzippedReport,
|
|
428
|
+
* contentType: 'application/json',
|
|
429
|
+
* contentEncoding: 'gzip'
|
|
430
|
+
* });
|
|
370
431
|
* ```
|
|
371
432
|
*/
|
|
372
433
|
async setRecord(record, options = {}) {
|
|
373
434
|
parseArgument(record, recordSchema);
|
|
374
435
|
const parsed = parseArgument(options, recordOptionsSchema, 'KeyValueStoreRecordOptions');
|
|
375
|
-
const { key } = record;
|
|
436
|
+
const { key, contentEncoding } = record;
|
|
376
437
|
let { value, contentType } = record;
|
|
377
|
-
const { timeoutSecs, doNotRetryTimeouts } = parsed;
|
|
378
|
-
|
|
438
|
+
const { timeoutSecs = 'long', signal, doNotRetryTimeouts } = parsed;
|
|
439
|
+
if (contentEncoding !== undefined && !runtime.isNode) {
|
|
440
|
+
throw new Error('The contentEncoding option can only be used in Node.js environment.');
|
|
441
|
+
}
|
|
442
|
+
const isBinaryValue = isStream(value) || isBuffer(value) || isBlob(value);
|
|
443
|
+
// The client forwards the header without inspecting the body, so a string or a value serialized to JSON, which
|
|
444
|
+
// cannot hold compressed bytes, would be stored under an encoding that misdescribes it.
|
|
445
|
+
const declaresCompression = contentEncoding !== undefined && contentEncoding.trim().toLowerCase() !== 'identity';
|
|
446
|
+
if (declaresCompression && !isBinaryValue) {
|
|
447
|
+
throw new TypeError(`Cannot upload a value of type ${typeof value} with Content-Encoding: ${contentEncoding}. ` +
|
|
448
|
+
'An encoding other than identity declares the value is already compressed, so pass the ' +
|
|
449
|
+
'compressed bytes as a Buffer, an ArrayBuffer, a typed array, a Blob, or a readable stream.');
|
|
450
|
+
}
|
|
379
451
|
// To allow saving Objects to JSON without providing content type
|
|
380
452
|
if (!contentType) {
|
|
381
|
-
if (
|
|
453
|
+
if (isBlob(value))
|
|
454
|
+
contentType = value.type || 'application/octet-stream';
|
|
455
|
+
else if (isBinaryValue)
|
|
382
456
|
contentType = 'application/octet-stream';
|
|
383
457
|
else if (typeof value === 'string')
|
|
384
458
|
contentType = 'text/plain; charset=utf-8';
|
|
@@ -386,7 +460,7 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
386
460
|
contentType = 'application/json; charset=utf-8';
|
|
387
461
|
}
|
|
388
462
|
const isContentTypeJson = contentType.startsWith('application/json');
|
|
389
|
-
if (isContentTypeJson && !
|
|
463
|
+
if (isContentTypeJson && !isBinaryValue && typeof value !== 'string') {
|
|
390
464
|
try {
|
|
391
465
|
value = JSON.stringify(value, null, 2);
|
|
392
466
|
}
|
|
@@ -396,13 +470,17 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
396
470
|
}
|
|
397
471
|
}
|
|
398
472
|
const uploadOpts = {
|
|
399
|
-
url: this.
|
|
473
|
+
url: this.buildUrl(['records', key]),
|
|
400
474
|
method: 'PUT',
|
|
401
|
-
params: this.
|
|
475
|
+
params: this.buildParams(),
|
|
402
476
|
data: value,
|
|
403
|
-
headers:
|
|
477
|
+
headers: {
|
|
478
|
+
'content-type': contentType,
|
|
479
|
+
...(contentEncoding !== undefined && { 'content-encoding': contentEncoding }),
|
|
480
|
+
},
|
|
404
481
|
doNotRetryTimeouts,
|
|
405
|
-
|
|
482
|
+
timeoutSecs,
|
|
483
|
+
signal,
|
|
406
484
|
};
|
|
407
485
|
await this.httpClient.call(uploadOpts);
|
|
408
486
|
}
|
|
@@ -410,6 +488,8 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
410
488
|
* Deletes a record from the key-value store.
|
|
411
489
|
*
|
|
412
490
|
* @param key - The record key to delete
|
|
491
|
+
* @param options - Request options
|
|
492
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
413
493
|
* @see https://docs.apify.com/api/v2/key-value-store-record-delete
|
|
414
494
|
*
|
|
415
495
|
* @example
|
|
@@ -417,13 +497,15 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
417
497
|
* await client.keyValueStore('my-store').deleteRecord('temp-data');
|
|
418
498
|
* ```
|
|
419
499
|
*/
|
|
420
|
-
async deleteRecord(key) {
|
|
500
|
+
async deleteRecord(key, options = {}) {
|
|
421
501
|
parseArgument(key, keySchema);
|
|
502
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
422
503
|
await this.httpClient.call({
|
|
423
|
-
url: this.
|
|
504
|
+
url: this.buildUrl(['records', key]),
|
|
424
505
|
method: 'DELETE',
|
|
425
|
-
params: this.
|
|
426
|
-
|
|
506
|
+
params: this.buildParams(),
|
|
507
|
+
timeoutSecs,
|
|
508
|
+
signal,
|
|
427
509
|
});
|
|
428
510
|
}
|
|
429
511
|
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { STORAGE_OWNERSHIP_FILTER } from '@apify/consts';
|
|
2
2
|
import type { ApiClientSubResourceOptions } from '../base/api_client.js';
|
|
3
3
|
import { ResourceCollectionClient } from '../base/resource_collection_client.js';
|
|
4
|
+
import type { TimeoutOptions } from '../timeouts.js';
|
|
4
5
|
import type { PaginatedList, PaginationOptions } from '../utils.js';
|
|
5
|
-
import type {
|
|
6
|
+
import type { KeyValueStoreResource } from './key_value_store.js';
|
|
6
7
|
/**
|
|
7
8
|
* Client for managing the collection of Key-value stores in your account.
|
|
8
9
|
*
|
|
@@ -45,21 +46,24 @@ export declare class KeyValueStoreCollectionClient extends ResourceCollectionCli
|
|
|
45
46
|
* ```
|
|
46
47
|
*
|
|
47
48
|
* @param options - Pagination options.
|
|
49
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
|
|
48
50
|
* @returns A paginated iterator of Key-value stores.
|
|
49
51
|
* @see https://docs.apify.com/api/v2/key-value-stores-get
|
|
50
52
|
*/
|
|
51
|
-
list(options?: KeyValueStoreCollectionClientListOptions): Promise<KeyValueStoreCollectionListResult> & AsyncIterable<
|
|
53
|
+
list(options?: KeyValueStoreCollectionClientListOptions): Promise<KeyValueStoreCollectionListResult> & AsyncIterable<KeyValueStoreResource>;
|
|
52
54
|
/**
|
|
53
55
|
* Gets or creates a key-value store with the specified name.
|
|
54
56
|
*
|
|
55
57
|
* @param name - Name of the key-value store. If not provided, a default store is used.
|
|
56
58
|
* @param options - Additional options like schema.
|
|
59
|
+
* @param options.schema - Schema of the key-value store.
|
|
60
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
57
61
|
* @returns The key-value store object.
|
|
58
62
|
* @see https://docs.apify.com/api/v2/key-value-stores-post
|
|
59
63
|
*/
|
|
60
|
-
getOrCreate(name?: string, options?: KeyValueStoreCollectionClientGetOrCreateOptions): Promise<
|
|
64
|
+
getOrCreate(name?: string, options?: KeyValueStoreCollectionClientGetOrCreateOptions): Promise<KeyValueStoreResource>;
|
|
61
65
|
}
|
|
62
|
-
export interface KeyValueStoreCollectionClientListOptions extends PaginationOptions {
|
|
66
|
+
export interface KeyValueStoreCollectionClientListOptions extends PaginationOptions, TimeoutOptions {
|
|
63
67
|
unnamed?: boolean;
|
|
64
68
|
desc?: boolean;
|
|
65
69
|
/**
|
|
@@ -71,9 +75,9 @@ export interface KeyValueStoreCollectionClientListOptions extends PaginationOpti
|
|
|
71
75
|
/**
|
|
72
76
|
* @since Added in 2.3.0
|
|
73
77
|
*/
|
|
74
|
-
export interface KeyValueStoreCollectionClientGetOrCreateOptions {
|
|
78
|
+
export interface KeyValueStoreCollectionClientGetOrCreateOptions extends TimeoutOptions {
|
|
75
79
|
schema?: Record<string, unknown>;
|
|
76
80
|
}
|
|
77
|
-
export type KeyValueStoreCollectionListResult = PaginatedList<
|
|
81
|
+
export type KeyValueStoreCollectionListResult = PaginatedList<KeyValueStoreResource> & {
|
|
78
82
|
unnamed: boolean;
|
|
79
83
|
};
|
|
@@ -2,12 +2,14 @@ import { z } from 'zod';
|
|
|
2
2
|
import { STORAGE_OWNERSHIP_FILTER } from '@apify/consts';
|
|
3
3
|
import { ResourceCollectionClient } from '../base/resource_collection_client.js';
|
|
4
4
|
import * as schemas from '../schemas.js';
|
|
5
|
+
import { optionalSignalSchema, optionalTimeoutSchema, timeoutOptionsShape } from '../timeouts.js';
|
|
5
6
|
import { anyObjectSchema, paginationOptionsShape, parseArgument } from '../utils.js';
|
|
6
7
|
const listOptionsSchema = z.strictObject({
|
|
7
8
|
unnamed: z.boolean().optional(),
|
|
8
9
|
...paginationOptionsShape,
|
|
9
10
|
desc: z.boolean().optional(),
|
|
10
11
|
ownership: z.enum(STORAGE_OWNERSHIP_FILTER).optional(),
|
|
12
|
+
...timeoutOptionsShape,
|
|
11
13
|
});
|
|
12
14
|
const nameSchema = z.string().optional();
|
|
13
15
|
const schemaSchema = anyObjectSchema.optional();
|
|
@@ -58,24 +60,33 @@ export class KeyValueStoreCollectionClient extends ResourceCollectionClient {
|
|
|
58
60
|
* ```
|
|
59
61
|
*
|
|
60
62
|
* @param options - Pagination options.
|
|
63
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
|
|
61
64
|
* @returns A paginated iterator of Key-value stores.
|
|
62
65
|
* @see https://docs.apify.com/api/v2/key-value-stores-get
|
|
63
66
|
*/
|
|
64
67
|
list(options = {}) {
|
|
65
68
|
const parsed = parseArgument(options, listOptionsSchema, 'KeyValueStoreCollectionClientListOptions');
|
|
66
|
-
return this.
|
|
69
|
+
return this.listResourcesPaginated(schemas.ListOfKeyValueStores(), parsed, 'medium');
|
|
67
70
|
}
|
|
68
71
|
/**
|
|
69
72
|
* Gets or creates a key-value store with the specified name.
|
|
70
73
|
*
|
|
71
74
|
* @param name - Name of the key-value store. If not provided, a default store is used.
|
|
72
75
|
* @param options - Additional options like schema.
|
|
76
|
+
* @param options.schema - Schema of the key-value store.
|
|
77
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
73
78
|
* @returns The key-value store object.
|
|
74
79
|
* @see https://docs.apify.com/api/v2/key-value-stores-post
|
|
75
80
|
*/
|
|
76
81
|
async getOrCreate(name, options) {
|
|
77
82
|
parseArgument(name, nameSchema);
|
|
78
83
|
parseArgument(options?.schema, schemaSchema); // TODO: Add schema validation
|
|
79
|
-
|
|
84
|
+
parseArgument(options?.timeoutSecs, optionalTimeoutSchema);
|
|
85
|
+
parseArgument(options?.signal, optionalSignalSchema);
|
|
86
|
+
// `timeoutSecs` and `signal` are not part of the resource, so the body carries only the rest, and stays
|
|
87
|
+
// absent when there is nothing else to send.
|
|
88
|
+
const { timeoutSecs = 'short', signal, ...resource } = options ?? {};
|
|
89
|
+
const hasResource = Object.keys(resource).length > 0;
|
|
90
|
+
return this.getOrCreateResource(schemas.KeyValueStoreResource(), name, hasResource ? resource : undefined, timeoutSecs, signal);
|
|
80
91
|
}
|
|
81
92
|
}
|
|
@@ -3,6 +3,7 @@ import type { Log } from '@apify/log';
|
|
|
3
3
|
import { Logger, LogLevel } from '@apify/log';
|
|
4
4
|
import type { ApiClientSubResourceOptions } from '../base/api_client.js';
|
|
5
5
|
import { ResourceClient } from '../base/resource_client.js';
|
|
6
|
+
import type { TimeoutOptions } from '../timeouts.js';
|
|
6
7
|
/**
|
|
7
8
|
* Client for accessing Actor run or build logs.
|
|
8
9
|
*
|
|
@@ -18,6 +19,9 @@ import { ResourceClient } from '../base/resource_client.js';
|
|
|
18
19
|
* const log = await runClient.log().get();
|
|
19
20
|
* console.log(log);
|
|
20
21
|
*
|
|
22
|
+
* // Get the log content as raw bytes
|
|
23
|
+
* const logBuffer = await runClient.log().get({ buffer: true });
|
|
24
|
+
*
|
|
21
25
|
* // Stream the log in real-time
|
|
22
26
|
* const stream = await runClient.log().stream();
|
|
23
27
|
* stream.on('line', (line) => console.log(line));
|
|
@@ -33,18 +37,34 @@ export declare class LogClient extends ResourceClient {
|
|
|
33
37
|
/**
|
|
34
38
|
* Retrieves the log as a string.
|
|
35
39
|
*
|
|
40
|
+
* The `buffer` option only selects the return type: the bytes come back as a Buffer (Node.js) or ArrayBuffer
|
|
41
|
+
* (browser), not decoded to a string. Either way the whole log is downloaded in one response; to read it
|
|
42
|
+
* incrementally, use {@link LogClient.stream}.
|
|
43
|
+
*
|
|
36
44
|
* @param options - Log retrieval options.
|
|
37
45
|
* @param options.raw - If `true`, returns raw log content without any processing. Default is `false`.
|
|
38
|
-
* @
|
|
46
|
+
* @param options.buffer - If `true`, the log is returned as a Buffer (Node.js) or ArrayBuffer (browser) instead of a
|
|
47
|
+
* string. It does not change how the log is fetched.
|
|
48
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
49
|
+
* @returns The log content, or `undefined` if it does not exist. A chained client such as `run.log()` throws an
|
|
50
|
+
* `ApifyApiError` on a 404, since the run itself may be what is missing.
|
|
39
51
|
* @see https://docs.apify.com/api/v2/log-get
|
|
40
52
|
*/
|
|
41
|
-
get(options?:
|
|
53
|
+
get(options?: LogGetOptions & {
|
|
54
|
+
buffer?: false;
|
|
55
|
+
}): Promise<string | undefined>;
|
|
56
|
+
get(options: LogGetOptions & {
|
|
57
|
+
buffer: true;
|
|
58
|
+
}): Promise<Buffer | undefined>;
|
|
59
|
+
get(options?: LogGetOptions): Promise<string | Buffer | undefined>;
|
|
42
60
|
/**
|
|
43
61
|
* Retrieves the log as a Readable stream. Only works in Node.js.
|
|
44
62
|
*
|
|
45
63
|
* @param options - Log retrieval options.
|
|
46
64
|
* @param options.raw - If `true`, returns raw log content without any processing. Default is `false`.
|
|
47
|
-
* @
|
|
65
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
66
|
+
* @returns The log content as a Readable stream, or `undefined` if it does not exist. A chained client such as
|
|
67
|
+
* `run.log()` throws an `ApifyApiError` on a 404, since the run itself may be what is missing.
|
|
48
68
|
* @see https://docs.apify.com/api/v2/log-get
|
|
49
69
|
*/
|
|
50
70
|
stream(options?: LogOptions): Promise<Readable | undefined>;
|
|
@@ -52,10 +72,21 @@ export declare class LogClient extends ResourceClient {
|
|
|
52
72
|
/**
|
|
53
73
|
* @since Added in 2.20.0
|
|
54
74
|
*/
|
|
55
|
-
export interface LogOptions {
|
|
75
|
+
export interface LogOptions extends TimeoutOptions {
|
|
56
76
|
/** @default false */
|
|
57
77
|
raw?: boolean;
|
|
58
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* Options for {@link LogClient.get}.
|
|
81
|
+
*/
|
|
82
|
+
export interface LogGetOptions extends LogOptions {
|
|
83
|
+
/**
|
|
84
|
+
* If `true`, the log is returned as a Buffer (Node.js) or ArrayBuffer (browser) instead of a string. It only selects
|
|
85
|
+
* the return type; to read the log incrementally, use {@link LogClient.stream}.
|
|
86
|
+
* @default false
|
|
87
|
+
*/
|
|
88
|
+
buffer?: boolean;
|
|
89
|
+
}
|
|
59
90
|
/**
|
|
60
91
|
* Logger for redirected actor logs.
|
|
61
92
|
* @since Added in 2.20.0
|
|
@@ -69,31 +100,17 @@ export declare class LoggerActorRedirect extends Logger {
|
|
|
69
100
|
* @since Added in 2.20.0
|
|
70
101
|
*/
|
|
71
102
|
export declare class StreamedLog {
|
|
72
|
-
private
|
|
73
|
-
private streamBuffer;
|
|
74
|
-
private splitMarker;
|
|
75
|
-
private relevancyTimeLimit;
|
|
76
|
-
private logClient;
|
|
77
|
-
private streamingTask;
|
|
78
|
-
private stopLogging;
|
|
103
|
+
#private;
|
|
79
104
|
constructor(options: StreamedLogOptions);
|
|
80
105
|
/**
|
|
81
106
|
* Start log redirection.
|
|
82
107
|
*/
|
|
83
108
|
start(): void;
|
|
84
109
|
/**
|
|
85
|
-
* Stop log redirection.
|
|
110
|
+
* Stop log redirection. Waits up to one second for lines already in flight, then aborts the log stream request. If
|
|
111
|
+
* no stream has delivered anything yet, reads the whole log in one request instead.
|
|
86
112
|
*/
|
|
87
113
|
stop(): Promise<void>;
|
|
88
|
-
/**
|
|
89
|
-
* Get log stream from response and redirect it to another log.
|
|
90
|
-
*/
|
|
91
|
-
private streamLog;
|
|
92
|
-
private logStreamChunks;
|
|
93
|
-
/**
|
|
94
|
-
* Parse the buffer and log complete messages.
|
|
95
|
-
*/
|
|
96
|
-
private logBufferContent;
|
|
97
114
|
}
|
|
98
115
|
/**
|
|
99
116
|
* @since Added in 2.20.0
|
|
@@ -105,4 +122,6 @@ export interface StreamedLogOptions {
|
|
|
105
122
|
toLog: Log;
|
|
106
123
|
/** Whether to redirect all logs from Actor run start (even logs from the past). */
|
|
107
124
|
fromStart?: boolean;
|
|
125
|
+
/** Ends the log stream once it aborts. */
|
|
126
|
+
signal?: AbortSignal;
|
|
108
127
|
}
|