apify-client 3.0.0-beta.7 → 3.0.0-beta.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -2
- package/dist/apify_api_error.d.ts +13 -27
- package/dist/apify_api_error.js +33 -23
- package/dist/apify_client.d.ts +125 -10
- package/dist/apify_client.js +133 -48
- package/dist/apify_client_error.d.ts +10 -0
- package/dist/apify_client_error.js +10 -0
- package/dist/base/api_client.d.ts +7 -7
- package/dist/base/api_client.js +36 -36
- package/dist/base/resource_client.d.ts +21 -8
- package/dist/base/resource_client.js +67 -33
- package/dist/base/resource_collection_client.d.ts +12 -4
- package/dist/base/resource_collection_client.js +34 -11
- package/dist/body_parser.d.ts +3 -4
- package/dist/body_parser.js +19 -26
- package/dist/bundle.js +27 -22
- package/dist/bundle.js.map +1 -1
- package/dist/generated/api.d.ts +785 -290
- package/dist/generated/schemas.d.ts +771 -798
- package/dist/generated/schemas.js +183 -179
- package/dist/http_clients/axios.d.ts +60 -0
- package/dist/http_clients/axios.js +117 -0
- package/dist/http_clients/base.d.ts +331 -0
- package/dist/http_clients/base.js +563 -0
- package/dist/http_clients/index.d.ts +4 -0
- package/dist/http_clients/index.js +2 -0
- package/dist/http_compressors/base.d.ts +40 -0
- package/dist/http_compressors/base.js +9 -0
- package/dist/http_compressors/brotli.d.ts +33 -0
- package/dist/http_compressors/brotli.js +39 -0
- package/dist/http_compressors/gzip.d.ts +33 -0
- package/dist/http_compressors/gzip.js +39 -0
- package/dist/http_compressors/index.d.ts +6 -0
- package/dist/http_compressors/index.js +2 -0
- package/dist/http_compressors/resolve.d.ts +23 -0
- package/dist/http_compressors/resolve.js +27 -0
- package/dist/index.d.ts +5 -1
- package/dist/index.js +4 -1
- package/dist/invalid_response_body_error.d.ts +14 -0
- package/dist/invalid_response_body_error.js +18 -0
- package/dist/models.d.ts +162 -194
- package/dist/models.js +6 -4
- package/dist/resource_clients/actor.d.ts +74 -34
- package/dist/resource_clients/actor.js +87 -56
- package/dist/resource_clients/actor_collection.d.ts +18 -12
- package/dist/resource_clients/actor_collection.js +11 -3
- package/dist/resource_clients/actor_env_var.d.ts +11 -4
- package/dist/resource_clients/actor_env_var.js +16 -6
- package/dist/resource_clients/actor_env_var_collection.d.ts +9 -4
- package/dist/resource_clients/actor_env_var_collection.js +11 -4
- package/dist/resource_clients/actor_version.d.ts +20 -5
- package/dist/resource_clients/actor_version.js +19 -9
- package/dist/resource_clients/actor_version_collection.d.ts +11 -4
- package/dist/resource_clients/actor_version_collection.js +13 -4
- package/dist/resource_clients/build.d.ts +20 -7
- package/dist/resource_clients/build.js +35 -15
- package/dist/resource_clients/build_collection.d.ts +7 -5
- package/dist/resource_clients/build_collection.js +4 -1
- package/dist/resource_clients/dataset.d.ts +81 -36
- package/dist/resource_clients/dataset.js +109 -120
- package/dist/resource_clients/dataset_collection.d.ts +10 -6
- package/dist/resource_clients/dataset_collection.js +13 -2
- package/dist/resource_clients/key_value_store.d.ts +101 -23
- package/dist/resource_clients/key_value_store.js +121 -45
- package/dist/resource_clients/key_value_store_collection.d.ts +10 -6
- package/dist/resource_clients/key_value_store_collection.js +13 -2
- package/dist/resource_clients/log.d.ts +40 -21
- package/dist/resource_clients/log.js +144 -66
- package/dist/resource_clients/request_queue.d.ts +76 -31
- package/dist/resource_clients/request_queue.js +178 -93
- package/dist/resource_clients/request_queue_collection.d.ts +9 -5
- package/dist/resource_clients/request_queue_collection.js +9 -3
- package/dist/resource_clients/run.d.ts +119 -28
- package/dist/resource_clients/run.js +190 -49
- package/dist/resource_clients/run_collection.d.ts +5 -3
- package/dist/resource_clients/run_collection.js +4 -1
- package/dist/resource_clients/schedule.d.ts +16 -7
- package/dist/resource_clients/schedule.js +30 -22
- package/dist/resource_clients/schedule_collection.d.ts +6 -2
- package/dist/resource_clients/schedule_collection.js +9 -3
- package/dist/resource_clients/store_collection.d.ts +6 -4
- package/dist/resource_clients/store_collection.js +4 -1
- package/dist/resource_clients/task.d.ts +36 -13
- package/dist/resource_clients/task.js +79 -46
- package/dist/resource_clients/task_collection.d.ts +9 -5
- package/dist/resource_clients/task_collection.js +9 -3
- package/dist/resource_clients/user.d.ts +19 -10
- package/dist/resource_clients/user.js +42 -38
- package/dist/resource_clients/webhook.d.ts +17 -8
- package/dist/resource_clients/webhook.js +30 -22
- package/dist/resource_clients/webhook_collection.d.ts +8 -4
- package/dist/resource_clients/webhook_collection.js +9 -3
- package/dist/resource_clients/webhook_dispatch.d.ts +4 -1
- package/dist/resource_clients/webhook_dispatch.js +7 -2
- package/dist/resource_clients/webhook_dispatch_collection.d.ts +3 -1
- package/dist/resource_clients/webhook_dispatch_collection.js +4 -1
- package/dist/response_validation_error.d.ts +2 -1
- package/dist/response_validation_error.js +2 -1
- package/dist/runtime/node.d.ts +6 -0
- package/dist/runtime/node.js +49 -0
- package/dist/runtime/types.d.ts +52 -0
- package/dist/runtime/types.js +1 -0
- package/dist/runtime/web.d.ts +5 -0
- package/dist/runtime/web.js +18 -0
- package/dist/schemas.d.ts +1 -1
- package/dist/schemas.js +1 -1
- package/dist/statistics.d.ts +1 -6
- package/dist/statistics.js +2 -2
- package/dist/timeouts.d.ts +84 -0
- package/dist/timeouts.js +37 -0
- package/dist/timezones.d.ts +1 -1
- package/dist/utils.d.ts +88 -57
- package/dist/utils.js +219 -157
- package/dist/wait_for_resources.d.ts +8 -0
- package/dist/wait_for_resources.js +39 -0
- package/package.json +32 -29
- package/dist/http_client.d.ts +0 -88
- package/dist/http_client.js +0 -255
- package/dist/interceptors.d.ts +0 -19
- package/dist/interceptors.js +0 -105
|
@@ -1,24 +1,30 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import log from '@apify/log';
|
|
3
3
|
import { createHmacSignatureAsync, createStorageContentSignatureAsync } from '@apify/utilities';
|
|
4
|
-
import {
|
|
4
|
+
import { ResourceClient } from '../base/resource_client.js';
|
|
5
5
|
import * as schemas from '../schemas.js';
|
|
6
|
-
import {
|
|
6
|
+
import { timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
|
|
7
|
+
import { runtime } from '#runtime';
|
|
8
|
+
import { anyObjectSchema, applyQueryParamsToUrl, catchNotFoundOrThrow, isBuffer, isStream, minForLimitParam, parseArgument, parseResponse, } from '../utils.js';
|
|
7
9
|
const listKeysOptionsSchema = z.strictObject({
|
|
8
10
|
limit: z.number().min(0).optional(),
|
|
11
|
+
chunkSize: z.number().positive().optional(),
|
|
9
12
|
exclusiveStartKey: z.string().optional(),
|
|
10
13
|
collection: z.string().optional(),
|
|
11
14
|
prefix: z.string().optional(),
|
|
12
15
|
signature: z.string().optional(),
|
|
16
|
+
...timeoutOptionsShape,
|
|
13
17
|
});
|
|
14
18
|
const nonEmptyKeySchema = z.string().min(1);
|
|
15
|
-
// `
|
|
19
|
+
// `chunkSize` (client-side only) and `signature` (which this method produces) are left out. The options type omits
|
|
20
|
+
// both to match.
|
|
16
21
|
const createKeysPublicUrlOptionsSchema = z.strictObject({
|
|
17
22
|
limit: z.number().min(0).optional(),
|
|
18
23
|
exclusiveStartKey: z.string().optional(),
|
|
19
24
|
collection: z.string().optional(),
|
|
20
25
|
prefix: z.string().optional(),
|
|
21
26
|
expiresInSecs: z.number().optional(),
|
|
27
|
+
...timeoutOptionsShape,
|
|
22
28
|
});
|
|
23
29
|
const keySchema = z.string();
|
|
24
30
|
const getRecordOptionsSchema = z.strictObject({
|
|
@@ -26,6 +32,7 @@ const getRecordOptionsSchema = z.strictObject({
|
|
|
26
32
|
stream: z.boolean().optional(),
|
|
27
33
|
disableRedirect: z.boolean().optional(),
|
|
28
34
|
signature: z.string().optional(),
|
|
35
|
+
...timeoutOptionsShape,
|
|
29
36
|
});
|
|
30
37
|
const recordSchema = z.strictObject({
|
|
31
38
|
key: z.string(),
|
|
@@ -37,11 +44,12 @@ const recordSchema = z.strictObject({
|
|
|
37
44
|
typeof value !== 'symbol' &&
|
|
38
45
|
typeof value !== 'bigint' &&
|
|
39
46
|
typeof value !== 'function' &&
|
|
40
|
-
(typeof value !== 'number' || Number.isFinite(value)), 'Expected a
|
|
47
|
+
(typeof value !== 'number' || Number.isFinite(value)), 'Expected a JSON-serializable value, binary data, or a stream'),
|
|
41
48
|
contentType: z.string().min(1).optional(),
|
|
49
|
+
contentEncoding: z.string().trim().min(1).optional(),
|
|
42
50
|
});
|
|
43
51
|
const recordOptionsSchema = z.strictObject({
|
|
44
|
-
|
|
52
|
+
...timeoutOptionsShape,
|
|
45
53
|
doNotRetryTimeouts: z.boolean().optional(),
|
|
46
54
|
});
|
|
47
55
|
/**
|
|
@@ -85,11 +93,14 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
85
93
|
/**
|
|
86
94
|
* Gets the key-value store object from the Apify API.
|
|
87
95
|
*
|
|
96
|
+
* @param options - Request options
|
|
97
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
88
98
|
* @returns The KeyValueStore object, or `undefined` if it does not exist
|
|
89
99
|
* @see https://docs.apify.com/api/v2/key-value-store-get
|
|
90
100
|
*/
|
|
91
|
-
async get() {
|
|
92
|
-
|
|
101
|
+
async get(options = {}) {
|
|
102
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
103
|
+
return this.getResource(schemas.KeyValueStoreResource(), {}, timeoutSecs, signal);
|
|
93
104
|
}
|
|
94
105
|
/**
|
|
95
106
|
* Updates the key-value store with specified fields.
|
|
@@ -98,20 +109,26 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
98
109
|
* @param newFields.name - New name for the store
|
|
99
110
|
* @param newFields.title - New title for the store
|
|
100
111
|
* @param newFields.generalAccess - General resource access level ('FOLLOW_USER_SETTING', 'ANYONE_WITH_ID_CAN_READ' or 'RESTRICTED')
|
|
112
|
+
* @param options - Request options
|
|
113
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
101
114
|
* @returns The updated KeyValueStore object
|
|
102
115
|
* @see https://docs.apify.com/api/v2/key-value-store-put
|
|
103
116
|
*/
|
|
104
|
-
async update(newFields) {
|
|
117
|
+
async update(newFields, options = {}) {
|
|
105
118
|
parseArgument(newFields, anyObjectSchema);
|
|
106
|
-
|
|
119
|
+
const { timeoutSecs = 'long', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
120
|
+
return this.updateResource(schemas.KeyValueStoreResource(), newFields, timeoutSecs, signal);
|
|
107
121
|
}
|
|
108
122
|
/**
|
|
109
123
|
* Deletes the key-value store.
|
|
110
124
|
*
|
|
125
|
+
* @param options - Request options
|
|
126
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
111
127
|
* @see https://docs.apify.com/api/v2/key-value-store-delete
|
|
112
128
|
*/
|
|
113
|
-
async delete() {
|
|
114
|
-
|
|
129
|
+
async delete(options = {}) {
|
|
130
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
131
|
+
return this.deleteResource(timeoutSecs, signal);
|
|
115
132
|
}
|
|
116
133
|
/**
|
|
117
134
|
* Lists all keys in the key-value store.
|
|
@@ -121,9 +138,12 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
121
138
|
*
|
|
122
139
|
* @param options - Listing options
|
|
123
140
|
* @param options.limit - Maximum number of keys to return. Default is 1000.
|
|
141
|
+
* @param options.chunkSize - Maximum number of keys requested per API call when iterating across pages. `limit`
|
|
142
|
+
* then caps the total across all pages.
|
|
124
143
|
* @param options.exclusiveStartKey - Key to start listing from (for pagination). The listing starts with the next key after this one.
|
|
125
144
|
* @param options.collection - Filter keys by collection name.
|
|
126
145
|
* @param options.prefix - Filter keys that start with this prefix.
|
|
146
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
|
|
127
147
|
* @returns Object containing `items` array of key metadata, pagination info (`count`, `limit`, `isTruncated`, `nextExclusiveStartKey`)
|
|
128
148
|
* @see https://docs.apify.com/api/v2/key-value-store-keys-get
|
|
129
149
|
*
|
|
@@ -149,17 +169,19 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
149
169
|
* ```
|
|
150
170
|
*/
|
|
151
171
|
listKeys(options = {}) {
|
|
152
|
-
|
|
172
|
+
// `chunkSize` only sizes the page requests; it is not an API parameter, so it must not reach the query string.
|
|
173
|
+
const { chunkSize, timeoutSecs = 'medium', signal, ...parsed } = parseArgument(options, listKeysOptionsSchema, 'KeyValueClientListKeysOptions');
|
|
153
174
|
const getPaginatedList = async (kvsListOptions = {}) => {
|
|
154
175
|
const response = await this.httpClient.call({
|
|
155
|
-
url: this.
|
|
176
|
+
url: this.buildUrl('keys'),
|
|
156
177
|
method: 'GET',
|
|
157
|
-
params: this.
|
|
158
|
-
|
|
178
|
+
params: this.buildParams(kvsListOptions),
|
|
179
|
+
timeoutSecs,
|
|
180
|
+
signal,
|
|
159
181
|
});
|
|
160
182
|
return parseResponse(response, schemas.ListOfKeys());
|
|
161
183
|
};
|
|
162
|
-
const paginatedListPromise = getPaginatedList(parsed);
|
|
184
|
+
const paginatedListPromise = getPaginatedList({ ...parsed, limit: minForLimitParam(parsed.limit, chunkSize) });
|
|
163
185
|
async function* asyncGenerator() {
|
|
164
186
|
let currentPage = await paginatedListPromise;
|
|
165
187
|
yield* currentPage.items;
|
|
@@ -172,7 +194,7 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
172
194
|
) {
|
|
173
195
|
const newOptions = {
|
|
174
196
|
...parsed,
|
|
175
|
-
limit: remainingItems,
|
|
197
|
+
limit: minForLimitParam(remainingItems, chunkSize),
|
|
176
198
|
exclusiveStartKey: currentPage.nextExclusiveStartKey,
|
|
177
199
|
};
|
|
178
200
|
currentPage = await getPaginatedList(newOptions);
|
|
@@ -194,6 +216,8 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
194
216
|
* requiring an API token.
|
|
195
217
|
*
|
|
196
218
|
* @param key - The record key
|
|
219
|
+
* @param options - Request options
|
|
220
|
+
* @param options.timeoutSecs - Timeout for the API request that fetches the store. Default is `'long'`.
|
|
197
221
|
* @returns A public URL string for accessing the record
|
|
198
222
|
*
|
|
199
223
|
* @example
|
|
@@ -204,10 +228,11 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
204
228
|
* ```
|
|
205
229
|
* @since Added in 2.14.0
|
|
206
230
|
*/
|
|
207
|
-
async getRecordPublicUrl(key) {
|
|
231
|
+
async getRecordPublicUrl(key, options = {}) {
|
|
208
232
|
parseArgument(key, nonEmptyKeySchema);
|
|
209
|
-
const
|
|
210
|
-
const
|
|
233
|
+
const { timeoutSecs = 'long', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
234
|
+
const store = await this.get({ timeoutSecs, signal });
|
|
235
|
+
const recordPublicUrl = new URL(this.buildPublicUrl(['records', key]));
|
|
211
236
|
if (store?.urlSigningSecretKey) {
|
|
212
237
|
const signature = await createHmacSignatureAsync(store.urlSigningSecretKey, key);
|
|
213
238
|
recordPublicUrl.searchParams.append('signature', signature);
|
|
@@ -224,6 +249,7 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
224
249
|
* @param options.expiresInSecs - Number of seconds until the signed URL expires. If omitted, the URL never expires.
|
|
225
250
|
* @param options.limit - Maximum number of keys to return.
|
|
226
251
|
* @param options.prefix - Filter keys by prefix.
|
|
252
|
+
* @param options.timeoutSecs - Timeout for the API request that fetches the store. Default is `'long'`.
|
|
227
253
|
* @returns A public URL string for accessing the keys list
|
|
228
254
|
*
|
|
229
255
|
* @example
|
|
@@ -238,10 +264,9 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
238
264
|
* @since Added in 2.13.0
|
|
239
265
|
*/
|
|
240
266
|
async createKeysPublicUrl(options = {}) {
|
|
241
|
-
const
|
|
242
|
-
const store = await this.get();
|
|
243
|
-
|
|
244
|
-
let createdPublicKeysUrl = new URL(this._publicUrl('keys'));
|
|
267
|
+
const { timeoutSecs = 'long', signal, expiresInSecs, ...queryOptions } = parseArgument(options, createKeysPublicUrlOptionsSchema, 'KeyValueClientCreateKeysUrlOptions');
|
|
268
|
+
const store = await this.get({ timeoutSecs, signal });
|
|
269
|
+
let createdPublicKeysUrl = new URL(this.buildPublicUrl('keys'));
|
|
245
270
|
if (store?.urlSigningSecretKey) {
|
|
246
271
|
const signature = await createStorageContentSignatureAsync({
|
|
247
272
|
resourceId: store.id,
|
|
@@ -259,6 +284,8 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
259
284
|
* This is more efficient than {@link getRecord} when you only need to check for existence.
|
|
260
285
|
*
|
|
261
286
|
* @param key - The record key to check
|
|
287
|
+
* @param options - Request options
|
|
288
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
262
289
|
* @returns `true` if the record exists, `false` if it does not
|
|
263
290
|
* @see https://docs.apify.com/api/v2/key-value-store-record-get
|
|
264
291
|
*
|
|
@@ -271,11 +298,14 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
271
298
|
* ```
|
|
272
299
|
* @since Added in 2.9.0
|
|
273
300
|
*/
|
|
274
|
-
async recordExists(key) {
|
|
301
|
+
async recordExists(key, options = {}) {
|
|
302
|
+
const { timeoutSecs = 'long', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
275
303
|
const requestOpts = {
|
|
276
|
-
url: this.
|
|
304
|
+
url: this.buildUrl(['records', key]),
|
|
277
305
|
method: 'HEAD',
|
|
278
|
-
params: this.
|
|
306
|
+
params: this.buildParams(),
|
|
307
|
+
timeoutSecs,
|
|
308
|
+
signal,
|
|
279
309
|
};
|
|
280
310
|
try {
|
|
281
311
|
await this.httpClient.call(requestOpts);
|
|
@@ -289,7 +319,7 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
289
319
|
async getRecord(key, options = {}) {
|
|
290
320
|
parseArgument(key, keySchema);
|
|
291
321
|
const parsed = parseArgument(options, getRecordOptionsSchema, 'KeyValueClientGetRecordOptions');
|
|
292
|
-
if (parsed.stream && !isNode
|
|
322
|
+
if (parsed.stream && !runtime.isNode) {
|
|
293
323
|
throw new Error('The stream option can only be used in Node.js environment.');
|
|
294
324
|
}
|
|
295
325
|
if ('disableRedirect' in parsed) {
|
|
@@ -300,13 +330,14 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
300
330
|
if (parsed.signature)
|
|
301
331
|
queryParams.signature = parsed.signature;
|
|
302
332
|
const requestOpts = {
|
|
303
|
-
url: this.
|
|
333
|
+
url: this.buildUrl(['records', key]),
|
|
304
334
|
method: 'GET',
|
|
305
|
-
params: this.
|
|
306
|
-
|
|
335
|
+
params: this.buildParams(queryParams),
|
|
336
|
+
timeoutSecs: parsed.timeoutSecs ?? 'long',
|
|
337
|
+
signal: parsed.signal,
|
|
307
338
|
};
|
|
308
339
|
if (parsed.buffer)
|
|
309
|
-
requestOpts.
|
|
340
|
+
requestOpts.responseType = 'buffer';
|
|
310
341
|
if (parsed.stream)
|
|
311
342
|
requestOpts.responseType = 'stream';
|
|
312
343
|
try {
|
|
@@ -329,19 +360,36 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
329
360
|
* The content type is automatically determined based on the value type, but can be
|
|
330
361
|
* overridden using the `contentType` property.
|
|
331
362
|
*
|
|
363
|
+
* A string is sent as it is, even under `application/json`, so a string stored under a JSON content type has
|
|
364
|
+
* to be valid JSON already, or {@link getRecord} fails to parse it back.
|
|
365
|
+
*
|
|
332
366
|
* **Note about streams:** If the value is a stream object (has `.pipe` and `.on` methods),
|
|
333
367
|
* the upload cannot be retried on failure or follow redirects. For reliable uploads,
|
|
334
368
|
* buffer the entire stream into memory first.
|
|
335
369
|
*
|
|
336
370
|
* @param record - The record to store
|
|
337
371
|
* @param record.key - Record key (unique identifier)
|
|
338
|
-
* @param record.value - Record value (
|
|
372
|
+
* @param record.value - Record value (a JSON-serializable value, Buffer, ArrayBuffer, typed array, or Readable)
|
|
339
373
|
* @param record.contentType - Optional MIME type. Auto-detected if not provided:
|
|
340
374
|
* - Objects: `'application/json; charset=utf-8'`
|
|
341
375
|
* - Strings: `'text/plain; charset=utf-8'`
|
|
342
|
-
* -
|
|
376
|
+
* - Binary values and streams: `'application/octet-stream'`
|
|
377
|
+
*
|
|
378
|
+
* Worth setting for media and archives: the client skips compressing a body
|
|
379
|
+
* whose content type already carries its own compression. The
|
|
380
|
+
* `'application/octet-stream'` fallback is treated as compressible, because
|
|
381
|
+
* it could hold any binary data.
|
|
382
|
+
* @param record.contentEncoding - Optional encoding already applied to `value`, sent as the `Content-Encoding`
|
|
383
|
+
* header. Set it to upload a pre-compressed value: the client then sends the
|
|
384
|
+
* bytes as they are and does not compress them again. The API accepts `gzip`,
|
|
385
|
+
* `br`, `deflate`, and `identity`, and stores the record exactly as uploaded, so
|
|
386
|
+
* this is also the encoding the record is served with. Only binary values and
|
|
387
|
+
* streams can carry a compression, so any encoding other than `identity` throws
|
|
388
|
+
* a `TypeError` when `value` is a string, or a plain object, array, or other
|
|
389
|
+
* value that the client serializes to JSON. Node.js only, since browsers cannot
|
|
390
|
+
* send the header to the API.
|
|
343
391
|
* @param options - Storage options
|
|
344
|
-
* @param options.timeoutSecs - Timeout for the
|
|
392
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
345
393
|
* @param options.doNotRetryTimeouts - If `true`, don't retry on timeout errors. Default is `false`.
|
|
346
394
|
* @see https://docs.apify.com/api/v2/key-value-store-record-put
|
|
347
395
|
*
|
|
@@ -367,15 +415,35 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
367
415
|
* value: imageBuffer,
|
|
368
416
|
* contentType: 'image/png'
|
|
369
417
|
* });
|
|
418
|
+
*
|
|
419
|
+
* // Store a gzipped file without compressing it again
|
|
420
|
+
* const gzippedReport = await readFile('report.json.gz');
|
|
421
|
+
* await client.keyValueStore('my-store').setRecord({
|
|
422
|
+
* key: 'report.json',
|
|
423
|
+
* value: gzippedReport,
|
|
424
|
+
* contentType: 'application/json',
|
|
425
|
+
* contentEncoding: 'gzip'
|
|
426
|
+
* });
|
|
370
427
|
* ```
|
|
371
428
|
*/
|
|
372
429
|
async setRecord(record, options = {}) {
|
|
373
430
|
parseArgument(record, recordSchema);
|
|
374
431
|
const parsed = parseArgument(options, recordOptionsSchema, 'KeyValueStoreRecordOptions');
|
|
375
|
-
const { key } = record;
|
|
432
|
+
const { key, contentEncoding } = record;
|
|
376
433
|
let { value, contentType } = record;
|
|
377
|
-
const { timeoutSecs, doNotRetryTimeouts } = parsed;
|
|
434
|
+
const { timeoutSecs = 'long', signal, doNotRetryTimeouts } = parsed;
|
|
435
|
+
if (contentEncoding !== undefined && !runtime.isNode) {
|
|
436
|
+
throw new Error('The contentEncoding option can only be used in Node.js environment.');
|
|
437
|
+
}
|
|
378
438
|
const isValueStreamOrBuffer = isStream(value) || isBuffer(value);
|
|
439
|
+
// The client forwards the header without inspecting the body, so a string or a value serialized to JSON, which
|
|
440
|
+
// cannot hold compressed bytes, would be stored under an encoding that misdescribes it.
|
|
441
|
+
const declaresCompression = contentEncoding !== undefined && contentEncoding.trim().toLowerCase() !== 'identity';
|
|
442
|
+
if (declaresCompression && !isValueStreamOrBuffer) {
|
|
443
|
+
throw new TypeError(`Cannot upload a value of type ${typeof value} with Content-Encoding: ${contentEncoding}. ` +
|
|
444
|
+
'An encoding other than identity declares the value is already compressed, so pass the ' +
|
|
445
|
+
'compressed bytes as a Buffer, an ArrayBuffer, a typed array, or a readable stream.');
|
|
446
|
+
}
|
|
379
447
|
// To allow saving Objects to JSON without providing content type
|
|
380
448
|
if (!contentType) {
|
|
381
449
|
if (isValueStreamOrBuffer)
|
|
@@ -396,13 +464,17 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
396
464
|
}
|
|
397
465
|
}
|
|
398
466
|
const uploadOpts = {
|
|
399
|
-
url: this.
|
|
467
|
+
url: this.buildUrl(['records', key]),
|
|
400
468
|
method: 'PUT',
|
|
401
|
-
params: this.
|
|
469
|
+
params: this.buildParams(),
|
|
402
470
|
data: value,
|
|
403
|
-
headers:
|
|
471
|
+
headers: {
|
|
472
|
+
'content-type': contentType,
|
|
473
|
+
...(contentEncoding !== undefined && { 'content-encoding': contentEncoding }),
|
|
474
|
+
},
|
|
404
475
|
doNotRetryTimeouts,
|
|
405
|
-
|
|
476
|
+
timeoutSecs,
|
|
477
|
+
signal,
|
|
406
478
|
};
|
|
407
479
|
await this.httpClient.call(uploadOpts);
|
|
408
480
|
}
|
|
@@ -410,6 +482,8 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
410
482
|
* Deletes a record from the key-value store.
|
|
411
483
|
*
|
|
412
484
|
* @param key - The record key to delete
|
|
485
|
+
* @param options - Request options
|
|
486
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
413
487
|
* @see https://docs.apify.com/api/v2/key-value-store-record-delete
|
|
414
488
|
*
|
|
415
489
|
* @example
|
|
@@ -417,13 +491,15 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
417
491
|
* await client.keyValueStore('my-store').deleteRecord('temp-data');
|
|
418
492
|
* ```
|
|
419
493
|
*/
|
|
420
|
-
async deleteRecord(key) {
|
|
494
|
+
async deleteRecord(key, options = {}) {
|
|
421
495
|
parseArgument(key, keySchema);
|
|
496
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
422
497
|
await this.httpClient.call({
|
|
423
|
-
url: this.
|
|
498
|
+
url: this.buildUrl(['records', key]),
|
|
424
499
|
method: 'DELETE',
|
|
425
|
-
params: this.
|
|
426
|
-
|
|
500
|
+
params: this.buildParams(),
|
|
501
|
+
timeoutSecs,
|
|
502
|
+
signal,
|
|
427
503
|
});
|
|
428
504
|
}
|
|
429
505
|
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { STORAGE_OWNERSHIP_FILTER } from '@apify/consts';
|
|
2
2
|
import type { ApiClientSubResourceOptions } from '../base/api_client.js';
|
|
3
3
|
import { ResourceCollectionClient } from '../base/resource_collection_client.js';
|
|
4
|
+
import type { TimeoutOptions } from '../timeouts.js';
|
|
4
5
|
import type { PaginatedList, PaginationOptions } from '../utils.js';
|
|
5
|
-
import type {
|
|
6
|
+
import type { KeyValueStoreResource } from './key_value_store.js';
|
|
6
7
|
/**
|
|
7
8
|
* Client for managing the collection of Key-value stores in your account.
|
|
8
9
|
*
|
|
@@ -45,21 +46,24 @@ export declare class KeyValueStoreCollectionClient extends ResourceCollectionCli
|
|
|
45
46
|
* ```
|
|
46
47
|
*
|
|
47
48
|
* @param options - Pagination options.
|
|
49
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
|
|
48
50
|
* @returns A paginated iterator of Key-value stores.
|
|
49
51
|
* @see https://docs.apify.com/api/v2/key-value-stores-get
|
|
50
52
|
*/
|
|
51
|
-
list(options?: KeyValueStoreCollectionClientListOptions): Promise<KeyValueStoreCollectionListResult> & AsyncIterable<
|
|
53
|
+
list(options?: KeyValueStoreCollectionClientListOptions): Promise<KeyValueStoreCollectionListResult> & AsyncIterable<KeyValueStoreResource>;
|
|
52
54
|
/**
|
|
53
55
|
* Gets or creates a key-value store with the specified name.
|
|
54
56
|
*
|
|
55
57
|
* @param name - Name of the key-value store. If not provided, a default store is used.
|
|
56
58
|
* @param options - Additional options like schema.
|
|
59
|
+
* @param options.schema - Schema of the key-value store.
|
|
60
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
57
61
|
* @returns The key-value store object.
|
|
58
62
|
* @see https://docs.apify.com/api/v2/key-value-stores-post
|
|
59
63
|
*/
|
|
60
|
-
getOrCreate(name?: string, options?: KeyValueStoreCollectionClientGetOrCreateOptions): Promise<
|
|
64
|
+
getOrCreate(name?: string, options?: KeyValueStoreCollectionClientGetOrCreateOptions): Promise<KeyValueStoreResource>;
|
|
61
65
|
}
|
|
62
|
-
export interface KeyValueStoreCollectionClientListOptions extends PaginationOptions {
|
|
66
|
+
export interface KeyValueStoreCollectionClientListOptions extends PaginationOptions, TimeoutOptions {
|
|
63
67
|
unnamed?: boolean;
|
|
64
68
|
desc?: boolean;
|
|
65
69
|
/**
|
|
@@ -71,9 +75,9 @@ export interface KeyValueStoreCollectionClientListOptions extends PaginationOpti
|
|
|
71
75
|
/**
|
|
72
76
|
* @since Added in 2.3.0
|
|
73
77
|
*/
|
|
74
|
-
export interface KeyValueStoreCollectionClientGetOrCreateOptions {
|
|
78
|
+
export interface KeyValueStoreCollectionClientGetOrCreateOptions extends TimeoutOptions {
|
|
75
79
|
schema?: Record<string, unknown>;
|
|
76
80
|
}
|
|
77
|
-
export type KeyValueStoreCollectionListResult = PaginatedList<
|
|
81
|
+
export type KeyValueStoreCollectionListResult = PaginatedList<KeyValueStoreResource> & {
|
|
78
82
|
unnamed: boolean;
|
|
79
83
|
};
|
|
@@ -2,12 +2,14 @@ import { z } from 'zod';
|
|
|
2
2
|
import { STORAGE_OWNERSHIP_FILTER } from '@apify/consts';
|
|
3
3
|
import { ResourceCollectionClient } from '../base/resource_collection_client.js';
|
|
4
4
|
import * as schemas from '../schemas.js';
|
|
5
|
+
import { optionalSignalSchema, optionalTimeoutSchema, timeoutOptionsShape } from '../timeouts.js';
|
|
5
6
|
import { anyObjectSchema, paginationOptionsShape, parseArgument } from '../utils.js';
|
|
6
7
|
const listOptionsSchema = z.strictObject({
|
|
7
8
|
unnamed: z.boolean().optional(),
|
|
8
9
|
...paginationOptionsShape,
|
|
9
10
|
desc: z.boolean().optional(),
|
|
10
11
|
ownership: z.enum(STORAGE_OWNERSHIP_FILTER).optional(),
|
|
12
|
+
...timeoutOptionsShape,
|
|
11
13
|
});
|
|
12
14
|
const nameSchema = z.string().optional();
|
|
13
15
|
const schemaSchema = anyObjectSchema.optional();
|
|
@@ -58,24 +60,33 @@ export class KeyValueStoreCollectionClient extends ResourceCollectionClient {
|
|
|
58
60
|
* ```
|
|
59
61
|
*
|
|
60
62
|
* @param options - Pagination options.
|
|
63
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
|
|
61
64
|
* @returns A paginated iterator of Key-value stores.
|
|
62
65
|
* @see https://docs.apify.com/api/v2/key-value-stores-get
|
|
63
66
|
*/
|
|
64
67
|
list(options = {}) {
|
|
65
68
|
const parsed = parseArgument(options, listOptionsSchema, 'KeyValueStoreCollectionClientListOptions');
|
|
66
|
-
return this.
|
|
69
|
+
return this.listResourcesPaginated(schemas.ListOfKeyValueStores(), parsed, 'medium');
|
|
67
70
|
}
|
|
68
71
|
/**
|
|
69
72
|
* Gets or creates a key-value store with the specified name.
|
|
70
73
|
*
|
|
71
74
|
* @param name - Name of the key-value store. If not provided, a default store is used.
|
|
72
75
|
* @param options - Additional options like schema.
|
|
76
|
+
* @param options.schema - Schema of the key-value store.
|
|
77
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
73
78
|
* @returns The key-value store object.
|
|
74
79
|
* @see https://docs.apify.com/api/v2/key-value-stores-post
|
|
75
80
|
*/
|
|
76
81
|
async getOrCreate(name, options) {
|
|
77
82
|
parseArgument(name, nameSchema);
|
|
78
83
|
parseArgument(options?.schema, schemaSchema); // TODO: Add schema validation
|
|
79
|
-
|
|
84
|
+
parseArgument(options?.timeoutSecs, optionalTimeoutSchema);
|
|
85
|
+
parseArgument(options?.signal, optionalSignalSchema);
|
|
86
|
+
// `timeoutSecs` and `signal` are not part of the resource, so the body carries only the rest, and stays
|
|
87
|
+
// absent when there is nothing else to send.
|
|
88
|
+
const { timeoutSecs = 'short', signal, ...resource } = options ?? {};
|
|
89
|
+
const hasResource = Object.keys(resource).length > 0;
|
|
90
|
+
return this.getOrCreateResource(schemas.KeyValueStoreResource(), name, hasResource ? resource : undefined, timeoutSecs, signal);
|
|
80
91
|
}
|
|
81
92
|
}
|
|
@@ -3,6 +3,7 @@ import type { Log } from '@apify/log';
|
|
|
3
3
|
import { Logger, LogLevel } from '@apify/log';
|
|
4
4
|
import type { ApiClientSubResourceOptions } from '../base/api_client.js';
|
|
5
5
|
import { ResourceClient } from '../base/resource_client.js';
|
|
6
|
+
import type { TimeoutOptions } from '../timeouts.js';
|
|
6
7
|
/**
|
|
7
8
|
* Client for accessing Actor run or build logs.
|
|
8
9
|
*
|
|
@@ -18,6 +19,9 @@ import { ResourceClient } from '../base/resource_client.js';
|
|
|
18
19
|
* const log = await runClient.log().get();
|
|
19
20
|
* console.log(log);
|
|
20
21
|
*
|
|
22
|
+
* // Get the log content as raw bytes
|
|
23
|
+
* const logBuffer = await runClient.log().get({ buffer: true });
|
|
24
|
+
*
|
|
21
25
|
* // Stream the log in real-time
|
|
22
26
|
* const stream = await runClient.log().stream();
|
|
23
27
|
* stream.on('line', (line) => console.log(line));
|
|
@@ -33,18 +37,34 @@ export declare class LogClient extends ResourceClient {
|
|
|
33
37
|
/**
|
|
34
38
|
* Retrieves the log as a string.
|
|
35
39
|
*
|
|
40
|
+
* The `buffer` option only selects the return type: the bytes come back as a Buffer (Node.js) or ArrayBuffer
|
|
41
|
+
* (browser), not decoded to a string. Either way the whole log is downloaded in one response; to read it
|
|
42
|
+
* incrementally, use {@link LogClient.stream}.
|
|
43
|
+
*
|
|
36
44
|
* @param options - Log retrieval options.
|
|
37
45
|
* @param options.raw - If `true`, returns raw log content without any processing. Default is `false`.
|
|
38
|
-
* @
|
|
46
|
+
* @param options.buffer - If `true`, the log is returned as a Buffer (Node.js) or ArrayBuffer (browser) instead of a
|
|
47
|
+
* string. It does not change how the log is fetched.
|
|
48
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
49
|
+
* @returns The log content, or `undefined` if it does not exist. A chained client such as `run.log()` throws an
|
|
50
|
+
* `ApifyApiError` on a 404, since the run itself may be what is missing.
|
|
39
51
|
* @see https://docs.apify.com/api/v2/log-get
|
|
40
52
|
*/
|
|
41
|
-
get(options?:
|
|
53
|
+
get(options?: LogGetOptions & {
|
|
54
|
+
buffer?: false;
|
|
55
|
+
}): Promise<string | undefined>;
|
|
56
|
+
get(options: LogGetOptions & {
|
|
57
|
+
buffer: true;
|
|
58
|
+
}): Promise<Buffer | undefined>;
|
|
59
|
+
get(options?: LogGetOptions): Promise<string | Buffer | undefined>;
|
|
42
60
|
/**
|
|
43
61
|
* Retrieves the log as a Readable stream. Only works in Node.js.
|
|
44
62
|
*
|
|
45
63
|
* @param options - Log retrieval options.
|
|
46
64
|
* @param options.raw - If `true`, returns raw log content without any processing. Default is `false`.
|
|
47
|
-
* @
|
|
65
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
66
|
+
* @returns The log content as a Readable stream, or `undefined` if it does not exist. A chained client such as
|
|
67
|
+
* `run.log()` throws an `ApifyApiError` on a 404, since the run itself may be what is missing.
|
|
48
68
|
* @see https://docs.apify.com/api/v2/log-get
|
|
49
69
|
*/
|
|
50
70
|
stream(options?: LogOptions): Promise<Readable | undefined>;
|
|
@@ -52,10 +72,21 @@ export declare class LogClient extends ResourceClient {
|
|
|
52
72
|
/**
|
|
53
73
|
* @since Added in 2.20.0
|
|
54
74
|
*/
|
|
55
|
-
export interface LogOptions {
|
|
75
|
+
export interface LogOptions extends TimeoutOptions {
|
|
56
76
|
/** @default false */
|
|
57
77
|
raw?: boolean;
|
|
58
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* Options for {@link LogClient.get}.
|
|
81
|
+
*/
|
|
82
|
+
export interface LogGetOptions extends LogOptions {
|
|
83
|
+
/**
|
|
84
|
+
* If `true`, the log is returned as a Buffer (Node.js) or ArrayBuffer (browser) instead of a string. It only selects
|
|
85
|
+
* the return type; to read the log incrementally, use {@link LogClient.stream}.
|
|
86
|
+
* @default false
|
|
87
|
+
*/
|
|
88
|
+
buffer?: boolean;
|
|
89
|
+
}
|
|
59
90
|
/**
|
|
60
91
|
* Logger for redirected actor logs.
|
|
61
92
|
* @since Added in 2.20.0
|
|
@@ -69,31 +100,17 @@ export declare class LoggerActorRedirect extends Logger {
|
|
|
69
100
|
* @since Added in 2.20.0
|
|
70
101
|
*/
|
|
71
102
|
export declare class StreamedLog {
|
|
72
|
-
private
|
|
73
|
-
private streamBuffer;
|
|
74
|
-
private splitMarker;
|
|
75
|
-
private relevancyTimeLimit;
|
|
76
|
-
private logClient;
|
|
77
|
-
private streamingTask;
|
|
78
|
-
private stopLogging;
|
|
103
|
+
#private;
|
|
79
104
|
constructor(options: StreamedLogOptions);
|
|
80
105
|
/**
|
|
81
106
|
* Start log redirection.
|
|
82
107
|
*/
|
|
83
108
|
start(): void;
|
|
84
109
|
/**
|
|
85
|
-
* Stop log redirection.
|
|
110
|
+
* Stop log redirection. Waits up to one second for lines already in flight, then aborts the log stream request. If
|
|
111
|
+
* no stream has delivered anything yet, reads the whole log in one request instead.
|
|
86
112
|
*/
|
|
87
113
|
stop(): Promise<void>;
|
|
88
|
-
/**
|
|
89
|
-
* Get log stream from response and redirect it to another log.
|
|
90
|
-
*/
|
|
91
|
-
private streamLog;
|
|
92
|
-
private logStreamChunks;
|
|
93
|
-
/**
|
|
94
|
-
* Parse the buffer and log complete messages.
|
|
95
|
-
*/
|
|
96
|
-
private logBufferContent;
|
|
97
114
|
}
|
|
98
115
|
/**
|
|
99
116
|
* @since Added in 2.20.0
|
|
@@ -105,4 +122,6 @@ export interface StreamedLogOptions {
|
|
|
105
122
|
toLog: Log;
|
|
106
123
|
/** Whether to redirect all logs from Actor run start (even logs from the past). */
|
|
107
124
|
fromStart?: boolean;
|
|
125
|
+
/** Ends the log stream once it aborts. */
|
|
126
|
+
signal?: AbortSignal;
|
|
108
127
|
}
|