apify-client 2.25.1-beta.8 → 3.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apify_client.js +34 -30
- package/dist/argument_validation_error.d.ts +17 -0
- package/dist/argument_validation_error.js +157 -0
- package/dist/base/api_client.d.ts +2 -2
- package/dist/base/api_client.js +13 -10
- package/dist/bundle.js +66 -28147
- package/dist/bundle.js.LICENSE.txt +33 -0
- package/dist/bundle.js.map +1 -1
- package/dist/generated/api.d.ts +20522 -0
- package/dist/generated/api.js +6 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.mjs +1 -0
- package/dist/models.d.ts +1072 -0
- package/dist/models.js +89 -0
- package/dist/resource_clients/actor.d.ts +2 -448
- package/dist/resource_clients/actor.js +60 -55
- package/dist/resource_clients/actor_collection.d.ts +2 -7
- package/dist/resource_clients/actor_collection.js +13 -11
- package/dist/resource_clients/actor_env_var.js +2 -3
- package/dist/resource_clients/actor_env_var_collection.js +3 -3
- package/dist/resource_clients/actor_version.d.ts +3 -40
- package/dist/resource_clients/actor_version.js +8 -12
- package/dist/resource_clients/actor_version_collection.js +3 -3
- package/dist/resource_clients/build.d.ts +2 -77
- package/dist/resource_clients/build.js +7 -10
- package/dist/resource_clients/build_collection.d.ts +2 -2
- package/dist/resource_clients/build_collection.js +8 -8
- package/dist/resource_clients/dataset.d.ts +8 -72
- package/dist/resource_clients/dataset.js +66 -57
- package/dist/resource_clients/dataset_collection.js +14 -12
- package/dist/resource_clients/key_value_store.d.ts +5 -73
- package/dist/resource_clients/key_value_store.js +69 -51
- package/dist/resource_clients/key_value_store_collection.js +14 -12
- package/dist/resource_clients/request_queue.d.ts +15 -174
- package/dist/resource_clients/request_queue.js +85 -91
- package/dist/resource_clients/request_queue_collection.js +12 -11
- package/dist/resource_clients/run.js +39 -40
- package/dist/resource_clients/run_collection.js +12 -11
- package/dist/resource_clients/schedule.d.ts +3 -81
- package/dist/resource_clients/schedule.js +4 -12
- package/dist/resource_clients/schedule_collection.js +10 -9
- package/dist/resource_clients/store_collection.d.ts +2 -30
- package/dist/resource_clients/store_collection.js +13 -13
- package/dist/resource_clients/task.d.ts +3 -73
- package/dist/resource_clients/task.js +36 -33
- package/dist/resource_clients/task_collection.d.ts +2 -1
- package/dist/resource_clients/task_collection.js +9 -9
- package/dist/resource_clients/user.d.ts +3 -179
- package/dist/resource_clients/user.js +3 -11
- package/dist/resource_clients/webhook.d.ts +2 -61
- package/dist/resource_clients/webhook.js +1 -3
- package/dist/resource_clients/webhook_collection.js +10 -9
- package/dist/resource_clients/webhook_dispatch.d.ts +3 -36
- package/dist/resource_clients/webhook_dispatch.js +3 -7
- package/dist/resource_clients/webhook_dispatch_collection.js +8 -8
- package/dist/statistics.js +4 -3
- package/dist/utils.d.ts +51 -5
- package/dist/utils.js +103 -15
- package/package.json +22 -7
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import type { STORAGE_GENERAL_ACCESS } from '@apify/consts';
|
|
2
2
|
import type { ApiClientSubResourceOptions } from '../base/api_client';
|
|
3
3
|
import { ResourceClient } from '../base/resource_client';
|
|
4
|
+
import type { Dataset, DatasetStatistics } from '../models';
|
|
4
5
|
import type { PaginatedIterator, PaginationOptions } from '../utils';
|
|
6
|
+
export type { Dataset, DatasetStatistics, DatasetStats, FieldStatistics } from '../models';
|
|
5
7
|
/**
|
|
6
8
|
* Client for managing a specific Dataset.
|
|
7
9
|
*
|
|
@@ -222,55 +224,6 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
222
224
|
createItemsPublicUrl(options?: DatasetClientCreateItemsUrlOptions): Promise<string>;
|
|
223
225
|
private _createPaginationList;
|
|
224
226
|
}
|
|
225
|
-
/**
|
|
226
|
-
* Represents a dataset storage on the Apify platform.
|
|
227
|
-
*
|
|
228
|
-
* Datasets store structured data as a sequence of items (records). Each item is a JSON object.
|
|
229
|
-
* Datasets are useful for storing results from web scraping, crawling, or data processing tasks.
|
|
230
|
-
*/
|
|
231
|
-
export interface Dataset {
|
|
232
|
-
id: string;
|
|
233
|
-
name?: string;
|
|
234
|
-
/**
|
|
235
|
-
* @since Added in 2.6.1
|
|
236
|
-
*/
|
|
237
|
-
title?: string;
|
|
238
|
-
userId: string;
|
|
239
|
-
/**
|
|
240
|
-
* @since Added in 2.21.0
|
|
241
|
-
*/
|
|
242
|
-
username?: string;
|
|
243
|
-
createdAt: Date;
|
|
244
|
-
modifiedAt: Date;
|
|
245
|
-
accessedAt: Date;
|
|
246
|
-
itemCount: number;
|
|
247
|
-
cleanItemCount: number;
|
|
248
|
-
actId?: string;
|
|
249
|
-
actRunId?: string;
|
|
250
|
-
stats: DatasetStats;
|
|
251
|
-
fields: string[];
|
|
252
|
-
/**
|
|
253
|
-
* @since Added in 2.12.2
|
|
254
|
-
*/
|
|
255
|
-
generalAccess?: STORAGE_GENERAL_ACCESS | null;
|
|
256
|
-
/**
|
|
257
|
-
* @since Added in 2.13.0
|
|
258
|
-
*/
|
|
259
|
-
urlSigningSecretKey?: string | null;
|
|
260
|
-
/**
|
|
261
|
-
* @since Added in 2.13.0
|
|
262
|
-
*/
|
|
263
|
-
itemsPublicUrl: string;
|
|
264
|
-
}
|
|
265
|
-
/**
|
|
266
|
-
* Statistics about dataset usage and storage.
|
|
267
|
-
*/
|
|
268
|
-
export interface DatasetStats {
|
|
269
|
-
readCount?: number;
|
|
270
|
-
writeCount?: number;
|
|
271
|
-
deleteCount?: number;
|
|
272
|
-
storageBytes?: number;
|
|
273
|
-
}
|
|
274
227
|
/**
|
|
275
228
|
* Options for updating a dataset.
|
|
276
229
|
*/
|
|
@@ -315,10 +268,11 @@ export interface DatasetClientListItemOptions extends PaginationOptions {
|
|
|
315
268
|
/**
|
|
316
269
|
* Options for creating a public URL to access dataset items.
|
|
317
270
|
*
|
|
318
|
-
* Extends {@link DatasetClientListItemOptions} with URL expiration control
|
|
271
|
+
* Extends {@link DatasetClientListItemOptions} with URL expiration control, minus `chunkSize` (it only
|
|
272
|
+
* sizes a client-side iteration) and `signature` (this method produces one).
|
|
319
273
|
* @since Added in 2.16.0
|
|
320
274
|
*/
|
|
321
|
-
export interface DatasetClientCreateItemsUrlOptions extends DatasetClientListItemOptions {
|
|
275
|
+
export interface DatasetClientCreateItemsUrlOptions extends Omit<DatasetClientListItemOptions, 'chunkSize' | 'signature'> {
|
|
322
276
|
expiresInSecs?: number;
|
|
323
277
|
}
|
|
324
278
|
/**
|
|
@@ -336,9 +290,10 @@ export declare enum DownloadItemsFormat {
|
|
|
336
290
|
/**
|
|
337
291
|
* Options for downloading dataset items in a specific format.
|
|
338
292
|
*
|
|
339
|
-
* Extends {@link DatasetClientListItemOptions} with format-specific options
|
|
293
|
+
* Extends {@link DatasetClientListItemOptions} with format-specific options, minus `chunkSize` - this
|
|
294
|
+
* downloads the whole range in one request, so there is nothing to size.
|
|
340
295
|
*/
|
|
341
|
-
export interface DatasetClientDownloadItemsOptions extends DatasetClientListItemOptions {
|
|
296
|
+
export interface DatasetClientDownloadItemsOptions extends Omit<DatasetClientListItemOptions, 'chunkSize'> {
|
|
342
297
|
attachment?: boolean;
|
|
343
298
|
bom?: boolean;
|
|
344
299
|
delimiter?: string;
|
|
@@ -346,22 +301,3 @@ export interface DatasetClientDownloadItemsOptions extends DatasetClientListItem
|
|
|
346
301
|
xmlRoot?: string;
|
|
347
302
|
xmlRow?: string;
|
|
348
303
|
}
|
|
349
|
-
/**
|
|
350
|
-
* Statistical information about dataset fields.
|
|
351
|
-
*
|
|
352
|
-
* Provides insights into the data structure and content of the dataset.
|
|
353
|
-
* @since Added in 2.11.2
|
|
354
|
-
*/
|
|
355
|
-
export interface DatasetStatistics {
|
|
356
|
-
fieldStatistics: Record<string, FieldStatistics>;
|
|
357
|
-
}
|
|
358
|
-
/**
|
|
359
|
-
* Statistics for a single field in a dataset.
|
|
360
|
-
* @since Added in 2.11.2
|
|
361
|
-
*/
|
|
362
|
-
export interface FieldStatistics {
|
|
363
|
-
min?: number;
|
|
364
|
-
max?: number;
|
|
365
|
-
nullCount?: number;
|
|
366
|
-
emptyCount?: number;
|
|
367
|
-
}
|
|
@@ -1,11 +1,64 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.DownloadItemsFormat = exports.DatasetClient = void 0;
|
|
4
|
-
const
|
|
5
|
-
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
4
|
+
const zod_1 = require("zod");
|
|
6
5
|
const utilities_1 = require("@apify/utilities");
|
|
7
6
|
const resource_client_1 = require("../base/resource_client");
|
|
8
7
|
const utils_1 = require("../utils");
|
|
8
|
+
// A predicate, not a `z.object()` arm, which would walk and copy every key of every pushed item.
|
|
9
|
+
const itemSchema = zod_1.z.custom(utils_1.isNonArrayObject, 'Expected an object');
|
|
10
|
+
const listItemsOptionsSchema = zod_1.z.strictObject({
|
|
11
|
+
clean: zod_1.z.boolean().optional(),
|
|
12
|
+
desc: zod_1.z.boolean().optional(),
|
|
13
|
+
flatten: zod_1.z.array(zod_1.z.string()).optional(),
|
|
14
|
+
fields: zod_1.z.array(zod_1.z.string()).optional(),
|
|
15
|
+
omit: zod_1.z.array(zod_1.z.string()).optional(),
|
|
16
|
+
...utils_1.paginationOptionsShape,
|
|
17
|
+
skipEmpty: zod_1.z.boolean().optional(),
|
|
18
|
+
skipHidden: zod_1.z.boolean().optional(),
|
|
19
|
+
unwind: zod_1.z.union([zod_1.z.string(), zod_1.z.array(zod_1.z.string())]).optional(),
|
|
20
|
+
view: zod_1.z.string().optional(),
|
|
21
|
+
signature: zod_1.z.string().optional(),
|
|
22
|
+
});
|
|
23
|
+
// Spells out `limit` / `offset` instead of spreading `paginationOptionsShape`: a download is one
|
|
24
|
+
// request, so `chunkSize` has nothing to size. The options type omits it to match.
|
|
25
|
+
const downloadItemsOptionsSchema = zod_1.z.strictObject({
|
|
26
|
+
attachment: zod_1.z.boolean().optional(),
|
|
27
|
+
bom: zod_1.z.boolean().optional(),
|
|
28
|
+
clean: zod_1.z.boolean().optional(),
|
|
29
|
+
delimiter: zod_1.z.string().optional(),
|
|
30
|
+
desc: zod_1.z.boolean().optional(),
|
|
31
|
+
flatten: zod_1.z.array(zod_1.z.string()).optional(),
|
|
32
|
+
fields: zod_1.z.array(zod_1.z.string()).optional(),
|
|
33
|
+
omit: zod_1.z.array(zod_1.z.string()).optional(),
|
|
34
|
+
limit: zod_1.z.number().min(0).optional(),
|
|
35
|
+
offset: zod_1.z.number().min(0).optional(),
|
|
36
|
+
skipEmpty: zod_1.z.boolean().optional(),
|
|
37
|
+
skipHeaderRow: zod_1.z.boolean().optional(),
|
|
38
|
+
skipHidden: zod_1.z.boolean().optional(),
|
|
39
|
+
unwind: zod_1.z.union([zod_1.z.string(), zod_1.z.array(zod_1.z.string())]).optional(),
|
|
40
|
+
view: zod_1.z.string().optional(),
|
|
41
|
+
xmlRoot: zod_1.z.string().optional(),
|
|
42
|
+
xmlRow: zod_1.z.string().optional(),
|
|
43
|
+
signature: zod_1.z.string().optional(),
|
|
44
|
+
});
|
|
45
|
+
const pushItemsSchema = zod_1.z.union([itemSchema, zod_1.z.string(), zod_1.z.array(zod_1.z.union([itemSchema, zod_1.z.string()]))]);
|
|
46
|
+
// Every option becomes a query parameter of the generated URL, so `chunkSize` (client-side only) and
|
|
47
|
+
// `signature` (which this method produces) are left out. The options type omits both to match.
|
|
48
|
+
const createItemsPublicUrlOptionsSchema = zod_1.z.strictObject({
|
|
49
|
+
clean: zod_1.z.boolean().optional(),
|
|
50
|
+
desc: zod_1.z.boolean().optional(),
|
|
51
|
+
flatten: zod_1.z.array(zod_1.z.string()).optional(),
|
|
52
|
+
fields: zod_1.z.array(zod_1.z.string()).optional(),
|
|
53
|
+
omit: zod_1.z.array(zod_1.z.string()).optional(),
|
|
54
|
+
limit: zod_1.z.number().min(0).optional(),
|
|
55
|
+
offset: zod_1.z.number().min(0).optional(),
|
|
56
|
+
skipEmpty: zod_1.z.boolean().optional(),
|
|
57
|
+
skipHidden: zod_1.z.boolean().optional(),
|
|
58
|
+
unwind: zod_1.z.union([zod_1.z.string(), zod_1.z.array(zod_1.z.string())]).optional(),
|
|
59
|
+
view: zod_1.z.string().optional(),
|
|
60
|
+
expiresInSecs: zod_1.z.number().optional(),
|
|
61
|
+
});
|
|
9
62
|
/**
|
|
10
63
|
* Client for managing a specific Dataset.
|
|
11
64
|
*
|
|
@@ -62,7 +115,7 @@ class DatasetClient extends resource_client_1.ResourceClient {
|
|
|
62
115
|
* @see https://docs.apify.com/api/v2/dataset-put
|
|
63
116
|
*/
|
|
64
117
|
async update(newFields) {
|
|
65
|
-
(0,
|
|
118
|
+
(0, utils_1.parseArgument)(newFields, utils_1.anyObjectSchema);
|
|
66
119
|
return this._update(newFields, resource_client_1.SMALL_TIMEOUT_MILLIS);
|
|
67
120
|
}
|
|
68
121
|
/**
|
|
@@ -118,21 +171,7 @@ class DatasetClient extends resource_client_1.ResourceClient {
|
|
|
118
171
|
* ```
|
|
119
172
|
*/
|
|
120
173
|
listItems(options = {}) {
|
|
121
|
-
(0,
|
|
122
|
-
clean: ow_1.default.optional.boolean,
|
|
123
|
-
desc: ow_1.default.optional.boolean,
|
|
124
|
-
flatten: ow_1.default.optional.array.ofType(ow_1.default.string),
|
|
125
|
-
fields: ow_1.default.optional.array.ofType(ow_1.default.string),
|
|
126
|
-
omit: ow_1.default.optional.array.ofType(ow_1.default.string),
|
|
127
|
-
limit: ow_1.default.optional.number.not.negative,
|
|
128
|
-
offset: ow_1.default.optional.number.not.negative,
|
|
129
|
-
chunkSize: ow_1.default.optional.number.positive,
|
|
130
|
-
skipEmpty: ow_1.default.optional.boolean,
|
|
131
|
-
skipHidden: ow_1.default.optional.boolean,
|
|
132
|
-
unwind: ow_1.default.optional.any(ow_1.default.string, ow_1.default.array.ofType(ow_1.default.string)),
|
|
133
|
-
view: ow_1.default.optional.string,
|
|
134
|
-
signature: ow_1.default.optional.string,
|
|
135
|
-
}));
|
|
174
|
+
const parsed = (0, utils_1.parseArgument)(options, listItemsOptionsSchema, 'DatasetClientListItemOptions');
|
|
136
175
|
const fetchItems = async (datasetListOptions = {}) => {
|
|
137
176
|
const response = await this.httpClient.call({
|
|
138
177
|
url: this._url('items'),
|
|
@@ -142,7 +181,7 @@ class DatasetClient extends resource_client_1.ResourceClient {
|
|
|
142
181
|
});
|
|
143
182
|
return this._createPaginationList(response, datasetListOptions.desc ?? false);
|
|
144
183
|
};
|
|
145
|
-
return this._listPaginatedFromCallback(fetchItems,
|
|
184
|
+
return this._listPaginatedFromCallback(fetchItems, parsed);
|
|
146
185
|
}
|
|
147
186
|
/**
|
|
148
187
|
* Downloads dataset items in a specific format.
|
|
@@ -185,33 +224,14 @@ class DatasetClient extends resource_client_1.ResourceClient {
|
|
|
185
224
|
* ```
|
|
186
225
|
*/
|
|
187
226
|
async downloadItems(format, options = {}) {
|
|
188
|
-
(0,
|
|
189
|
-
(0,
|
|
190
|
-
attachment: ow_1.default.optional.boolean,
|
|
191
|
-
bom: ow_1.default.optional.boolean,
|
|
192
|
-
clean: ow_1.default.optional.boolean,
|
|
193
|
-
delimiter: ow_1.default.optional.string,
|
|
194
|
-
desc: ow_1.default.optional.boolean,
|
|
195
|
-
flatten: ow_1.default.optional.array.ofType(ow_1.default.string),
|
|
196
|
-
fields: ow_1.default.optional.array.ofType(ow_1.default.string),
|
|
197
|
-
omit: ow_1.default.optional.array.ofType(ow_1.default.string),
|
|
198
|
-
limit: ow_1.default.optional.number.not.negative,
|
|
199
|
-
offset: ow_1.default.optional.number.not.negative,
|
|
200
|
-
skipEmpty: ow_1.default.optional.boolean,
|
|
201
|
-
skipHeaderRow: ow_1.default.optional.boolean,
|
|
202
|
-
skipHidden: ow_1.default.optional.boolean,
|
|
203
|
-
unwind: ow_1.default.any(ow_1.default.optional.string, ow_1.default.optional.array.ofType(ow_1.default.string)),
|
|
204
|
-
view: ow_1.default.optional.string,
|
|
205
|
-
xmlRoot: ow_1.default.optional.string,
|
|
206
|
-
xmlRow: ow_1.default.optional.string,
|
|
207
|
-
signature: ow_1.default.optional.string,
|
|
208
|
-
}));
|
|
227
|
+
(0, utils_1.parseArgument)(format, itemFormatSchema);
|
|
228
|
+
const parsed = (0, utils_1.parseArgument)(options, downloadItemsOptionsSchema, 'DatasetClientDownloadItemsOptions');
|
|
209
229
|
const { data } = await this.httpClient.call({
|
|
210
230
|
url: this._url('items'),
|
|
211
231
|
method: 'GET',
|
|
212
232
|
params: this._params({
|
|
213
233
|
format,
|
|
214
|
-
...
|
|
234
|
+
...parsed,
|
|
215
235
|
}),
|
|
216
236
|
forceBuffer: true,
|
|
217
237
|
timeout: resource_client_1.DEFAULT_TIMEOUT_MILLIS,
|
|
@@ -251,7 +271,7 @@ class DatasetClient extends resource_client_1.ResourceClient {
|
|
|
251
271
|
* ```
|
|
252
272
|
*/
|
|
253
273
|
async pushItems(items) {
|
|
254
|
-
(0,
|
|
274
|
+
(0, utils_1.parseArgument)(items, pushItemsSchema);
|
|
255
275
|
await this.httpClient.call({
|
|
256
276
|
url: this._url('items'),
|
|
257
277
|
method: 'POST',
|
|
@@ -323,22 +343,9 @@ class DatasetClient extends resource_client_1.ResourceClient {
|
|
|
323
343
|
* @since Added in 2.13.0
|
|
324
344
|
*/
|
|
325
345
|
async createItemsPublicUrl(options = {}) {
|
|
326
|
-
(0,
|
|
327
|
-
clean: ow_1.default.optional.boolean,
|
|
328
|
-
desc: ow_1.default.optional.boolean,
|
|
329
|
-
flatten: ow_1.default.optional.array.ofType(ow_1.default.string),
|
|
330
|
-
fields: ow_1.default.optional.array.ofType(ow_1.default.string),
|
|
331
|
-
omit: ow_1.default.optional.array.ofType(ow_1.default.string),
|
|
332
|
-
limit: ow_1.default.optional.number.not.negative,
|
|
333
|
-
offset: ow_1.default.optional.number.not.negative,
|
|
334
|
-
skipEmpty: ow_1.default.optional.boolean,
|
|
335
|
-
skipHidden: ow_1.default.optional.boolean,
|
|
336
|
-
unwind: ow_1.default.optional.any(ow_1.default.string, ow_1.default.array.ofType(ow_1.default.string)),
|
|
337
|
-
view: ow_1.default.optional.string,
|
|
338
|
-
expiresInSecs: ow_1.default.optional.number,
|
|
339
|
-
}));
|
|
346
|
+
const parsed = (0, utils_1.parseArgument)(options, createItemsPublicUrlOptionsSchema, 'DatasetClientCreateItemsUrlOptions');
|
|
340
347
|
const dataset = await this.get();
|
|
341
|
-
const { expiresInSecs, ...queryOptions } =
|
|
348
|
+
const { expiresInSecs, ...queryOptions } = parsed;
|
|
342
349
|
let createdItemsPublicUrl = new URL(this._publicUrl('items'));
|
|
343
350
|
if (dataset?.urlSigningSecretKey) {
|
|
344
351
|
const signature = await (0, utilities_1.createStorageContentSignatureAsync)({
|
|
@@ -378,3 +385,5 @@ var DownloadItemsFormat;
|
|
|
378
385
|
DownloadItemsFormat["RSS"] = "rss";
|
|
379
386
|
})(DownloadItemsFormat || (exports.DownloadItemsFormat = DownloadItemsFormat = {}));
|
|
380
387
|
const validItemFormats = [...new Set(Object.values(DownloadItemsFormat).map((item) => item.toLowerCase()))];
|
|
388
|
+
// Declared below `validItemFormats` because it references it at module load time.
|
|
389
|
+
const itemFormatSchema = zod_1.z.enum(validItemFormats);
|
|
@@ -1,10 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.DatasetCollectionClient = void 0;
|
|
4
|
-
const
|
|
5
|
-
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
4
|
+
const zod_1 = require("zod");
|
|
6
5
|
const consts_1 = require("@apify/consts");
|
|
7
6
|
const resource_collection_client_1 = require("../base/resource_collection_client");
|
|
7
|
+
const utils_1 = require("../utils");
|
|
8
|
+
const listOptionsSchema = zod_1.z.strictObject({
|
|
9
|
+
unnamed: zod_1.z.boolean().optional(),
|
|
10
|
+
...utils_1.paginationOptionsShape,
|
|
11
|
+
desc: zod_1.z.boolean().optional(),
|
|
12
|
+
ownership: zod_1.z.enum(consts_1.STORAGE_OWNERSHIP_FILTER).optional(),
|
|
13
|
+
});
|
|
14
|
+
const nameSchema = zod_1.z.string().optional();
|
|
15
|
+
const schemaSchema = utils_1.anyObjectSchema.optional();
|
|
8
16
|
/**
|
|
9
17
|
* Client for managing the collection of datasets in your account.
|
|
10
18
|
*
|
|
@@ -56,14 +64,8 @@ class DatasetCollectionClient extends resource_collection_client_1.ResourceColle
|
|
|
56
64
|
* @see https://docs.apify.com/api/v2/datasets-get
|
|
57
65
|
*/
|
|
58
66
|
list(options = {}) {
|
|
59
|
-
(0,
|
|
60
|
-
|
|
61
|
-
limit: ow_1.default.optional.number.not.negative,
|
|
62
|
-
offset: ow_1.default.optional.number.not.negative,
|
|
63
|
-
desc: ow_1.default.optional.boolean,
|
|
64
|
-
ownership: ow_1.default.optional.string.oneOf(Object.values(consts_1.STORAGE_OWNERSHIP_FILTER)),
|
|
65
|
-
}));
|
|
66
|
-
return this._listPaginated(options);
|
|
67
|
+
const parsed = (0, utils_1.parseArgument)(options, listOptionsSchema, 'DatasetCollectionClientListOptions');
|
|
68
|
+
return this._listPaginated(parsed);
|
|
67
69
|
}
|
|
68
70
|
/**
|
|
69
71
|
* Gets or creates a dataset with the specified name.
|
|
@@ -74,8 +76,8 @@ class DatasetCollectionClient extends resource_collection_client_1.ResourceColle
|
|
|
74
76
|
* @see https://docs.apify.com/api/v2/datasets-post
|
|
75
77
|
*/
|
|
76
78
|
async getOrCreate(name, options) {
|
|
77
|
-
(0,
|
|
78
|
-
(0,
|
|
79
|
+
(0, utils_1.parseArgument)(name, nameSchema);
|
|
80
|
+
(0, utils_1.parseArgument)(options?.schema, schemaSchema); // TODO: Add schema validation
|
|
79
81
|
return this._getOrCreate(name, options);
|
|
80
82
|
}
|
|
81
83
|
}
|
|
@@ -3,6 +3,8 @@ import type { JsonValue } from 'type-fest';
|
|
|
3
3
|
import type { STORAGE_GENERAL_ACCESS } from '@apify/consts';
|
|
4
4
|
import type { ApiClientSubResourceOptions } from '../base/api_client';
|
|
5
5
|
import { ResourceClient } from '../base/resource_client';
|
|
6
|
+
import type { KeyValueClientListKeysResult, KeyValueListItem, KeyValueStore } from '../models';
|
|
7
|
+
export type { KeyValueClientListKeysResult, KeyValueListItem, KeyValueStore, KeyValueStoreStats } from '../models';
|
|
6
8
|
/**
|
|
7
9
|
* Client for managing a specific key-value store.
|
|
8
10
|
*
|
|
@@ -231,53 +233,6 @@ export declare class KeyValueStoreClient extends ResourceClient {
|
|
|
231
233
|
*/
|
|
232
234
|
deleteRecord(key: string): Promise<void>;
|
|
233
235
|
}
|
|
234
|
-
/**
|
|
235
|
-
* Represents a Key-Value Store storage on the Apify platform.
|
|
236
|
-
*
|
|
237
|
-
* Key-value stores are used to store arbitrary data records or files. Each record is identified
|
|
238
|
-
* by a unique key and can contain any data - JSON objects, strings, binary files, etc.
|
|
239
|
-
*/
|
|
240
|
-
export interface KeyValueStore {
|
|
241
|
-
id: string;
|
|
242
|
-
name?: string;
|
|
243
|
-
/**
|
|
244
|
-
* @since Added in 2.6.1
|
|
245
|
-
*/
|
|
246
|
-
title?: string;
|
|
247
|
-
userId: string;
|
|
248
|
-
/**
|
|
249
|
-
* @since Added in 2.21.0
|
|
250
|
-
*/
|
|
251
|
-
username?: string;
|
|
252
|
-
createdAt: Date;
|
|
253
|
-
modifiedAt: Date;
|
|
254
|
-
accessedAt: Date;
|
|
255
|
-
actId?: string;
|
|
256
|
-
actRunId?: string;
|
|
257
|
-
stats?: KeyValueStoreStats;
|
|
258
|
-
/**
|
|
259
|
-
* @since Added in 2.12.2
|
|
260
|
-
*/
|
|
261
|
-
generalAccess?: STORAGE_GENERAL_ACCESS | null;
|
|
262
|
-
/**
|
|
263
|
-
* @since Added in 2.13.0
|
|
264
|
-
*/
|
|
265
|
-
urlSigningSecretKey?: string | null;
|
|
266
|
-
/**
|
|
267
|
-
* @since Added in 2.13.0
|
|
268
|
-
*/
|
|
269
|
-
keysPublicUrl: string;
|
|
270
|
-
}
|
|
271
|
-
/**
|
|
272
|
-
* Statistics about Key-Value Store usage and storage.
|
|
273
|
-
*/
|
|
274
|
-
export interface KeyValueStoreStats {
|
|
275
|
-
readCount?: number;
|
|
276
|
-
writeCount?: number;
|
|
277
|
-
deleteCount?: number;
|
|
278
|
-
listCount?: number;
|
|
279
|
-
storageBytes?: number;
|
|
280
|
-
}
|
|
281
236
|
/**
|
|
282
237
|
* Options for updating a Key-Value Store.
|
|
283
238
|
*/
|
|
@@ -311,36 +266,13 @@ export interface KeyValueClientListKeysOptions {
|
|
|
311
266
|
/**
|
|
312
267
|
* Options for creating a public URL to list keys in a Key-Value Store.
|
|
313
268
|
*
|
|
314
|
-
* Extends {@link KeyValueClientListKeysOptions} with URL expiration control
|
|
269
|
+
* Extends {@link KeyValueClientListKeysOptions} with URL expiration control, minus `signature` (this
|
|
270
|
+
* method produces one).
|
|
315
271
|
* @since Added in 2.16.0
|
|
316
272
|
*/
|
|
317
|
-
export interface KeyValueClientCreateKeysUrlOptions extends KeyValueClientListKeysOptions {
|
|
273
|
+
export interface KeyValueClientCreateKeysUrlOptions extends Omit<KeyValueClientListKeysOptions, 'signature'> {
|
|
318
274
|
expiresInSecs?: number;
|
|
319
275
|
}
|
|
320
|
-
/**
|
|
321
|
-
* Result of listing keys in a Key-Value Store.
|
|
322
|
-
*
|
|
323
|
-
* Contains paginated list of keys with metadata and pagination information.
|
|
324
|
-
*/
|
|
325
|
-
export interface KeyValueClientListKeysResult {
|
|
326
|
-
count: number;
|
|
327
|
-
limit: number;
|
|
328
|
-
exclusiveStartKey: string;
|
|
329
|
-
isTruncated: boolean;
|
|
330
|
-
nextExclusiveStartKey: string;
|
|
331
|
-
items: KeyValueListItem[];
|
|
332
|
-
}
|
|
333
|
-
/**
|
|
334
|
-
* Metadata about a single key in a Key-Value Store.
|
|
335
|
-
*/
|
|
336
|
-
export interface KeyValueListItem {
|
|
337
|
-
key: string;
|
|
338
|
-
size: number;
|
|
339
|
-
/**
|
|
340
|
-
* @since Added in 2.15.1
|
|
341
|
-
*/
|
|
342
|
-
recordPublicUrl: string;
|
|
343
|
-
}
|
|
344
276
|
/**
|
|
345
277
|
* Options for retrieving a record from a Key-Value Store.
|
|
346
278
|
*/
|
|
@@ -2,11 +2,51 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.KeyValueStoreClient = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
|
-
const
|
|
5
|
+
const zod_1 = require("zod");
|
|
6
6
|
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
7
7
|
const utilities_1 = require("@apify/utilities");
|
|
8
8
|
const resource_client_1 = require("../base/resource_client");
|
|
9
9
|
const utils_1 = require("../utils");
|
|
10
|
+
const listKeysOptionsSchema = zod_1.z.strictObject({
|
|
11
|
+
limit: zod_1.z.number().min(0).optional(),
|
|
12
|
+
exclusiveStartKey: zod_1.z.string().optional(),
|
|
13
|
+
collection: zod_1.z.string().optional(),
|
|
14
|
+
prefix: zod_1.z.string().optional(),
|
|
15
|
+
signature: zod_1.z.string().optional(),
|
|
16
|
+
});
|
|
17
|
+
const nonEmptyKeySchema = zod_1.z.string().min(1);
|
|
18
|
+
// `signature` is left out - this method produces one. The options type omits it to match.
|
|
19
|
+
const createKeysPublicUrlOptionsSchema = zod_1.z.strictObject({
|
|
20
|
+
limit: zod_1.z.number().min(0).optional(),
|
|
21
|
+
exclusiveStartKey: zod_1.z.string().optional(),
|
|
22
|
+
collection: zod_1.z.string().optional(),
|
|
23
|
+
prefix: zod_1.z.string().optional(),
|
|
24
|
+
expiresInSecs: zod_1.z.number().optional(),
|
|
25
|
+
});
|
|
26
|
+
const keySchema = zod_1.z.string();
|
|
27
|
+
const getRecordOptionsSchema = zod_1.z.strictObject({
|
|
28
|
+
buffer: zod_1.z.boolean().optional(),
|
|
29
|
+
stream: zod_1.z.boolean().optional(),
|
|
30
|
+
disableRedirect: zod_1.z.boolean().optional(),
|
|
31
|
+
signature: zod_1.z.string().optional(),
|
|
32
|
+
});
|
|
33
|
+
const recordSchema = zod_1.z.strictObject({
|
|
34
|
+
key: zod_1.z.string(),
|
|
35
|
+
// Values are arbitrary, and a `z.object()` arm would walk every key of a large buffer.
|
|
36
|
+
// Symbols, bigints, functions and non-finite numbers are rejected because they cannot be serialized
|
|
37
|
+
// to JSON - symbols and bigints throw, functions stringify to `undefined` (an empty request body),
|
|
38
|
+
// and `NaN` / `Infinity` would silently become `null`.
|
|
39
|
+
value: zod_1.z.custom((value) => value !== undefined &&
|
|
40
|
+
typeof value !== 'symbol' &&
|
|
41
|
+
typeof value !== 'bigint' &&
|
|
42
|
+
typeof value !== 'function' &&
|
|
43
|
+
(typeof value !== 'number' || Number.isFinite(value)), 'Expected a defined, JSON-serializable value'),
|
|
44
|
+
contentType: zod_1.z.string().min(1).optional(),
|
|
45
|
+
});
|
|
46
|
+
const recordOptionsSchema = zod_1.z.strictObject({
|
|
47
|
+
timeoutSecs: zod_1.z.number().optional(),
|
|
48
|
+
doNotRetryTimeouts: zod_1.z.boolean().optional(),
|
|
49
|
+
});
|
|
10
50
|
/**
|
|
11
51
|
* Client for managing a specific key-value store.
|
|
12
52
|
*
|
|
@@ -65,7 +105,7 @@ class KeyValueStoreClient extends resource_client_1.ResourceClient {
|
|
|
65
105
|
* @see https://docs.apify.com/api/v2/key-value-store-put
|
|
66
106
|
*/
|
|
67
107
|
async update(newFields) {
|
|
68
|
-
(0,
|
|
108
|
+
(0, utils_1.parseArgument)(newFields, utils_1.anyObjectSchema);
|
|
69
109
|
return this._update(newFields, resource_client_1.DEFAULT_TIMEOUT_MILLIS);
|
|
70
110
|
}
|
|
71
111
|
/**
|
|
@@ -112,13 +152,7 @@ class KeyValueStoreClient extends resource_client_1.ResourceClient {
|
|
|
112
152
|
* ```
|
|
113
153
|
*/
|
|
114
154
|
listKeys(options = {}) {
|
|
115
|
-
(0,
|
|
116
|
-
limit: ow_1.default.optional.number.not.negative,
|
|
117
|
-
exclusiveStartKey: ow_1.default.optional.string,
|
|
118
|
-
collection: ow_1.default.optional.string,
|
|
119
|
-
prefix: ow_1.default.optional.string,
|
|
120
|
-
signature: ow_1.default.optional.string,
|
|
121
|
-
}));
|
|
155
|
+
const parsed = (0, utils_1.parseArgument)(options, listKeysOptionsSchema, 'KeyValueClientListKeysOptions');
|
|
122
156
|
const getPaginatedList = async (kvsListOptions = {}) => {
|
|
123
157
|
const response = await this.httpClient.call({
|
|
124
158
|
url: this._url('keys'),
|
|
@@ -128,17 +162,19 @@ class KeyValueStoreClient extends resource_client_1.ResourceClient {
|
|
|
128
162
|
});
|
|
129
163
|
return (0, utils_1.cast)((0, utils_1.parseDateFields)((0, utils_1.pluckData)(response.data)));
|
|
130
164
|
};
|
|
131
|
-
const paginatedListPromise = getPaginatedList(
|
|
165
|
+
const paginatedListPromise = getPaginatedList(parsed);
|
|
132
166
|
async function* asyncGenerator() {
|
|
133
167
|
let currentPage = await paginatedListPromise;
|
|
134
168
|
yield* currentPage.items;
|
|
135
|
-
let remainingItems =
|
|
169
|
+
let remainingItems = parsed.limit ? parsed.limit - currentPage.items.length : undefined;
|
|
136
170
|
while (currentPage.items.length > 0 && // Continue only if at least some items were returned in the last page.
|
|
137
|
-
|
|
171
|
+
// Continue only if there is some next key. The API answers with `null` on the last page and
|
|
172
|
+
// omits the field entirely for a listing that was not truncated.
|
|
173
|
+
currentPage.nextExclusiveStartKey != null &&
|
|
138
174
|
(remainingItems === undefined || remainingItems > 0) // Continue only if the limit was not exceeded.
|
|
139
175
|
) {
|
|
140
176
|
const newOptions = {
|
|
141
|
-
...
|
|
177
|
+
...parsed,
|
|
142
178
|
limit: remainingItems,
|
|
143
179
|
exclusiveStartKey: currentPage.nextExclusiveStartKey,
|
|
144
180
|
};
|
|
@@ -172,9 +208,9 @@ class KeyValueStoreClient extends resource_client_1.ResourceClient {
|
|
|
172
208
|
* @since Added in 2.14.0
|
|
173
209
|
*/
|
|
174
210
|
async getRecordPublicUrl(key) {
|
|
175
|
-
(0,
|
|
211
|
+
(0, utils_1.parseArgument)(key, nonEmptyKeySchema);
|
|
176
212
|
const store = await this.get();
|
|
177
|
-
const recordPublicUrl = new URL(this._publicUrl(
|
|
213
|
+
const recordPublicUrl = new URL(this._publicUrl(['records', key]));
|
|
178
214
|
if (store?.urlSigningSecretKey) {
|
|
179
215
|
const signature = await (0, utilities_1.createHmacSignatureAsync)(store.urlSigningSecretKey, key);
|
|
180
216
|
recordPublicUrl.searchParams.append('signature', signature);
|
|
@@ -205,15 +241,9 @@ class KeyValueStoreClient extends resource_client_1.ResourceClient {
|
|
|
205
241
|
* @since Added in 2.13.0
|
|
206
242
|
*/
|
|
207
243
|
async createKeysPublicUrl(options = {}) {
|
|
208
|
-
(0,
|
|
209
|
-
limit: ow_1.default.optional.number.not.negative,
|
|
210
|
-
exclusiveStartKey: ow_1.default.optional.string,
|
|
211
|
-
collection: ow_1.default.optional.string,
|
|
212
|
-
prefix: ow_1.default.optional.string,
|
|
213
|
-
expiresInSecs: ow_1.default.optional.number,
|
|
214
|
-
}));
|
|
244
|
+
const parsed = (0, utils_1.parseArgument)(options, createKeysPublicUrlOptionsSchema, 'KeyValueClientCreateKeysUrlOptions');
|
|
215
245
|
const store = await this.get();
|
|
216
|
-
const { expiresInSecs, ...queryOptions } =
|
|
246
|
+
const { expiresInSecs, ...queryOptions } = parsed;
|
|
217
247
|
let createdPublicKeysUrl = new URL(this._publicUrl('keys'));
|
|
218
248
|
if (store?.urlSigningSecretKey) {
|
|
219
249
|
const signature = await (0, utilities_1.createStorageContentSignatureAsync)({
|
|
@@ -246,7 +276,7 @@ class KeyValueStoreClient extends resource_client_1.ResourceClient {
|
|
|
246
276
|
*/
|
|
247
277
|
async recordExists(key) {
|
|
248
278
|
const requestOpts = {
|
|
249
|
-
url: this._url(
|
|
279
|
+
url: this._url(['records', key]),
|
|
250
280
|
method: 'HEAD',
|
|
251
281
|
params: this._params(),
|
|
252
282
|
};
|
|
@@ -260,32 +290,27 @@ class KeyValueStoreClient extends resource_client_1.ResourceClient {
|
|
|
260
290
|
return false;
|
|
261
291
|
}
|
|
262
292
|
async getRecord(key, options = {}) {
|
|
263
|
-
(0,
|
|
264
|
-
(0,
|
|
265
|
-
|
|
266
|
-
stream: ow_1.default.optional.boolean,
|
|
267
|
-
disableRedirect: ow_1.default.optional.boolean,
|
|
268
|
-
signature: ow_1.default.optional.string,
|
|
269
|
-
}));
|
|
270
|
-
if (options.stream && !(0, utils_1.isNode)()) {
|
|
293
|
+
(0, utils_1.parseArgument)(key, keySchema);
|
|
294
|
+
const parsed = (0, utils_1.parseArgument)(options, getRecordOptionsSchema, 'KeyValueClientGetRecordOptions');
|
|
295
|
+
if (parsed.stream && !(0, utils_1.isNode)()) {
|
|
271
296
|
throw new Error('The stream option can only be used in Node.js environment.');
|
|
272
297
|
}
|
|
273
|
-
if ('disableRedirect' in
|
|
298
|
+
if ('disableRedirect' in parsed) {
|
|
274
299
|
log_1.default.deprecated('The disableRedirect option for getRecord() is deprecated. ' +
|
|
275
300
|
'It has no effect and will be removed in the following major release.');
|
|
276
301
|
}
|
|
277
302
|
const queryParams = { attachment: 'true' };
|
|
278
|
-
if (
|
|
279
|
-
queryParams.signature =
|
|
303
|
+
if (parsed.signature)
|
|
304
|
+
queryParams.signature = parsed.signature;
|
|
280
305
|
const requestOpts = {
|
|
281
|
-
url: this._url(
|
|
306
|
+
url: this._url(['records', key]),
|
|
282
307
|
method: 'GET',
|
|
283
308
|
params: this._params(queryParams),
|
|
284
309
|
timeout: resource_client_1.DEFAULT_TIMEOUT_MILLIS,
|
|
285
310
|
};
|
|
286
|
-
if (
|
|
311
|
+
if (parsed.buffer)
|
|
287
312
|
requestOpts.forceBuffer = true;
|
|
288
|
-
if (
|
|
313
|
+
if (parsed.stream)
|
|
289
314
|
requestOpts.responseType = 'stream';
|
|
290
315
|
try {
|
|
291
316
|
const response = await this.httpClient.call(requestOpts);
|
|
@@ -348,18 +373,11 @@ class KeyValueStoreClient extends resource_client_1.ResourceClient {
|
|
|
348
373
|
* ```
|
|
349
374
|
*/
|
|
350
375
|
async setRecord(record, options = {}) {
|
|
351
|
-
(0,
|
|
352
|
-
|
|
353
|
-
value: ow_1.default.any(ow_1.default.null, ow_1.default.string, ow_1.default.number, ow_1.default.object, ow_1.default.boolean),
|
|
354
|
-
contentType: ow_1.default.optional.string.nonEmpty,
|
|
355
|
-
}));
|
|
356
|
-
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
357
|
-
timeoutSecs: ow_1.default.optional.number,
|
|
358
|
-
doNotRetryTimeouts: ow_1.default.optional.boolean,
|
|
359
|
-
}));
|
|
376
|
+
(0, utils_1.parseArgument)(record, recordSchema);
|
|
377
|
+
const parsed = (0, utils_1.parseArgument)(options, recordOptionsSchema, 'KeyValueStoreRecordOptions');
|
|
360
378
|
const { key } = record;
|
|
361
379
|
let { value, contentType } = record;
|
|
362
|
-
const { timeoutSecs, doNotRetryTimeouts } =
|
|
380
|
+
const { timeoutSecs, doNotRetryTimeouts } = parsed;
|
|
363
381
|
const isValueStreamOrBuffer = (0, utils_1.isStream)(value) || (0, utils_1.isBuffer)(value);
|
|
364
382
|
// To allow saving Objects to JSON without providing content type
|
|
365
383
|
if (!contentType) {
|
|
@@ -381,7 +399,7 @@ class KeyValueStoreClient extends resource_client_1.ResourceClient {
|
|
|
381
399
|
}
|
|
382
400
|
}
|
|
383
401
|
const uploadOpts = {
|
|
384
|
-
url: this._url(
|
|
402
|
+
url: this._url(['records', key]),
|
|
385
403
|
method: 'PUT',
|
|
386
404
|
params: this._params(),
|
|
387
405
|
data: value,
|
|
@@ -403,9 +421,9 @@ class KeyValueStoreClient extends resource_client_1.ResourceClient {
|
|
|
403
421
|
* ```
|
|
404
422
|
*/
|
|
405
423
|
async deleteRecord(key) {
|
|
406
|
-
(0,
|
|
424
|
+
(0, utils_1.parseArgument)(key, keySchema);
|
|
407
425
|
await this.httpClient.call({
|
|
408
|
-
url: this._url(
|
|
426
|
+
url: this._url(['records', key]),
|
|
409
427
|
method: 'DELETE',
|
|
410
428
|
params: this._params(),
|
|
411
429
|
timeout: resource_client_1.SMALL_TIMEOUT_MILLIS,
|