apify-client 3.0.0-beta.7 → 3.0.0-beta.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -2
- package/dist/apify_api_error.d.ts +13 -27
- package/dist/apify_api_error.js +33 -23
- package/dist/apify_client.d.ts +125 -10
- package/dist/apify_client.js +133 -48
- package/dist/apify_client_error.d.ts +10 -0
- package/dist/apify_client_error.js +10 -0
- package/dist/base/api_client.d.ts +7 -7
- package/dist/base/api_client.js +36 -36
- package/dist/base/resource_client.d.ts +21 -8
- package/dist/base/resource_client.js +67 -33
- package/dist/base/resource_collection_client.d.ts +12 -4
- package/dist/base/resource_collection_client.js +34 -11
- package/dist/body_parser.d.ts +3 -4
- package/dist/body_parser.js +19 -26
- package/dist/bundle.js +27 -22
- package/dist/bundle.js.map +1 -1
- package/dist/generated/api.d.ts +918 -372
- package/dist/generated/schemas.d.ts +889 -871
- package/dist/generated/schemas.js +234 -209
- package/dist/http_clients/axios.d.ts +60 -0
- package/dist/http_clients/axios.js +117 -0
- package/dist/http_clients/base.d.ts +331 -0
- package/dist/http_clients/base.js +563 -0
- package/dist/http_clients/index.d.ts +4 -0
- package/dist/http_clients/index.js +2 -0
- package/dist/http_compressors/base.d.ts +40 -0
- package/dist/http_compressors/base.js +9 -0
- package/dist/http_compressors/brotli.d.ts +33 -0
- package/dist/http_compressors/brotli.js +39 -0
- package/dist/http_compressors/gzip.d.ts +33 -0
- package/dist/http_compressors/gzip.js +39 -0
- package/dist/http_compressors/index.d.ts +6 -0
- package/dist/http_compressors/index.js +2 -0
- package/dist/http_compressors/resolve.d.ts +23 -0
- package/dist/http_compressors/resolve.js +27 -0
- package/dist/index.d.ts +5 -1
- package/dist/index.js +4 -1
- package/dist/invalid_response_body_error.d.ts +14 -0
- package/dist/invalid_response_body_error.js +18 -0
- package/dist/models.d.ts +167 -194
- package/dist/models.js +6 -4
- package/dist/resource_clients/actor.d.ts +74 -34
- package/dist/resource_clients/actor.js +87 -56
- package/dist/resource_clients/actor_collection.d.ts +18 -12
- package/dist/resource_clients/actor_collection.js +11 -3
- package/dist/resource_clients/actor_env_var.d.ts +11 -4
- package/dist/resource_clients/actor_env_var.js +16 -6
- package/dist/resource_clients/actor_env_var_collection.d.ts +9 -4
- package/dist/resource_clients/actor_env_var_collection.js +11 -4
- package/dist/resource_clients/actor_version.d.ts +20 -5
- package/dist/resource_clients/actor_version.js +19 -9
- package/dist/resource_clients/actor_version_collection.d.ts +11 -4
- package/dist/resource_clients/actor_version_collection.js +13 -4
- package/dist/resource_clients/build.d.ts +20 -7
- package/dist/resource_clients/build.js +35 -15
- package/dist/resource_clients/build_collection.d.ts +7 -5
- package/dist/resource_clients/build_collection.js +4 -1
- package/dist/resource_clients/dataset.d.ts +81 -36
- package/dist/resource_clients/dataset.js +109 -120
- package/dist/resource_clients/dataset_collection.d.ts +10 -6
- package/dist/resource_clients/dataset_collection.js +13 -2
- package/dist/resource_clients/key_value_store.d.ts +101 -23
- package/dist/resource_clients/key_value_store.js +121 -45
- package/dist/resource_clients/key_value_store_collection.d.ts +10 -6
- package/dist/resource_clients/key_value_store_collection.js +13 -2
- package/dist/resource_clients/log.d.ts +40 -21
- package/dist/resource_clients/log.js +144 -66
- package/dist/resource_clients/request_queue.d.ts +76 -31
- package/dist/resource_clients/request_queue.js +178 -93
- package/dist/resource_clients/request_queue_collection.d.ts +9 -5
- package/dist/resource_clients/request_queue_collection.js +9 -3
- package/dist/resource_clients/run.d.ts +119 -28
- package/dist/resource_clients/run.js +190 -49
- package/dist/resource_clients/run_collection.d.ts +5 -3
- package/dist/resource_clients/run_collection.js +4 -1
- package/dist/resource_clients/schedule.d.ts +16 -7
- package/dist/resource_clients/schedule.js +30 -22
- package/dist/resource_clients/schedule_collection.d.ts +6 -2
- package/dist/resource_clients/schedule_collection.js +9 -3
- package/dist/resource_clients/store_collection.d.ts +6 -4
- package/dist/resource_clients/store_collection.js +4 -1
- package/dist/resource_clients/task.d.ts +36 -13
- package/dist/resource_clients/task.js +79 -46
- package/dist/resource_clients/task_collection.d.ts +9 -5
- package/dist/resource_clients/task_collection.js +9 -3
- package/dist/resource_clients/user.d.ts +19 -10
- package/dist/resource_clients/user.js +42 -38
- package/dist/resource_clients/webhook.d.ts +17 -8
- package/dist/resource_clients/webhook.js +30 -22
- package/dist/resource_clients/webhook_collection.d.ts +8 -4
- package/dist/resource_clients/webhook_collection.js +9 -3
- package/dist/resource_clients/webhook_dispatch.d.ts +4 -1
- package/dist/resource_clients/webhook_dispatch.js +7 -2
- package/dist/resource_clients/webhook_dispatch_collection.d.ts +3 -1
- package/dist/resource_clients/webhook_dispatch_collection.js +4 -1
- package/dist/response_validation_error.d.ts +2 -1
- package/dist/response_validation_error.js +2 -1
- package/dist/runtime/node.d.ts +6 -0
- package/dist/runtime/node.js +49 -0
- package/dist/runtime/types.d.ts +52 -0
- package/dist/runtime/types.js +1 -0
- package/dist/runtime/web.d.ts +5 -0
- package/dist/runtime/web.js +18 -0
- package/dist/schemas.d.ts +201 -4
- package/dist/schemas.js +19 -4
- package/dist/statistics.d.ts +1 -6
- package/dist/statistics.js +2 -2
- package/dist/timeouts.d.ts +84 -0
- package/dist/timeouts.js +37 -0
- package/dist/timezones.d.ts +1 -1
- package/dist/utils.d.ts +88 -57
- package/dist/utils.js +219 -157
- package/dist/wait_for_resources.d.ts +8 -0
- package/dist/wait_for_resources.js +39 -0
- package/package.json +32 -29
- package/dist/http_client.d.ts +0 -88
- package/dist/http_client.js +0 -255
- package/dist/interceptors.d.ts +0 -19
- package/dist/interceptors.js +0 -105
|
@@ -1,9 +1,23 @@
|
|
|
1
|
+
import type { Readable } from 'node:stream';
|
|
1
2
|
import type { STORAGE_GENERAL_ACCESS } from '@apify/consts';
|
|
2
3
|
import type { ApiClientSubResourceOptions } from '../base/api_client.js';
|
|
3
4
|
import { ResourceClient } from '../base/resource_client.js';
|
|
4
|
-
import type {
|
|
5
|
+
import type { DatasetResource, DatasetStatistics } from '../models.js';
|
|
6
|
+
import type { TimeoutOptions } from '../timeouts.js';
|
|
5
7
|
import type { PaginatedIterator, PaginationOptions } from '../utils.js';
|
|
6
|
-
|
|
8
|
+
/**
|
|
9
|
+
* Supported formats for downloading dataset items.
|
|
10
|
+
*/
|
|
11
|
+
export declare enum DownloadItemsFormat {
|
|
12
|
+
JSON = "json",
|
|
13
|
+
JSONL = "jsonl",
|
|
14
|
+
XML = "xml",
|
|
15
|
+
HTML = "html",
|
|
16
|
+
CSV = "csv",
|
|
17
|
+
XLSX = "xlsx",
|
|
18
|
+
RSS = "rss"
|
|
19
|
+
}
|
|
20
|
+
export type { DatasetResource, DatasetFieldStatistics, DatasetStatistics, DatasetStats } from '../models.js';
|
|
7
21
|
/**
|
|
8
22
|
* Client for managing a specific Dataset.
|
|
9
23
|
*
|
|
@@ -34,6 +48,7 @@ export type { Dataset, DatasetStatistics, DatasetStats, FieldStatistics } from '
|
|
|
34
48
|
* @see https://docs.apify.com/platform/storage/dataset
|
|
35
49
|
*/
|
|
36
50
|
export declare class DatasetClient<Data extends Record<string | number, any> = Record<string | number, unknown>> extends ResourceClient {
|
|
51
|
+
#private;
|
|
37
52
|
/**
|
|
38
53
|
* @hidden
|
|
39
54
|
*/
|
|
@@ -41,24 +56,30 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
41
56
|
/**
|
|
42
57
|
* Gets the dataset object from the Apify API.
|
|
43
58
|
*
|
|
59
|
+
* @param options - Request options
|
|
60
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
44
61
|
* @returns The Dataset object, or `undefined` if it does not exist
|
|
45
62
|
* @see https://docs.apify.com/api/v2/dataset-get
|
|
46
63
|
*/
|
|
47
|
-
get(): Promise<
|
|
64
|
+
get(options?: TimeoutOptions): Promise<DatasetResource | undefined>;
|
|
48
65
|
/**
|
|
49
66
|
* Updates the dataset with specified fields.
|
|
50
67
|
*
|
|
51
68
|
* @param newFields - Fields to update in the dataset
|
|
69
|
+
* @param options - Request options
|
|
70
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
52
71
|
* @returns The updated Dataset object
|
|
53
72
|
* @see https://docs.apify.com/api/v2/dataset-put
|
|
54
73
|
*/
|
|
55
|
-
update(newFields: DatasetClientUpdateOptions): Promise<
|
|
74
|
+
update(newFields: DatasetClientUpdateOptions, options?: TimeoutOptions): Promise<DatasetResource>;
|
|
56
75
|
/**
|
|
57
76
|
* Deletes the dataset.
|
|
58
77
|
*
|
|
78
|
+
* @param options - Request options
|
|
79
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
59
80
|
* @see https://docs.apify.com/api/v2/dataset-delete
|
|
60
81
|
*/
|
|
61
|
-
delete(): Promise<void>;
|
|
82
|
+
delete(options?: TimeoutOptions): Promise<void>;
|
|
62
83
|
/**
|
|
63
84
|
* Lists items in the dataset.
|
|
64
85
|
*
|
|
@@ -77,8 +98,9 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
77
98
|
* @param options.skipEmpty - If `true`, skips empty items. Default is `false`.
|
|
78
99
|
* @param options.skipHidden - If `true`, skips hidden fields (fields starting with `#`). Default is `false`.
|
|
79
100
|
* @param options.flatten - Array of field names to flatten. Nested objects are converted to dot notation (e.g., `obj.field`).
|
|
80
|
-
* @param options.unwind -
|
|
101
|
+
* @param options.unwind - Array of field names to unwind. Each array value creates a separate item.
|
|
81
102
|
* @param options.view - Name of a predefined view to use for field selection.
|
|
103
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'long'`.
|
|
82
104
|
* @returns A paginated list with `items`, `total` count, `offset`, `count`, and `limit`
|
|
83
105
|
* @see https://docs.apify.com/api/v2/dataset-items-get
|
|
84
106
|
*
|
|
@@ -111,6 +133,11 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
111
133
|
* dataset items, this method returns the items serialized to the provided format
|
|
112
134
|
* (JSON, CSV, Excel, etc.) as a Buffer. Useful for exporting data for further processing.
|
|
113
135
|
*
|
|
136
|
+
* In Node.js you can use the `stream` option to get a Readable stream, so a large export is processed in chunks
|
|
137
|
+
* without holding all of it in memory. The stream holds its connection open until you consume it to the end or
|
|
138
|
+
* destroy it. Iterating it with `for await` or passing it to `pipeline()` from `node:stream/promises` does either
|
|
139
|
+
* one for you.
|
|
140
|
+
*
|
|
114
141
|
* @param format - Output format: `'json'`, `'jsonl'`, `'csv'`, `'xlsx'`, `'xml'`, `'rss'`, or `'html'`
|
|
115
142
|
* @param options - Download and formatting options (extends all options from {@link listItems})
|
|
116
143
|
* @param options.attachment - If `true`, the response will have `Content-Disposition: attachment` header.
|
|
@@ -121,7 +148,12 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
121
148
|
* @param options.xmlRow - Name of the XML element for each item. Default is `'item'`.
|
|
122
149
|
* @param options.fields - Array of field names to include in the export.
|
|
123
150
|
* @param options.omit - Array of field names to exclude from the export.
|
|
124
|
-
* @
|
|
151
|
+
* @param options.stream - If `true`, the items are returned as a Readable stream. Node.js only.
|
|
152
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`. With `stream`, it bounds the
|
|
153
|
+
* wait for the API to start responding; once the stream is returned, it only limits inactivity: the stream errors
|
|
154
|
+
* if no data arrives for that long. To stop the download early or cap its total duration, destroy the stream or
|
|
155
|
+
* abort `signal`.
|
|
156
|
+
* @returns Buffer containing the serialized data in the specified format, or a Readable stream of it with `stream`
|
|
125
157
|
* @see https://docs.apify.com/api/v2/dataset-items-get
|
|
126
158
|
*
|
|
127
159
|
* @example
|
|
@@ -142,19 +174,34 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
142
174
|
* xmlRoot: 'products',
|
|
143
175
|
* xmlRow: 'product'
|
|
144
176
|
* });
|
|
177
|
+
*
|
|
178
|
+
* // Stream a CSV export straight to a file
|
|
179
|
+
* const { createWriteStream } = require('node:fs');
|
|
180
|
+
* const { pipeline } = require('node:stream/promises');
|
|
181
|
+
* const stream = await client.dataset('my-dataset').downloadItems('csv', { stream: true });
|
|
182
|
+
* await pipeline(stream, createWriteStream('output.csv'));
|
|
145
183
|
* ```
|
|
146
184
|
*/
|
|
147
|
-
downloadItems(format: DownloadItemsFormat
|
|
185
|
+
downloadItems(format: `${DownloadItemsFormat}`, options?: DatasetClientDownloadItemsOptions & {
|
|
186
|
+
stream?: false;
|
|
187
|
+
}): Promise<Buffer>;
|
|
188
|
+
downloadItems(format: `${DownloadItemsFormat}`, options: DatasetClientDownloadItemsOptions & {
|
|
189
|
+
stream: true;
|
|
190
|
+
}): Promise<Readable>;
|
|
191
|
+
downloadItems(format: `${DownloadItemsFormat}`, options?: DatasetClientDownloadItemsOptions): Promise<Buffer | Readable>;
|
|
148
192
|
/**
|
|
149
193
|
* Stores one or more items into the dataset.
|
|
150
194
|
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
195
|
+
* Each item will be stored as a separate record in the dataset. Objects are automatically
|
|
196
|
+
* serialized to JSON. If you provide an array, all items will be stored in order. This method
|
|
197
|
+
* is idempotent - calling it multiple times with the same data will not create duplicates, but
|
|
198
|
+
* will append items each time.
|
|
155
199
|
*
|
|
156
|
-
* @param items - A single item
|
|
157
|
-
*
|
|
200
|
+
* @param items - A single item, an array of items, or a string that is the JSON serialization of
|
|
201
|
+
* either - the API only accepts an object or an array of objects, so a plain string
|
|
202
|
+
* is not a valid item on its own.
|
|
203
|
+
* @param options - Request options
|
|
204
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
|
|
158
205
|
* @see https://docs.apify.com/api/v2/dataset-items-post
|
|
159
206
|
*
|
|
160
207
|
* @example
|
|
@@ -172,23 +219,22 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
172
219
|
* { url: 'https://test.com', title: 'Test' },
|
|
173
220
|
* { url: 'https://demo.com', title: 'Demo' }
|
|
174
221
|
* ]);
|
|
175
|
-
*
|
|
176
|
-
* // Store string items
|
|
177
|
-
* await client.dataset('my-dataset').pushItems(['item1', 'item2', 'item3']);
|
|
178
222
|
* ```
|
|
179
223
|
*/
|
|
180
|
-
pushItems(items: Data | Data[] | string
|
|
224
|
+
pushItems(items: Data | Data[] | string, options?: TimeoutOptions): Promise<void>;
|
|
181
225
|
/**
|
|
182
226
|
* Gets statistical information about the dataset.
|
|
183
227
|
*
|
|
184
228
|
* Returns statistics for each field in the dataset, including information about
|
|
185
229
|
* data types, null counts, and value ranges.
|
|
186
230
|
*
|
|
187
|
-
* @
|
|
231
|
+
* @param options - Request options
|
|
232
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
233
|
+
* @returns Dataset statistics
|
|
188
234
|
* @see https://docs.apify.com/api/v2/dataset-statistics-get
|
|
189
235
|
* @since Added in 2.11.2
|
|
190
236
|
*/
|
|
191
|
-
getStatistics(): Promise<DatasetStatistics
|
|
237
|
+
getStatistics(options?: TimeoutOptions): Promise<DatasetStatistics>;
|
|
192
238
|
/**
|
|
193
239
|
* Generates a public URL for accessing dataset items.
|
|
194
240
|
*
|
|
@@ -199,8 +245,10 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
199
245
|
* @param options - URL generation options (extends all options from {@link listItems})
|
|
200
246
|
* @param options.expiresInSecs - Number of seconds until the signed URL expires. If omitted, the URL never expires.
|
|
201
247
|
* @param options.fields - Array of field names to include in the response.
|
|
248
|
+
* @param options.format - Format of the items served by the URL. Defaults to `json`.
|
|
202
249
|
* @param options.limit - Maximum number of items to return.
|
|
203
250
|
* @param options.offset - Number of items to skip.
|
|
251
|
+
* @param options.timeoutSecs - Timeout for the API request that fetches the dataset. Default is `'long'`.
|
|
204
252
|
* @returns A public URL string for accessing the dataset items
|
|
205
253
|
*
|
|
206
254
|
* @example
|
|
@@ -222,7 +270,6 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
222
270
|
* @since Added in 2.13.0
|
|
223
271
|
*/
|
|
224
272
|
createItemsPublicUrl(options?: DatasetClientCreateItemsUrlOptions): Promise<string>;
|
|
225
|
-
private _createPaginationList;
|
|
226
273
|
}
|
|
227
274
|
/**
|
|
228
275
|
* Options for updating a dataset.
|
|
@@ -244,7 +291,7 @@ export interface DatasetClientUpdateOptions {
|
|
|
244
291
|
* Provides various filtering, pagination, and transformation options to customize
|
|
245
292
|
* the output format and content of retrieved items.
|
|
246
293
|
*/
|
|
247
|
-
export interface DatasetClientListItemOptions extends PaginationOptions {
|
|
294
|
+
export interface DatasetClientListItemOptions extends PaginationOptions, TimeoutOptions {
|
|
248
295
|
clean?: boolean;
|
|
249
296
|
desc?: boolean;
|
|
250
297
|
/**
|
|
@@ -255,7 +302,7 @@ export interface DatasetClientListItemOptions extends PaginationOptions {
|
|
|
255
302
|
omit?: string[];
|
|
256
303
|
skipEmpty?: boolean;
|
|
257
304
|
skipHidden?: boolean;
|
|
258
|
-
unwind?: string
|
|
305
|
+
unwind?: string[];
|
|
259
306
|
/**
|
|
260
307
|
* @since Added in 2.2.0
|
|
261
308
|
*/
|
|
@@ -268,25 +315,18 @@ export interface DatasetClientListItemOptions extends PaginationOptions {
|
|
|
268
315
|
/**
|
|
269
316
|
* Options for creating a public URL to access dataset items.
|
|
270
317
|
*
|
|
271
|
-
* Extends {@link DatasetClientListItemOptions} with URL expiration control, minus `chunkSize` (it only
|
|
318
|
+
* Extends {@link DatasetClientListItemOptions} with output format and URL expiration control, minus `chunkSize` (it only
|
|
272
319
|
* sizes a client-side iteration) and `signature` (this method produces one).
|
|
273
320
|
* @since Added in 2.16.0
|
|
274
321
|
*/
|
|
275
322
|
export interface DatasetClientCreateItemsUrlOptions extends Omit<DatasetClientListItemOptions, 'chunkSize' | 'signature'> {
|
|
323
|
+
/**
|
|
324
|
+
* Format of the items served by the URL. Defaults to `json`.
|
|
325
|
+
* @since Added in 2.26.0
|
|
326
|
+
*/
|
|
327
|
+
format?: `${DownloadItemsFormat}`;
|
|
276
328
|
expiresInSecs?: number;
|
|
277
329
|
}
|
|
278
|
-
/**
|
|
279
|
-
* Supported formats for downloading dataset items.
|
|
280
|
-
*/
|
|
281
|
-
export declare enum DownloadItemsFormat {
|
|
282
|
-
JSON = "json",
|
|
283
|
-
JSONL = "jsonl",
|
|
284
|
-
XML = "xml",
|
|
285
|
-
HTML = "html",
|
|
286
|
-
CSV = "csv",
|
|
287
|
-
XLSX = "xlsx",
|
|
288
|
-
RSS = "rss"
|
|
289
|
-
}
|
|
290
330
|
/**
|
|
291
331
|
* Options for downloading dataset items in a specific format.
|
|
292
332
|
*
|
|
@@ -300,4 +340,9 @@ export interface DatasetClientDownloadItemsOptions extends Omit<DatasetClientLis
|
|
|
300
340
|
skipHeaderRow?: boolean;
|
|
301
341
|
xmlRoot?: string;
|
|
302
342
|
xmlRow?: string;
|
|
343
|
+
/**
|
|
344
|
+
* If `true`, the items are returned as a Readable stream. Node.js only.
|
|
345
|
+
* @default false
|
|
346
|
+
*/
|
|
347
|
+
stream?: boolean;
|
|
303
348
|
}
|
|
@@ -1,8 +1,24 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { createStorageContentSignatureAsync } from '@apify/utilities';
|
|
3
|
-
import {
|
|
3
|
+
import { ResourceClient } from '../base/resource_client.js';
|
|
4
4
|
import * as schemas from '../schemas.js';
|
|
5
|
-
import {
|
|
5
|
+
import { timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
|
|
6
|
+
import { runtime } from '#runtime';
|
|
7
|
+
import { anyObjectSchema, applyQueryParamsToUrl, cast, isNonArrayObject, paginationOptionsShape, parseArgument, parseResponse, SCANNED_COUNT, } from '../utils.js';
|
|
8
|
+
/**
|
|
9
|
+
* Supported formats for downloading dataset items.
|
|
10
|
+
*/
|
|
11
|
+
export var DownloadItemsFormat;
|
|
12
|
+
(function (DownloadItemsFormat) {
|
|
13
|
+
DownloadItemsFormat["JSON"] = "json";
|
|
14
|
+
DownloadItemsFormat["JSONL"] = "jsonl";
|
|
15
|
+
DownloadItemsFormat["XML"] = "xml";
|
|
16
|
+
DownloadItemsFormat["HTML"] = "html";
|
|
17
|
+
DownloadItemsFormat["CSV"] = "csv";
|
|
18
|
+
DownloadItemsFormat["XLSX"] = "xlsx";
|
|
19
|
+
DownloadItemsFormat["RSS"] = "rss";
|
|
20
|
+
})(DownloadItemsFormat || (DownloadItemsFormat = {}));
|
|
21
|
+
const itemFormatSchema = z.enum(DownloadItemsFormat);
|
|
6
22
|
// A predicate, not a `z.object()` arm, which would walk and copy every key of every pushed item.
|
|
7
23
|
const itemSchema = z.custom(isNonArrayObject, 'Expected an object');
|
|
8
24
|
const listItemsOptionsSchema = z.strictObject({
|
|
@@ -14,9 +30,10 @@ const listItemsOptionsSchema = z.strictObject({
|
|
|
14
30
|
...paginationOptionsShape,
|
|
15
31
|
skipEmpty: z.boolean().optional(),
|
|
16
32
|
skipHidden: z.boolean().optional(),
|
|
17
|
-
unwind: z.
|
|
33
|
+
unwind: z.array(z.string()).optional(),
|
|
18
34
|
view: z.string().optional(),
|
|
19
35
|
signature: z.string().optional(),
|
|
36
|
+
...timeoutOptionsShape,
|
|
20
37
|
});
|
|
21
38
|
// Spells out `limit` / `offset` instead of spreading `paginationOptionsShape`: a download is one
|
|
22
39
|
// request, so `chunkSize` has nothing to size. The options type omits it to match.
|
|
@@ -34,28 +51,33 @@ const downloadItemsOptionsSchema = z.strictObject({
|
|
|
34
51
|
skipEmpty: z.boolean().optional(),
|
|
35
52
|
skipHeaderRow: z.boolean().optional(),
|
|
36
53
|
skipHidden: z.boolean().optional(),
|
|
37
|
-
unwind: z.
|
|
54
|
+
unwind: z.array(z.string()).optional(),
|
|
38
55
|
view: z.string().optional(),
|
|
39
56
|
xmlRoot: z.string().optional(),
|
|
40
57
|
xmlRow: z.string().optional(),
|
|
41
58
|
signature: z.string().optional(),
|
|
59
|
+
stream: z.boolean().optional(),
|
|
60
|
+
...timeoutOptionsShape,
|
|
42
61
|
});
|
|
43
|
-
const pushItemsSchema = z.union([itemSchema, z.string(), z.array(
|
|
44
|
-
//
|
|
45
|
-
// `signature` (which this method produces) are left out. The options type
|
|
62
|
+
const pushItemsSchema = z.union([itemSchema, z.string(), z.array(itemSchema)]);
|
|
63
|
+
// Apart from `timeoutSecs`, `signal` and `expiresInSecs`, every option becomes a query parameter of the generated
|
|
64
|
+
// URL, so `chunkSize` (client-side only) and `signature` (which this method produces) are left out. The options type
|
|
65
|
+
// omits both to match.
|
|
46
66
|
const createItemsPublicUrlOptionsSchema = z.strictObject({
|
|
47
67
|
clean: z.boolean().optional(),
|
|
48
68
|
desc: z.boolean().optional(),
|
|
49
69
|
flatten: z.array(z.string()).optional(),
|
|
50
70
|
fields: z.array(z.string()).optional(),
|
|
71
|
+
format: itemFormatSchema.optional(),
|
|
51
72
|
omit: z.array(z.string()).optional(),
|
|
52
73
|
limit: z.number().min(0).optional(),
|
|
53
74
|
offset: z.number().min(0).optional(),
|
|
54
75
|
skipEmpty: z.boolean().optional(),
|
|
55
76
|
skipHidden: z.boolean().optional(),
|
|
56
|
-
unwind: z.
|
|
77
|
+
unwind: z.array(z.string()).optional(),
|
|
57
78
|
view: z.string().optional(),
|
|
58
79
|
expiresInSecs: z.number().optional(),
|
|
80
|
+
...timeoutOptionsShape,
|
|
59
81
|
});
|
|
60
82
|
/**
|
|
61
83
|
* Client for managing a specific Dataset.
|
|
@@ -99,30 +121,39 @@ export class DatasetClient extends ResourceClient {
|
|
|
99
121
|
/**
|
|
100
122
|
* Gets the dataset object from the Apify API.
|
|
101
123
|
*
|
|
124
|
+
* @param options - Request options
|
|
125
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
102
126
|
* @returns The Dataset object, or `undefined` if it does not exist
|
|
103
127
|
* @see https://docs.apify.com/api/v2/dataset-get
|
|
104
128
|
*/
|
|
105
|
-
async get() {
|
|
106
|
-
|
|
129
|
+
async get(options = {}) {
|
|
130
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
131
|
+
return this.getResource(schemas.DatasetResource(), {}, timeoutSecs, signal);
|
|
107
132
|
}
|
|
108
133
|
/**
|
|
109
134
|
* Updates the dataset with specified fields.
|
|
110
135
|
*
|
|
111
136
|
* @param newFields - Fields to update in the dataset
|
|
137
|
+
* @param options - Request options
|
|
138
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
112
139
|
* @returns The updated Dataset object
|
|
113
140
|
* @see https://docs.apify.com/api/v2/dataset-put
|
|
114
141
|
*/
|
|
115
|
-
async update(newFields) {
|
|
142
|
+
async update(newFields, options = {}) {
|
|
116
143
|
parseArgument(newFields, anyObjectSchema);
|
|
117
|
-
|
|
144
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
145
|
+
return this.updateResource(schemas.DatasetResource(), newFields, timeoutSecs, signal);
|
|
118
146
|
}
|
|
119
147
|
/**
|
|
120
148
|
* Deletes the dataset.
|
|
121
149
|
*
|
|
150
|
+
* @param options - Request options
|
|
151
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
122
152
|
* @see https://docs.apify.com/api/v2/dataset-delete
|
|
123
153
|
*/
|
|
124
|
-
async delete() {
|
|
125
|
-
|
|
154
|
+
async delete(options = {}) {
|
|
155
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
156
|
+
return this.deleteResource(timeoutSecs, signal);
|
|
126
157
|
}
|
|
127
158
|
/**
|
|
128
159
|
* Lists items in the dataset.
|
|
@@ -142,8 +173,9 @@ export class DatasetClient extends ResourceClient {
|
|
|
142
173
|
* @param options.skipEmpty - If `true`, skips empty items. Default is `false`.
|
|
143
174
|
* @param options.skipHidden - If `true`, skips hidden fields (fields starting with `#`). Default is `false`.
|
|
144
175
|
* @param options.flatten - Array of field names to flatten. Nested objects are converted to dot notation (e.g., `obj.field`).
|
|
145
|
-
* @param options.unwind -
|
|
176
|
+
* @param options.unwind - Array of field names to unwind. Each array value creates a separate item.
|
|
146
177
|
* @param options.view - Name of a predefined view to use for field selection.
|
|
178
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'long'`.
|
|
147
179
|
* @returns A paginated list with `items`, `total` count, `offset`, `count`, and `limit`
|
|
148
180
|
* @see https://docs.apify.com/api/v2/dataset-items-get
|
|
149
181
|
*
|
|
@@ -169,83 +201,51 @@ export class DatasetClient extends ResourceClient {
|
|
|
169
201
|
* ```
|
|
170
202
|
*/
|
|
171
203
|
listItems(options = {}) {
|
|
172
|
-
const
|
|
204
|
+
const { timeoutSecs = 'long', signal, ...listOptions } = parseArgument(options, listItemsOptionsSchema, 'DatasetClientListItemOptions');
|
|
173
205
|
const fetchItems = async (datasetListOptions = {}) => {
|
|
174
206
|
const response = await this.httpClient.call({
|
|
175
|
-
url: this.
|
|
207
|
+
url: this.buildUrl('items'),
|
|
176
208
|
method: 'GET',
|
|
177
|
-
params: this.
|
|
178
|
-
|
|
209
|
+
params: this.buildParams(datasetListOptions),
|
|
210
|
+
timeoutSecs,
|
|
211
|
+
signal,
|
|
179
212
|
});
|
|
180
|
-
return this
|
|
213
|
+
return this.#createPaginationList(response, datasetListOptions.desc ?? false);
|
|
181
214
|
};
|
|
182
|
-
return this.
|
|
215
|
+
return this.listPaginatedFromCallback(fetchItems, listOptions);
|
|
183
216
|
}
|
|
184
|
-
/**
|
|
185
|
-
* Downloads dataset items in a specific format.
|
|
186
|
-
*
|
|
187
|
-
* Unlike {@link listItems} which returns a {@link PaginatedList} with an array of individual
|
|
188
|
-
* dataset items, this method returns the items serialized to the provided format
|
|
189
|
-
* (JSON, CSV, Excel, etc.) as a Buffer. Useful for exporting data for further processing.
|
|
190
|
-
*
|
|
191
|
-
* @param format - Output format: `'json'`, `'jsonl'`, `'csv'`, `'xlsx'`, `'xml'`, `'rss'`, or `'html'`
|
|
192
|
-
* @param options - Download and formatting options (extends all options from {@link listItems})
|
|
193
|
-
* @param options.attachment - If `true`, the response will have `Content-Disposition: attachment` header.
|
|
194
|
-
* @param options.bom - If `true`, adds UTF-8 BOM to the beginning of the file (useful for Excel compatibility).
|
|
195
|
-
* @param options.delimiter - CSV delimiter character. Default is `,` (comma).
|
|
196
|
-
* @param options.skipHeaderRow - If `true`, CSV export will not include the header row with field names.
|
|
197
|
-
* @param options.xmlRoot - Name of the root XML element. Default is `'items'`.
|
|
198
|
-
* @param options.xmlRow - Name of the XML element for each item. Default is `'item'`.
|
|
199
|
-
* @param options.fields - Array of field names to include in the export.
|
|
200
|
-
* @param options.omit - Array of field names to exclude from the export.
|
|
201
|
-
* @returns Buffer containing the serialized data in the specified format
|
|
202
|
-
* @see https://docs.apify.com/api/v2/dataset-items-get
|
|
203
|
-
*
|
|
204
|
-
* @example
|
|
205
|
-
* ```javascript
|
|
206
|
-
* // Download as CSV with BOM for Excel compatibility
|
|
207
|
-
* const csvBuffer = await client.dataset('my-dataset').downloadItems('csv', { bom: true });
|
|
208
|
-
* require('fs').writeFileSync('output.csv', csvBuffer);
|
|
209
|
-
*
|
|
210
|
-
* // Download as Excel with custom options
|
|
211
|
-
* const xlsxBuffer = await client.dataset('my-dataset').downloadItems('xlsx', {
|
|
212
|
-
* fields: ['url', 'title', 'price'],
|
|
213
|
-
* skipEmpty: true,
|
|
214
|
-
* limit: 1000
|
|
215
|
-
* });
|
|
216
|
-
*
|
|
217
|
-
* // Download as XML with custom element names
|
|
218
|
-
* const xmlBuffer = await client.dataset('my-dataset').downloadItems('xml', {
|
|
219
|
-
* xmlRoot: 'products',
|
|
220
|
-
* xmlRow: 'product'
|
|
221
|
-
* });
|
|
222
|
-
* ```
|
|
223
|
-
*/
|
|
224
217
|
async downloadItems(format, options = {}) {
|
|
225
218
|
parseArgument(format, itemFormatSchema);
|
|
226
|
-
const
|
|
219
|
+
const { timeoutSecs = 'long', signal, stream, ...query } = parseArgument(options, downloadItemsOptionsSchema, 'DatasetClientDownloadItemsOptions');
|
|
220
|
+
if (stream && !runtime.isNode) {
|
|
221
|
+
throw new Error('The stream option can only be used in Node.js environment.');
|
|
222
|
+
}
|
|
227
223
|
const { data } = await this.httpClient.call({
|
|
228
|
-
url: this.
|
|
224
|
+
url: this.buildUrl('items'),
|
|
229
225
|
method: 'GET',
|
|
230
|
-
params: this.
|
|
226
|
+
params: this.buildParams({
|
|
231
227
|
format,
|
|
232
|
-
...
|
|
228
|
+
...query,
|
|
233
229
|
}),
|
|
234
|
-
|
|
235
|
-
|
|
230
|
+
responseType: stream ? 'stream' : 'buffer',
|
|
231
|
+
timeoutSecs,
|
|
232
|
+
signal,
|
|
236
233
|
});
|
|
237
234
|
return cast(data);
|
|
238
235
|
}
|
|
239
236
|
/**
|
|
240
237
|
* Stores one or more items into the dataset.
|
|
241
238
|
*
|
|
242
|
-
*
|
|
243
|
-
*
|
|
244
|
-
*
|
|
245
|
-
*
|
|
239
|
+
* Each item will be stored as a separate record in the dataset. Objects are automatically
|
|
240
|
+
* serialized to JSON. If you provide an array, all items will be stored in order. This method
|
|
241
|
+
* is idempotent - calling it multiple times with the same data will not create duplicates, but
|
|
242
|
+
* will append items each time.
|
|
246
243
|
*
|
|
247
|
-
* @param items - A single item
|
|
248
|
-
*
|
|
244
|
+
* @param items - A single item, an array of items, or a string that is the JSON serialization of
|
|
245
|
+
* either - the API only accepts an object or an array of objects, so a plain string
|
|
246
|
+
* is not a valid item on its own.
|
|
247
|
+
* @param options - Request options
|
|
248
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
|
|
249
249
|
* @see https://docs.apify.com/api/v2/dataset-items-post
|
|
250
250
|
*
|
|
251
251
|
* @example
|
|
@@ -263,23 +263,22 @@ export class DatasetClient extends ResourceClient {
|
|
|
263
263
|
* { url: 'https://test.com', title: 'Test' },
|
|
264
264
|
* { url: 'https://demo.com', title: 'Demo' }
|
|
265
265
|
* ]);
|
|
266
|
-
*
|
|
267
|
-
* // Store string items
|
|
268
|
-
* await client.dataset('my-dataset').pushItems(['item1', 'item2', 'item3']);
|
|
269
266
|
* ```
|
|
270
267
|
*/
|
|
271
|
-
async pushItems(items) {
|
|
268
|
+
async pushItems(items, options = {}) {
|
|
272
269
|
parseArgument(items, pushItemsSchema);
|
|
270
|
+
const { timeoutSecs = 'medium', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
273
271
|
await this.httpClient.call({
|
|
274
|
-
url: this.
|
|
272
|
+
url: this.buildUrl('items'),
|
|
275
273
|
method: 'POST',
|
|
276
274
|
headers: {
|
|
277
275
|
'content-type': 'application/json; charset=utf-8',
|
|
278
276
|
},
|
|
279
277
|
data: items,
|
|
280
|
-
params: this.
|
|
281
|
-
doNotRetryTimeouts: true,
|
|
282
|
-
|
|
278
|
+
params: this.buildParams(),
|
|
279
|
+
doNotRetryTimeouts: true,
|
|
280
|
+
timeoutSecs,
|
|
281
|
+
signal,
|
|
283
282
|
});
|
|
284
283
|
}
|
|
285
284
|
/**
|
|
@@ -288,25 +287,22 @@ export class DatasetClient extends ResourceClient {
|
|
|
288
287
|
* Returns statistics for each field in the dataset, including information about
|
|
289
288
|
* data types, null counts, and value ranges.
|
|
290
289
|
*
|
|
291
|
-
* @
|
|
290
|
+
* @param options - Request options
|
|
291
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
292
|
+
* @returns Dataset statistics
|
|
292
293
|
* @see https://docs.apify.com/api/v2/dataset-statistics-get
|
|
293
294
|
* @since Added in 2.11.2
|
|
294
295
|
*/
|
|
295
|
-
async getStatistics() {
|
|
296
|
-
const
|
|
297
|
-
|
|
296
|
+
async getStatistics(options = {}) {
|
|
297
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
298
|
+
const response = await this.httpClient.call({
|
|
299
|
+
url: this.buildUrl('statistics'),
|
|
298
300
|
method: 'GET',
|
|
299
|
-
params: this.
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
return parseResponse(response, schemas.DatasetStatistics());
|
|
305
|
-
}
|
|
306
|
-
catch (err) {
|
|
307
|
-
catchNotFoundOrThrow(err);
|
|
308
|
-
}
|
|
309
|
-
return undefined;
|
|
301
|
+
params: this.buildParams(),
|
|
302
|
+
timeoutSecs,
|
|
303
|
+
signal,
|
|
304
|
+
});
|
|
305
|
+
return parseResponse(response, schemas.DatasetStatistics());
|
|
310
306
|
}
|
|
311
307
|
/**
|
|
312
308
|
* Generates a public URL for accessing dataset items.
|
|
@@ -318,8 +314,10 @@ export class DatasetClient extends ResourceClient {
|
|
|
318
314
|
* @param options - URL generation options (extends all options from {@link listItems})
|
|
319
315
|
* @param options.expiresInSecs - Number of seconds until the signed URL expires. If omitted, the URL never expires.
|
|
320
316
|
* @param options.fields - Array of field names to include in the response.
|
|
317
|
+
* @param options.format - Format of the items served by the URL. Defaults to `json`.
|
|
321
318
|
* @param options.limit - Maximum number of items to return.
|
|
322
319
|
* @param options.offset - Number of items to skip.
|
|
320
|
+
* @param options.timeoutSecs - Timeout for the API request that fetches the dataset. Default is `'long'`.
|
|
323
321
|
* @returns A public URL string for accessing the dataset items
|
|
324
322
|
*
|
|
325
323
|
* @example
|
|
@@ -341,10 +339,9 @@ export class DatasetClient extends ResourceClient {
|
|
|
341
339
|
* @since Added in 2.13.0
|
|
342
340
|
*/
|
|
343
341
|
async createItemsPublicUrl(options = {}) {
|
|
344
|
-
const
|
|
345
|
-
const dataset = await this.get();
|
|
346
|
-
|
|
347
|
-
let createdItemsPublicUrl = new URL(this._publicUrl('items'));
|
|
342
|
+
const { timeoutSecs = 'long', signal, expiresInSecs, ...queryOptions } = parseArgument(options, createItemsPublicUrlOptionsSchema, 'DatasetClientCreateItemsUrlOptions');
|
|
343
|
+
const dataset = await this.get({ timeoutSecs, signal });
|
|
344
|
+
let createdItemsPublicUrl = new URL(this.buildPublicUrl('items'));
|
|
348
345
|
if (dataset?.urlSigningSecretKey) {
|
|
349
346
|
const signature = await createStorageContentSignatureAsync({
|
|
350
347
|
resourceId: dataset.id,
|
|
@@ -356,31 +353,23 @@ export class DatasetClient extends ResourceClient {
|
|
|
356
353
|
createdItemsPublicUrl = applyQueryParamsToUrl(createdItemsPublicUrl, queryOptions);
|
|
357
354
|
return createdItemsPublicUrl.toString();
|
|
358
355
|
}
|
|
359
|
-
|
|
360
|
-
|
|
356
|
+
#createPaginationList(response, userProvidedDesc) {
|
|
357
|
+
const descHeader = response.headers['x-apify-pagination-desc'];
|
|
358
|
+
const page = {
|
|
361
359
|
items: response.data,
|
|
362
360
|
total: Number(response.headers['x-apify-pagination-total']),
|
|
363
361
|
offset: Number(response.headers['x-apify-pagination-offset']),
|
|
364
|
-
|
|
362
|
+
// `x-apify-pagination-count` reports the rows scanned, which filters and `unwind` can leave above or below
|
|
363
|
+
// the items returned.
|
|
364
|
+
count: response.data.length,
|
|
365
365
|
limit: Number(response.headers['x-apify-pagination-limit']), // API returns 999999999999 when no limit is used
|
|
366
366
|
// TODO: Replace this once https://github.com/apify/apify-core/issues/3503 is solved
|
|
367
|
-
desc: JSON.parse(
|
|
367
|
+
desc: typeof descHeader === 'string' ? JSON.parse(descHeader) : userProvidedDesc,
|
|
368
368
|
};
|
|
369
|
+
const scannedCount = response.headers['x-apify-pagination-count'];
|
|
370
|
+
if (scannedCount === undefined)
|
|
371
|
+
return page;
|
|
372
|
+
// The offset iterator paginates by the scanned number, so it travels with the page outside its public shape.
|
|
373
|
+
return Object.defineProperty(page, SCANNED_COUNT, { value: Number(scannedCount) });
|
|
369
374
|
}
|
|
370
375
|
}
|
|
371
|
-
/**
|
|
372
|
-
* Supported formats for downloading dataset items.
|
|
373
|
-
*/
|
|
374
|
-
export var DownloadItemsFormat;
|
|
375
|
-
(function (DownloadItemsFormat) {
|
|
376
|
-
DownloadItemsFormat["JSON"] = "json";
|
|
377
|
-
DownloadItemsFormat["JSONL"] = "jsonl";
|
|
378
|
-
DownloadItemsFormat["XML"] = "xml";
|
|
379
|
-
DownloadItemsFormat["HTML"] = "html";
|
|
380
|
-
DownloadItemsFormat["CSV"] = "csv";
|
|
381
|
-
DownloadItemsFormat["XLSX"] = "xlsx";
|
|
382
|
-
DownloadItemsFormat["RSS"] = "rss";
|
|
383
|
-
})(DownloadItemsFormat || (DownloadItemsFormat = {}));
|
|
384
|
-
const validItemFormats = [...new Set(Object.values(DownloadItemsFormat).map((item) => item.toLowerCase()))];
|
|
385
|
-
// Declared below `validItemFormats` because it references it at module load time.
|
|
386
|
-
const itemFormatSchema = z.enum(validItemFormats);
|