apify-client 3.0.0-beta.3 → 3.0.0-beta.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -3
- package/dist/apify_api_error.d.ts +62 -21
- package/dist/apify_api_error.js +96 -23
- package/dist/apify_client.d.ts +114 -9
- package/dist/apify_client.js +133 -48
- package/dist/base/api_client.d.ts +7 -7
- package/dist/base/api_client.js +24 -16
- package/dist/base/resource_client.d.ts +23 -9
- package/dist/base/resource_client.js +57 -29
- package/dist/base/resource_collection_client.d.ts +8 -4
- package/dist/base/resource_collection_client.js +26 -15
- package/dist/body_parser.d.ts +3 -4
- package/dist/body_parser.js +19 -26
- package/dist/bundle.js +29 -22
- package/dist/bundle.js.map +1 -1
- package/dist/generated/api.d.ts +287 -214
- package/dist/generated/schemas.d.ts +6699 -0
- package/dist/generated/schemas.js +1521 -0
- package/dist/http_clients/axios.d.ts +60 -0
- package/dist/http_clients/axios.js +117 -0
- package/dist/http_clients/base.d.ts +333 -0
- package/dist/http_clients/base.js +563 -0
- package/dist/http_clients/index.d.ts +4 -0
- package/dist/http_clients/index.js +2 -0
- package/dist/http_compressors/base.d.ts +40 -0
- package/dist/http_compressors/base.js +9 -0
- package/dist/http_compressors/brotli.d.ts +33 -0
- package/dist/http_compressors/brotli.js +39 -0
- package/dist/http_compressors/gzip.d.ts +33 -0
- package/dist/http_compressors/gzip.js +39 -0
- package/dist/http_compressors/index.d.ts +6 -0
- package/dist/http_compressors/index.js +2 -0
- package/dist/http_compressors/resolve.d.ts +23 -0
- package/dist/http_compressors/resolve.js +27 -0
- package/dist/index.d.ts +5 -1
- package/dist/index.js +4 -1
- package/dist/invalid_response_body_error.d.ts +13 -0
- package/dist/invalid_response_body_error.js +17 -0
- package/dist/lazy_schema.d.ts +8 -0
- package/dist/lazy_schema.js +11 -0
- package/dist/models.d.ts +32 -52
- package/dist/resource_clients/actor.d.ts +46 -31
- package/dist/resource_clients/actor.js +78 -58
- package/dist/resource_clients/actor_collection.d.ts +9 -5
- package/dist/resource_clients/actor_collection.js +12 -3
- package/dist/resource_clients/actor_env_var.d.ts +10 -3
- package/dist/resource_clients/actor_env_var.js +17 -6
- package/dist/resource_clients/actor_env_var_collection.d.ts +14 -18
- package/dist/resource_clients/actor_env_var_collection.js +18 -11
- package/dist/resource_clients/actor_version.d.ts +18 -3
- package/dist/resource_clients/actor_version.js +20 -9
- package/dist/resource_clients/actor_version_collection.d.ts +16 -17
- package/dist/resource_clients/actor_version_collection.js +20 -11
- package/dist/resource_clients/build.d.ts +18 -6
- package/dist/resource_clients/build.js +37 -17
- package/dist/resource_clients/build_collection.d.ts +3 -1
- package/dist/resource_clients/build_collection.js +5 -1
- package/dist/resource_clients/dataset.d.ts +30 -18
- package/dist/resource_clients/dataset.js +85 -60
- package/dist/resource_clients/dataset_collection.d.ts +6 -2
- package/dist/resource_clients/dataset_collection.js +14 -2
- package/dist/resource_clients/key_value_store.d.ts +58 -16
- package/dist/resource_clients/key_value_store.js +80 -41
- package/dist/resource_clients/key_value_store_collection.d.ts +6 -2
- package/dist/resource_clients/key_value_store_collection.js +14 -2
- package/dist/resource_clients/log.d.ts +11 -19
- package/dist/resource_clients/log.js +64 -46
- package/dist/resource_clients/request_queue.d.ts +56 -26
- package/dist/resource_clients/request_queue.js +197 -116
- package/dist/resource_clients/request_queue_collection.d.ts +6 -2
- package/dist/resource_clients/request_queue_collection.js +10 -3
- package/dist/resource_clients/run.d.ts +52 -18
- package/dist/resource_clients/run.js +93 -47
- package/dist/resource_clients/run_collection.d.ts +3 -1
- package/dist/resource_clients/run_collection.js +5 -1
- package/dist/resource_clients/schedule.d.ts +16 -7
- package/dist/resource_clients/schedule.js +32 -21
- package/dist/resource_clients/schedule_collection.d.ts +6 -2
- package/dist/resource_clients/schedule_collection.js +10 -3
- package/dist/resource_clients/store_collection.d.ts +3 -1
- package/dist/resource_clients/store_collection.js +5 -1
- package/dist/resource_clients/task.d.ts +28 -10
- package/dist/resource_clients/task.js +71 -46
- package/dist/resource_clients/task_collection.d.ts +6 -2
- package/dist/resource_clients/task_collection.js +10 -3
- package/dist/resource_clients/user.d.ts +17 -8
- package/dist/resource_clients/user.js +43 -39
- package/dist/resource_clients/webhook.d.ts +14 -5
- package/dist/resource_clients/webhook.js +31 -22
- package/dist/resource_clients/webhook_collection.d.ts +6 -2
- package/dist/resource_clients/webhook_collection.js +10 -3
- package/dist/resource_clients/webhook_dispatch.d.ts +4 -1
- package/dist/resource_clients/webhook_dispatch.js +8 -2
- package/dist/resource_clients/webhook_dispatch_collection.d.ts +3 -1
- package/dist/resource_clients/webhook_dispatch_collection.js +5 -1
- package/dist/response_validation_error.d.ts +26 -0
- package/dist/response_validation_error.js +37 -0
- package/dist/runtime/node.d.ts +6 -0
- package/dist/runtime/node.js +49 -0
- package/dist/runtime/types.d.ts +52 -0
- package/dist/runtime/types.js +1 -0
- package/dist/runtime/web.d.ts +5 -0
- package/dist/runtime/web.js +18 -0
- package/dist/schemas.d.ts +15 -0
- package/dist/schemas.js +15 -0
- package/dist/statistics.d.ts +1 -6
- package/dist/statistics.js +2 -2
- package/dist/timeouts.d.ts +84 -0
- package/dist/timeouts.js +37 -0
- package/dist/timezones.d.ts +1 -1
- package/dist/utils.d.ts +86 -46
- package/dist/utils.js +218 -164
- package/package.json +42 -38
- package/dist/http_client.d.ts +0 -88
- package/dist/http_client.js +0 -255
- package/dist/interceptors.d.ts +0 -19
- package/dist/interceptors.js +0 -105
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { ResourceClient } from '../base/resource_client.js';
|
|
3
|
-
import
|
|
3
|
+
import * as schemas from '../schemas.js';
|
|
4
|
+
import { timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
|
|
5
|
+
import { parseArgument, parseResponse } from '../utils.js';
|
|
4
6
|
import { LogClient } from './log.js';
|
|
5
|
-
const getOptionsSchema = z.strictObject({ waitForFinish: z.number().optional() });
|
|
6
|
-
const waitForFinishOptionsSchema = z.strictObject({ waitSecs: z.number().optional() });
|
|
7
|
+
const getOptionsSchema = z.strictObject({ waitForFinish: z.number().optional(), ...timeoutOptionsShape });
|
|
8
|
+
const waitForFinishOptionsSchema = z.strictObject({ waitSecs: z.number().optional(), ...timeoutOptionsShape });
|
|
7
9
|
/**
|
|
8
10
|
* Client for managing a specific Actor build.
|
|
9
11
|
*
|
|
@@ -42,6 +44,8 @@ export class BuildClient extends ResourceClient {
|
|
|
42
44
|
*
|
|
43
45
|
* @param options - Get options
|
|
44
46
|
* @param options.waitForFinish - Maximum time to wait (in seconds, max 60s) for the build to finish on the API side before returning. Default is 0 (returns immediately).
|
|
47
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`, extended to cover `waitForFinish`
|
|
48
|
+
* when the API is asked to hold the response.
|
|
45
49
|
* @returns The Build object, or `undefined` if it does not exist
|
|
46
50
|
* @see https://docs.apify.com/api/v2/actor-build-get
|
|
47
51
|
*
|
|
@@ -56,14 +60,16 @@ export class BuildClient extends ResourceClient {
|
|
|
56
60
|
* ```
|
|
57
61
|
*/
|
|
58
62
|
async get(options = {}) {
|
|
59
|
-
const
|
|
60
|
-
return this.
|
|
63
|
+
const { timeoutSecs, signal, ...params } = parseArgument(options, getOptionsSchema, 'BuildClientGetOptions');
|
|
64
|
+
return this.getResource(schemas.Build(), params, this.timeoutForWaitForFinish(timeoutSecs, 'short', params.waitForFinish), signal);
|
|
61
65
|
}
|
|
62
66
|
/**
|
|
63
67
|
* Aborts the Actor build.
|
|
64
68
|
*
|
|
65
69
|
* Stops the build process immediately. The build will have an `ABORTED` status.
|
|
66
70
|
*
|
|
71
|
+
* @param options - Request options
|
|
72
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
67
73
|
* @returns The updated Build object with `ABORTED` status
|
|
68
74
|
* @see https://docs.apify.com/api/v2/actor-build-abort-post
|
|
69
75
|
*
|
|
@@ -72,35 +78,46 @@ export class BuildClient extends ResourceClient {
|
|
|
72
78
|
* await client.build('build-id').abort();
|
|
73
79
|
* ```
|
|
74
80
|
*/
|
|
75
|
-
async abort() {
|
|
81
|
+
async abort(options = {}) {
|
|
82
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
76
83
|
const response = await this.httpClient.call({
|
|
77
|
-
url: this.
|
|
84
|
+
url: this.buildUrl('abort'),
|
|
78
85
|
method: 'POST',
|
|
79
|
-
params: this.
|
|
86
|
+
params: this.buildParams(),
|
|
87
|
+
timeoutSecs,
|
|
88
|
+
signal,
|
|
80
89
|
});
|
|
81
|
-
return
|
|
90
|
+
return parseResponse(response, schemas.Build());
|
|
82
91
|
}
|
|
83
92
|
/**
|
|
84
93
|
* Deletes the Actor build.
|
|
85
94
|
*
|
|
95
|
+
* @param options - Request options
|
|
96
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
86
97
|
* @see https://docs.apify.com/api/v2/actor-build-delete
|
|
87
98
|
* @since Added in 2.8.1
|
|
88
99
|
*/
|
|
89
|
-
async delete() {
|
|
90
|
-
|
|
100
|
+
async delete(options = {}) {
|
|
101
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
102
|
+
return this.deleteResource(timeoutSecs, signal);
|
|
91
103
|
}
|
|
92
104
|
/**
|
|
93
105
|
* Retrieves the OpenAPI definition for the Actor build.
|
|
94
106
|
*
|
|
107
|
+
* @param options - Request options
|
|
108
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
|
|
95
109
|
* @returns The OpenAPI definition object.
|
|
96
110
|
* @see https://docs.apify.com/api/v2/actor-build-openapi-json-get
|
|
97
111
|
* @since Added in 2.11.2
|
|
98
112
|
*/
|
|
99
|
-
async getOpenApiDefinition() {
|
|
113
|
+
async getOpenApiDefinition(options = {}) {
|
|
114
|
+
const { timeoutSecs = 'medium', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
100
115
|
const response = await this.httpClient.call({
|
|
101
|
-
url: this.
|
|
116
|
+
url: this.buildUrl('openapi.json'),
|
|
102
117
|
method: 'GET',
|
|
103
|
-
params: this.
|
|
118
|
+
params: this.buildParams(),
|
|
119
|
+
timeoutSecs,
|
|
120
|
+
signal,
|
|
104
121
|
});
|
|
105
122
|
return response.data;
|
|
106
123
|
}
|
|
@@ -119,7 +136,8 @@ export class BuildClient extends ResourceClient {
|
|
|
119
136
|
*
|
|
120
137
|
* @param options - Wait options
|
|
121
138
|
* @param options.waitSecs - Maximum time to wait for the build to finish, in seconds. If omitted, waits indefinitely.
|
|
122
|
-
* @
|
|
139
|
+
* @param options.timeoutSecs - Timeout for each polling API request. Default is `'noTimeout'`.
|
|
140
|
+
* @returns The Build object (finished or still building if the timeout was reached)
|
|
123
141
|
*
|
|
124
142
|
* @example
|
|
125
143
|
* ```javascript
|
|
@@ -136,11 +154,13 @@ export class BuildClient extends ResourceClient {
|
|
|
136
154
|
*/
|
|
137
155
|
async waitForFinish(options = {}) {
|
|
138
156
|
const parsed = parseArgument(options, waitForFinishOptionsSchema, 'BuildClientWaitForFinishOptions');
|
|
139
|
-
return this.
|
|
157
|
+
return this.waitForJobFinish(schemas.Build(), parsed);
|
|
140
158
|
}
|
|
141
159
|
/**
|
|
142
160
|
* Returns a client for accessing the log of this Actor build.
|
|
143
161
|
*
|
|
162
|
+
* A 404 from this client throws an `ApifyApiError`, since the build itself may be what is missing.
|
|
163
|
+
*
|
|
144
164
|
* @returns A client for accessing the build's log
|
|
145
165
|
* @see https://docs.apify.com/api/v2/actor-build-log-get
|
|
146
166
|
*
|
|
@@ -153,7 +173,7 @@ export class BuildClient extends ResourceClient {
|
|
|
153
173
|
* @since Added in 2.9.0
|
|
154
174
|
*/
|
|
155
175
|
log() {
|
|
156
|
-
return new LogClient(this.
|
|
176
|
+
return new LogClient(this.subResourceOptions({
|
|
157
177
|
resourcePath: 'log',
|
|
158
178
|
}));
|
|
159
179
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ApiClientOptionsWithOptionalResourcePath } from '../base/api_client.js';
|
|
2
2
|
import { ResourceCollectionClient } from '../base/resource_collection_client.js';
|
|
3
3
|
import type { BuildCollectionClientListItem } from '../models.js';
|
|
4
|
+
import type { TimeoutOptions } from '../timeouts.js';
|
|
4
5
|
import type { PaginatedIterator, PaginatedList, PaginationOptions } from '../utils.js';
|
|
5
6
|
export type { BuildCollectionClientListItem } from '../models.js';
|
|
6
7
|
/**
|
|
@@ -46,12 +47,13 @@ export declare class BuildCollectionClient extends ResourceCollectionClient {
|
|
|
46
47
|
* ```
|
|
47
48
|
*
|
|
48
49
|
* @param options - Pagination and sorting options.
|
|
50
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
|
|
49
51
|
* @returns A paginated iterator of Actor builds.
|
|
50
52
|
* @see https://docs.apify.com/api/v2/actor-builds-get
|
|
51
53
|
*/
|
|
52
54
|
list(options?: BuildCollectionClientListOptions): PaginatedIterator<BuildCollectionClientListItem>;
|
|
53
55
|
}
|
|
54
|
-
export interface BuildCollectionClientListOptions extends PaginationOptions {
|
|
56
|
+
export interface BuildCollectionClientListOptions extends PaginationOptions, TimeoutOptions {
|
|
55
57
|
desc?: boolean;
|
|
56
58
|
}
|
|
57
59
|
export type BuildCollectionClientListResult = PaginatedList<BuildCollectionClientListItem>;
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { ResourceCollectionClient } from '../base/resource_collection_client.js';
|
|
3
|
+
import * as schemas from '../schemas.js';
|
|
4
|
+
import { timeoutOptionsShape } from '../timeouts.js';
|
|
3
5
|
import { paginationOptionsShape, parseArgument } from '../utils.js';
|
|
4
6
|
const listOptionsSchema = z.strictObject({
|
|
5
7
|
...paginationOptionsShape,
|
|
6
8
|
desc: z.boolean().optional(),
|
|
9
|
+
...timeoutOptionsShape,
|
|
7
10
|
});
|
|
8
11
|
/**
|
|
9
12
|
* Client for managing the collection of Actor builds.
|
|
@@ -53,11 +56,12 @@ export class BuildCollectionClient extends ResourceCollectionClient {
|
|
|
53
56
|
* ```
|
|
54
57
|
*
|
|
55
58
|
* @param options - Pagination and sorting options.
|
|
59
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
|
|
56
60
|
* @returns A paginated iterator of Actor builds.
|
|
57
61
|
* @see https://docs.apify.com/api/v2/actor-builds-get
|
|
58
62
|
*/
|
|
59
63
|
list(options = {}) {
|
|
60
64
|
const parsed = parseArgument(options, listOptionsSchema, 'BuildCollectionClientListOptions');
|
|
61
|
-
return this.
|
|
65
|
+
return this.listResourcesPaginated(schemas.ListOfBuilds(), parsed, 'medium');
|
|
62
66
|
}
|
|
63
67
|
}
|
|
@@ -2,6 +2,7 @@ import type { STORAGE_GENERAL_ACCESS } from '@apify/consts';
|
|
|
2
2
|
import type { ApiClientSubResourceOptions } from '../base/api_client.js';
|
|
3
3
|
import { ResourceClient } from '../base/resource_client.js';
|
|
4
4
|
import type { Dataset, DatasetStatistics } from '../models.js';
|
|
5
|
+
import type { TimeoutOptions } from '../timeouts.js';
|
|
5
6
|
import type { PaginatedIterator, PaginationOptions } from '../utils.js';
|
|
6
7
|
export type { Dataset, DatasetStatistics, DatasetStats, FieldStatistics } from '../models.js';
|
|
7
8
|
/**
|
|
@@ -34,6 +35,7 @@ export type { Dataset, DatasetStatistics, DatasetStats, FieldStatistics } from '
|
|
|
34
35
|
* @see https://docs.apify.com/platform/storage/dataset
|
|
35
36
|
*/
|
|
36
37
|
export declare class DatasetClient<Data extends Record<string | number, any> = Record<string | number, unknown>> extends ResourceClient {
|
|
38
|
+
#private;
|
|
37
39
|
/**
|
|
38
40
|
* @hidden
|
|
39
41
|
*/
|
|
@@ -41,24 +43,30 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
41
43
|
/**
|
|
42
44
|
* Gets the dataset object from the Apify API.
|
|
43
45
|
*
|
|
46
|
+
* @param options - Request options
|
|
47
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
44
48
|
* @returns The Dataset object, or `undefined` if it does not exist
|
|
45
49
|
* @see https://docs.apify.com/api/v2/dataset-get
|
|
46
50
|
*/
|
|
47
|
-
get(): Promise<Dataset | undefined>;
|
|
51
|
+
get(options?: TimeoutOptions): Promise<Dataset | undefined>;
|
|
48
52
|
/**
|
|
49
53
|
* Updates the dataset with specified fields.
|
|
50
54
|
*
|
|
51
55
|
* @param newFields - Fields to update in the dataset
|
|
56
|
+
* @param options - Request options
|
|
57
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
52
58
|
* @returns The updated Dataset object
|
|
53
59
|
* @see https://docs.apify.com/api/v2/dataset-put
|
|
54
60
|
*/
|
|
55
|
-
update(newFields: DatasetClientUpdateOptions): Promise<Dataset>;
|
|
61
|
+
update(newFields: DatasetClientUpdateOptions, options?: TimeoutOptions): Promise<Dataset>;
|
|
56
62
|
/**
|
|
57
63
|
* Deletes the dataset.
|
|
58
64
|
*
|
|
65
|
+
* @param options - Request options
|
|
66
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
59
67
|
* @see https://docs.apify.com/api/v2/dataset-delete
|
|
60
68
|
*/
|
|
61
|
-
delete(): Promise<void>;
|
|
69
|
+
delete(options?: TimeoutOptions): Promise<void>;
|
|
62
70
|
/**
|
|
63
71
|
* Lists items in the dataset.
|
|
64
72
|
*
|
|
@@ -79,6 +87,7 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
79
87
|
* @param options.flatten - Array of field names to flatten. Nested objects are converted to dot notation (e.g., `obj.field`).
|
|
80
88
|
* @param options.unwind - Field name or array of field names to unwind. Each array value creates a separate item.
|
|
81
89
|
* @param options.view - Name of a predefined view to use for field selection.
|
|
90
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'long'`.
|
|
82
91
|
* @returns A paginated list with `items`, `total` count, `offset`, `count`, and `limit`
|
|
83
92
|
* @see https://docs.apify.com/api/v2/dataset-items-get
|
|
84
93
|
*
|
|
@@ -121,6 +130,7 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
121
130
|
* @param options.xmlRow - Name of the XML element for each item. Default is `'item'`.
|
|
122
131
|
* @param options.fields - Array of field names to include in the export.
|
|
123
132
|
* @param options.omit - Array of field names to exclude from the export.
|
|
133
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
124
134
|
* @returns Buffer containing the serialized data in the specified format
|
|
125
135
|
* @see https://docs.apify.com/api/v2/dataset-items-get
|
|
126
136
|
*
|
|
@@ -144,17 +154,20 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
144
154
|
* });
|
|
145
155
|
* ```
|
|
146
156
|
*/
|
|
147
|
-
downloadItems(format: DownloadItemsFormat
|
|
157
|
+
downloadItems(format: `${DownloadItemsFormat}`, options?: DatasetClientDownloadItemsOptions): Promise<Buffer>;
|
|
148
158
|
/**
|
|
149
159
|
* Stores one or more items into the dataset.
|
|
150
160
|
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
161
|
+
* Each item will be stored as a separate record in the dataset. Objects are automatically
|
|
162
|
+
* serialized to JSON. If you provide an array, all items will be stored in order. This method
|
|
163
|
+
* is idempotent - calling it multiple times with the same data will not create duplicates, but
|
|
164
|
+
* will append items each time.
|
|
155
165
|
*
|
|
156
|
-
* @param items - A single item
|
|
157
|
-
*
|
|
166
|
+
* @param items - A single item, an array of items, or a string that is the JSON serialization of
|
|
167
|
+
* either - the API only accepts an object or an array of objects, so a plain string
|
|
168
|
+
* is not a valid item on its own.
|
|
169
|
+
* @param options - Request options
|
|
170
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
|
|
158
171
|
* @see https://docs.apify.com/api/v2/dataset-items-post
|
|
159
172
|
*
|
|
160
173
|
* @example
|
|
@@ -172,23 +185,22 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
172
185
|
* { url: 'https://test.com', title: 'Test' },
|
|
173
186
|
* { url: 'https://demo.com', title: 'Demo' }
|
|
174
187
|
* ]);
|
|
175
|
-
*
|
|
176
|
-
* // Store string items
|
|
177
|
-
* await client.dataset('my-dataset').pushItems(['item1', 'item2', 'item3']);
|
|
178
188
|
* ```
|
|
179
189
|
*/
|
|
180
|
-
pushItems(items: Data | Data[] | string
|
|
190
|
+
pushItems(items: Data | Data[] | string, options?: TimeoutOptions): Promise<void>;
|
|
181
191
|
/**
|
|
182
192
|
* Gets statistical information about the dataset.
|
|
183
193
|
*
|
|
184
194
|
* Returns statistics for each field in the dataset, including information about
|
|
185
195
|
* data types, null counts, and value ranges.
|
|
186
196
|
*
|
|
187
|
-
* @
|
|
197
|
+
* @param options - Request options
|
|
198
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
199
|
+
* @returns Dataset statistics
|
|
188
200
|
* @see https://docs.apify.com/api/v2/dataset-statistics-get
|
|
189
201
|
* @since Added in 2.11.2
|
|
190
202
|
*/
|
|
191
|
-
getStatistics(): Promise<DatasetStatistics
|
|
203
|
+
getStatistics(options?: TimeoutOptions): Promise<DatasetStatistics>;
|
|
192
204
|
/**
|
|
193
205
|
* Generates a public URL for accessing dataset items.
|
|
194
206
|
*
|
|
@@ -201,6 +213,7 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
201
213
|
* @param options.fields - Array of field names to include in the response.
|
|
202
214
|
* @param options.limit - Maximum number of items to return.
|
|
203
215
|
* @param options.offset - Number of items to skip.
|
|
216
|
+
* @param options.timeoutSecs - Timeout for the API request that fetches the dataset. Default is `'long'`.
|
|
204
217
|
* @returns A public URL string for accessing the dataset items
|
|
205
218
|
*
|
|
206
219
|
* @example
|
|
@@ -222,7 +235,6 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
222
235
|
* @since Added in 2.13.0
|
|
223
236
|
*/
|
|
224
237
|
createItemsPublicUrl(options?: DatasetClientCreateItemsUrlOptions): Promise<string>;
|
|
225
|
-
private _createPaginationList;
|
|
226
238
|
}
|
|
227
239
|
/**
|
|
228
240
|
* Options for updating a dataset.
|
|
@@ -244,7 +256,7 @@ export interface DatasetClientUpdateOptions {
|
|
|
244
256
|
* Provides various filtering, pagination, and transformation options to customize
|
|
245
257
|
* the output format and content of retrieved items.
|
|
246
258
|
*/
|
|
247
|
-
export interface DatasetClientListItemOptions extends PaginationOptions {
|
|
259
|
+
export interface DatasetClientListItemOptions extends PaginationOptions, TimeoutOptions {
|
|
248
260
|
clean?: boolean;
|
|
249
261
|
desc?: boolean;
|
|
250
262
|
/**
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { createStorageContentSignatureAsync } from '@apify/utilities';
|
|
3
|
-
import {
|
|
4
|
-
import
|
|
3
|
+
import { ResourceClient } from '../base/resource_client.js';
|
|
4
|
+
import * as schemas from '../schemas.js';
|
|
5
|
+
import { timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
|
|
6
|
+
import { anyObjectSchema, applyQueryParamsToUrl, cast, isNonArrayObject, paginationOptionsShape, parseArgument, parseResponse, SCANNED_COUNT, } from '../utils.js';
|
|
5
7
|
// A predicate, not a `z.object()` arm, which would walk and copy every key of every pushed item.
|
|
6
8
|
const itemSchema = z.custom(isNonArrayObject, 'Expected an object');
|
|
7
9
|
const listItemsOptionsSchema = z.strictObject({
|
|
@@ -16,6 +18,7 @@ const listItemsOptionsSchema = z.strictObject({
|
|
|
16
18
|
unwind: z.union([z.string(), z.array(z.string())]).optional(),
|
|
17
19
|
view: z.string().optional(),
|
|
18
20
|
signature: z.string().optional(),
|
|
21
|
+
...timeoutOptionsShape,
|
|
19
22
|
});
|
|
20
23
|
// Spells out `limit` / `offset` instead of spreading `paginationOptionsShape`: a download is one
|
|
21
24
|
// request, so `chunkSize` has nothing to size. The options type omits it to match.
|
|
@@ -38,10 +41,12 @@ const downloadItemsOptionsSchema = z.strictObject({
|
|
|
38
41
|
xmlRoot: z.string().optional(),
|
|
39
42
|
xmlRow: z.string().optional(),
|
|
40
43
|
signature: z.string().optional(),
|
|
44
|
+
...timeoutOptionsShape,
|
|
41
45
|
});
|
|
42
|
-
const pushItemsSchema = z.union([itemSchema, z.string(), z.array(
|
|
43
|
-
//
|
|
44
|
-
// `signature` (which this method produces) are left out. The options type
|
|
46
|
+
const pushItemsSchema = z.union([itemSchema, z.string(), z.array(itemSchema)]);
|
|
47
|
+
// Apart from `timeoutSecs`, `signal` and `expiresInSecs`, every option becomes a query parameter of the generated
|
|
48
|
+
// URL, so `chunkSize` (client-side only) and `signature` (which this method produces) are left out. The options type
|
|
49
|
+
// omits both to match.
|
|
45
50
|
const createItemsPublicUrlOptionsSchema = z.strictObject({
|
|
46
51
|
clean: z.boolean().optional(),
|
|
47
52
|
desc: z.boolean().optional(),
|
|
@@ -55,6 +60,7 @@ const createItemsPublicUrlOptionsSchema = z.strictObject({
|
|
|
55
60
|
unwind: z.union([z.string(), z.array(z.string())]).optional(),
|
|
56
61
|
view: z.string().optional(),
|
|
57
62
|
expiresInSecs: z.number().optional(),
|
|
63
|
+
...timeoutOptionsShape,
|
|
58
64
|
});
|
|
59
65
|
/**
|
|
60
66
|
* Client for managing a specific Dataset.
|
|
@@ -98,30 +104,39 @@ export class DatasetClient extends ResourceClient {
|
|
|
98
104
|
/**
|
|
99
105
|
* Gets the dataset object from the Apify API.
|
|
100
106
|
*
|
|
107
|
+
* @param options - Request options
|
|
108
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
101
109
|
* @returns The Dataset object, or `undefined` if it does not exist
|
|
102
110
|
* @see https://docs.apify.com/api/v2/dataset-get
|
|
103
111
|
*/
|
|
104
|
-
async get() {
|
|
105
|
-
|
|
112
|
+
async get(options = {}) {
|
|
113
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
114
|
+
return this.getResource(schemas.Dataset(), {}, timeoutSecs, signal);
|
|
106
115
|
}
|
|
107
116
|
/**
|
|
108
117
|
* Updates the dataset with specified fields.
|
|
109
118
|
*
|
|
110
119
|
* @param newFields - Fields to update in the dataset
|
|
120
|
+
* @param options - Request options
|
|
121
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
111
122
|
* @returns The updated Dataset object
|
|
112
123
|
* @see https://docs.apify.com/api/v2/dataset-put
|
|
113
124
|
*/
|
|
114
|
-
async update(newFields) {
|
|
125
|
+
async update(newFields, options = {}) {
|
|
115
126
|
parseArgument(newFields, anyObjectSchema);
|
|
116
|
-
|
|
127
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
128
|
+
return this.updateResource(schemas.Dataset(), newFields, timeoutSecs, signal);
|
|
117
129
|
}
|
|
118
130
|
/**
|
|
119
131
|
* Deletes the dataset.
|
|
120
132
|
*
|
|
133
|
+
* @param options - Request options
|
|
134
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
121
135
|
* @see https://docs.apify.com/api/v2/dataset-delete
|
|
122
136
|
*/
|
|
123
|
-
async delete() {
|
|
124
|
-
|
|
137
|
+
async delete(options = {}) {
|
|
138
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
139
|
+
return this.deleteResource(timeoutSecs, signal);
|
|
125
140
|
}
|
|
126
141
|
/**
|
|
127
142
|
* Lists items in the dataset.
|
|
@@ -143,6 +158,7 @@ export class DatasetClient extends ResourceClient {
|
|
|
143
158
|
* @param options.flatten - Array of field names to flatten. Nested objects are converted to dot notation (e.g., `obj.field`).
|
|
144
159
|
* @param options.unwind - Field name or array of field names to unwind. Each array value creates a separate item.
|
|
145
160
|
* @param options.view - Name of a predefined view to use for field selection.
|
|
161
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'long'`.
|
|
146
162
|
* @returns A paginated list with `items`, `total` count, `offset`, `count`, and `limit`
|
|
147
163
|
* @see https://docs.apify.com/api/v2/dataset-items-get
|
|
148
164
|
*
|
|
@@ -168,17 +184,18 @@ export class DatasetClient extends ResourceClient {
|
|
|
168
184
|
* ```
|
|
169
185
|
*/
|
|
170
186
|
listItems(options = {}) {
|
|
171
|
-
const
|
|
187
|
+
const { timeoutSecs = 'long', signal, ...listOptions } = parseArgument(options, listItemsOptionsSchema, 'DatasetClientListItemOptions');
|
|
172
188
|
const fetchItems = async (datasetListOptions = {}) => {
|
|
173
189
|
const response = await this.httpClient.call({
|
|
174
|
-
url: this.
|
|
190
|
+
url: this.buildUrl('items'),
|
|
175
191
|
method: 'GET',
|
|
176
|
-
params: this.
|
|
177
|
-
|
|
192
|
+
params: this.buildParams(datasetListOptions),
|
|
193
|
+
timeoutSecs,
|
|
194
|
+
signal,
|
|
178
195
|
});
|
|
179
|
-
return this
|
|
196
|
+
return this.#createPaginationList(response, datasetListOptions.desc ?? false);
|
|
180
197
|
};
|
|
181
|
-
return this.
|
|
198
|
+
return this.listPaginatedFromCallback(fetchItems, listOptions);
|
|
182
199
|
}
|
|
183
200
|
/**
|
|
184
201
|
* Downloads dataset items in a specific format.
|
|
@@ -197,6 +214,7 @@ export class DatasetClient extends ResourceClient {
|
|
|
197
214
|
* @param options.xmlRow - Name of the XML element for each item. Default is `'item'`.
|
|
198
215
|
* @param options.fields - Array of field names to include in the export.
|
|
199
216
|
* @param options.omit - Array of field names to exclude from the export.
|
|
217
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
200
218
|
* @returns Buffer containing the serialized data in the specified format
|
|
201
219
|
* @see https://docs.apify.com/api/v2/dataset-items-get
|
|
202
220
|
*
|
|
@@ -222,29 +240,33 @@ export class DatasetClient extends ResourceClient {
|
|
|
222
240
|
*/
|
|
223
241
|
async downloadItems(format, options = {}) {
|
|
224
242
|
parseArgument(format, itemFormatSchema);
|
|
225
|
-
const
|
|
243
|
+
const { timeoutSecs = 'long', signal, ...query } = parseArgument(options, downloadItemsOptionsSchema, 'DatasetClientDownloadItemsOptions');
|
|
226
244
|
const { data } = await this.httpClient.call({
|
|
227
|
-
url: this.
|
|
245
|
+
url: this.buildUrl('items'),
|
|
228
246
|
method: 'GET',
|
|
229
|
-
params: this.
|
|
247
|
+
params: this.buildParams({
|
|
230
248
|
format,
|
|
231
|
-
...
|
|
249
|
+
...query,
|
|
232
250
|
}),
|
|
233
|
-
|
|
234
|
-
|
|
251
|
+
responseType: 'buffer',
|
|
252
|
+
timeoutSecs,
|
|
253
|
+
signal,
|
|
235
254
|
});
|
|
236
255
|
return cast(data);
|
|
237
256
|
}
|
|
238
257
|
/**
|
|
239
258
|
* Stores one or more items into the dataset.
|
|
240
259
|
*
|
|
241
|
-
*
|
|
242
|
-
*
|
|
243
|
-
*
|
|
244
|
-
*
|
|
260
|
+
* Each item will be stored as a separate record in the dataset. Objects are automatically
|
|
261
|
+
* serialized to JSON. If you provide an array, all items will be stored in order. This method
|
|
262
|
+
* is idempotent - calling it multiple times with the same data will not create duplicates, but
|
|
263
|
+
* will append items each time.
|
|
245
264
|
*
|
|
246
|
-
* @param items - A single item
|
|
247
|
-
*
|
|
265
|
+
* @param items - A single item, an array of items, or a string that is the JSON serialization of
|
|
266
|
+
* either - the API only accepts an object or an array of objects, so a plain string
|
|
267
|
+
* is not a valid item on its own.
|
|
268
|
+
* @param options - Request options
|
|
269
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
|
|
248
270
|
* @see https://docs.apify.com/api/v2/dataset-items-post
|
|
249
271
|
*
|
|
250
272
|
* @example
|
|
@@ -262,23 +284,22 @@ export class DatasetClient extends ResourceClient {
|
|
|
262
284
|
* { url: 'https://test.com', title: 'Test' },
|
|
263
285
|
* { url: 'https://demo.com', title: 'Demo' }
|
|
264
286
|
* ]);
|
|
265
|
-
*
|
|
266
|
-
* // Store string items
|
|
267
|
-
* await client.dataset('my-dataset').pushItems(['item1', 'item2', 'item3']);
|
|
268
287
|
* ```
|
|
269
288
|
*/
|
|
270
|
-
async pushItems(items) {
|
|
289
|
+
async pushItems(items, options = {}) {
|
|
271
290
|
parseArgument(items, pushItemsSchema);
|
|
291
|
+
const { timeoutSecs = 'medium', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
272
292
|
await this.httpClient.call({
|
|
273
|
-
url: this.
|
|
293
|
+
url: this.buildUrl('items'),
|
|
274
294
|
method: 'POST',
|
|
275
295
|
headers: {
|
|
276
296
|
'content-type': 'application/json; charset=utf-8',
|
|
277
297
|
},
|
|
278
298
|
data: items,
|
|
279
|
-
params: this.
|
|
280
|
-
doNotRetryTimeouts: true,
|
|
281
|
-
|
|
299
|
+
params: this.buildParams(),
|
|
300
|
+
doNotRetryTimeouts: true,
|
|
301
|
+
timeoutSecs,
|
|
302
|
+
signal,
|
|
282
303
|
});
|
|
283
304
|
}
|
|
284
305
|
/**
|
|
@@ -287,25 +308,22 @@ export class DatasetClient extends ResourceClient {
|
|
|
287
308
|
* Returns statistics for each field in the dataset, including information about
|
|
288
309
|
* data types, null counts, and value ranges.
|
|
289
310
|
*
|
|
290
|
-
* @
|
|
311
|
+
* @param options - Request options
|
|
312
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
313
|
+
* @returns Dataset statistics
|
|
291
314
|
* @see https://docs.apify.com/api/v2/dataset-statistics-get
|
|
292
315
|
* @since Added in 2.11.2
|
|
293
316
|
*/
|
|
294
|
-
async getStatistics() {
|
|
295
|
-
const
|
|
296
|
-
|
|
317
|
+
async getStatistics(options = {}) {
|
|
318
|
+
const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
|
|
319
|
+
const response = await this.httpClient.call({
|
|
320
|
+
url: this.buildUrl('statistics'),
|
|
297
321
|
method: 'GET',
|
|
298
|
-
params: this.
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
return cast(pluckData(response.data));
|
|
304
|
-
}
|
|
305
|
-
catch (err) {
|
|
306
|
-
catchNotFoundOrThrow(err);
|
|
307
|
-
}
|
|
308
|
-
return undefined;
|
|
322
|
+
params: this.buildParams(),
|
|
323
|
+
timeoutSecs,
|
|
324
|
+
signal,
|
|
325
|
+
});
|
|
326
|
+
return parseResponse(response, schemas.DatasetStatistics());
|
|
309
327
|
}
|
|
310
328
|
/**
|
|
311
329
|
* Generates a public URL for accessing dataset items.
|
|
@@ -319,6 +337,7 @@ export class DatasetClient extends ResourceClient {
|
|
|
319
337
|
* @param options.fields - Array of field names to include in the response.
|
|
320
338
|
* @param options.limit - Maximum number of items to return.
|
|
321
339
|
* @param options.offset - Number of items to skip.
|
|
340
|
+
* @param options.timeoutSecs - Timeout for the API request that fetches the dataset. Default is `'long'`.
|
|
322
341
|
* @returns A public URL string for accessing the dataset items
|
|
323
342
|
*
|
|
324
343
|
* @example
|
|
@@ -340,10 +359,9 @@ export class DatasetClient extends ResourceClient {
|
|
|
340
359
|
* @since Added in 2.13.0
|
|
341
360
|
*/
|
|
342
361
|
async createItemsPublicUrl(options = {}) {
|
|
343
|
-
const
|
|
344
|
-
const dataset = await this.get();
|
|
345
|
-
|
|
346
|
-
let createdItemsPublicUrl = new URL(this._publicUrl('items'));
|
|
362
|
+
const { timeoutSecs = 'long', signal, expiresInSecs, ...queryOptions } = parseArgument(options, createItemsPublicUrlOptionsSchema, 'DatasetClientCreateItemsUrlOptions');
|
|
363
|
+
const dataset = await this.get({ timeoutSecs, signal });
|
|
364
|
+
let createdItemsPublicUrl = new URL(this.buildPublicUrl('items'));
|
|
347
365
|
if (dataset?.urlSigningSecretKey) {
|
|
348
366
|
const signature = await createStorageContentSignatureAsync({
|
|
349
367
|
resourceId: dataset.id,
|
|
@@ -355,16 +373,23 @@ export class DatasetClient extends ResourceClient {
|
|
|
355
373
|
createdItemsPublicUrl = applyQueryParamsToUrl(createdItemsPublicUrl, queryOptions);
|
|
356
374
|
return createdItemsPublicUrl.toString();
|
|
357
375
|
}
|
|
358
|
-
|
|
359
|
-
|
|
376
|
+
#createPaginationList(response, userProvidedDesc) {
|
|
377
|
+
const descHeader = response.headers['x-apify-pagination-desc'];
|
|
378
|
+
const page = {
|
|
360
379
|
items: response.data,
|
|
361
380
|
total: Number(response.headers['x-apify-pagination-total']),
|
|
362
381
|
offset: Number(response.headers['x-apify-pagination-offset']),
|
|
363
|
-
|
|
382
|
+
// `x-apify-pagination-count` reports the rows scanned, which filters and `unwind` can leave above or below
|
|
383
|
+
// the items returned.
|
|
384
|
+
count: response.data.length,
|
|
364
385
|
limit: Number(response.headers['x-apify-pagination-limit']), // API returns 999999999999 when no limit is used
|
|
365
386
|
// TODO: Replace this once https://github.com/apify/apify-core/issues/3503 is solved
|
|
366
|
-
desc: JSON.parse(
|
|
387
|
+
desc: typeof descHeader === 'string' ? JSON.parse(descHeader) : userProvidedDesc,
|
|
367
388
|
};
|
|
389
|
+
// The offset iterator paginates by the scanned number, so it travels with the page outside its public shape.
|
|
390
|
+
return Object.defineProperty(page, SCANNED_COUNT, {
|
|
391
|
+
value: Number(response.headers['x-apify-pagination-count']) || 0,
|
|
392
|
+
});
|
|
368
393
|
}
|
|
369
394
|
}
|
|
370
395
|
/**
|