apify-client 2.23.5-beta.2 → 2.23.5-beta.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apify_api_error.js +37 -84
- package/dist/apify_client.js +6 -36
- package/dist/base/api_client.js +10 -56
- package/dist/bundle.js +58 -5
- package/dist/bundle.js.map +1 -1
- package/dist/http_client.js +15 -77
- package/dist/interceptors.js +5 -17
- package/dist/resource_clients/actor.d.ts +53 -3
- package/dist/resource_clients/actor.js +58 -7
- package/dist/resource_clients/actor_collection.d.ts +2 -0
- package/dist/resource_clients/dataset.js +3 -5
- package/dist/resource_clients/dataset_collection.js +1 -1
- package/dist/resource_clients/key_value_store.js +2 -2
- package/dist/resource_clients/key_value_store_collection.js +1 -1
- package/dist/resource_clients/log.js +7 -42
- package/dist/resource_clients/request_queue.js +14 -36
- package/dist/resource_clients/run.js +5 -7
- package/dist/resource_clients/task.d.ts +2 -4
- package/dist/statistics.js +13 -30
- package/dist/utils.js +6 -31
- package/package.json +7 -7
package/dist/http_client.js
CHANGED
|
@@ -11,79 +11,19 @@ const utils_1 = require("./utils");
|
|
|
11
11
|
const { version } = (0, utils_1.getVersionData)();
|
|
12
12
|
const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429;
|
|
13
13
|
class HttpClient {
|
|
14
|
+
stats;
|
|
15
|
+
maxRetries;
|
|
16
|
+
minDelayBetweenRetriesMillis;
|
|
17
|
+
userProvidedRequestInterceptors;
|
|
18
|
+
logger;
|
|
19
|
+
timeoutMillis;
|
|
20
|
+
httpAgent;
|
|
21
|
+
httpsAgent;
|
|
22
|
+
axios;
|
|
23
|
+
workflowKey;
|
|
24
|
+
nodeInitPromise;
|
|
25
|
+
userAgentSuffix;
|
|
14
26
|
constructor(options) {
|
|
15
|
-
Object.defineProperty(this, "stats", {
|
|
16
|
-
enumerable: true,
|
|
17
|
-
configurable: true,
|
|
18
|
-
writable: true,
|
|
19
|
-
value: void 0
|
|
20
|
-
});
|
|
21
|
-
Object.defineProperty(this, "maxRetries", {
|
|
22
|
-
enumerable: true,
|
|
23
|
-
configurable: true,
|
|
24
|
-
writable: true,
|
|
25
|
-
value: void 0
|
|
26
|
-
});
|
|
27
|
-
Object.defineProperty(this, "minDelayBetweenRetriesMillis", {
|
|
28
|
-
enumerable: true,
|
|
29
|
-
configurable: true,
|
|
30
|
-
writable: true,
|
|
31
|
-
value: void 0
|
|
32
|
-
});
|
|
33
|
-
Object.defineProperty(this, "userProvidedRequestInterceptors", {
|
|
34
|
-
enumerable: true,
|
|
35
|
-
configurable: true,
|
|
36
|
-
writable: true,
|
|
37
|
-
value: void 0
|
|
38
|
-
});
|
|
39
|
-
Object.defineProperty(this, "logger", {
|
|
40
|
-
enumerable: true,
|
|
41
|
-
configurable: true,
|
|
42
|
-
writable: true,
|
|
43
|
-
value: void 0
|
|
44
|
-
});
|
|
45
|
-
Object.defineProperty(this, "timeoutMillis", {
|
|
46
|
-
enumerable: true,
|
|
47
|
-
configurable: true,
|
|
48
|
-
writable: true,
|
|
49
|
-
value: void 0
|
|
50
|
-
});
|
|
51
|
-
Object.defineProperty(this, "httpAgent", {
|
|
52
|
-
enumerable: true,
|
|
53
|
-
configurable: true,
|
|
54
|
-
writable: true,
|
|
55
|
-
value: void 0
|
|
56
|
-
});
|
|
57
|
-
Object.defineProperty(this, "httpsAgent", {
|
|
58
|
-
enumerable: true,
|
|
59
|
-
configurable: true,
|
|
60
|
-
writable: true,
|
|
61
|
-
value: void 0
|
|
62
|
-
});
|
|
63
|
-
Object.defineProperty(this, "axios", {
|
|
64
|
-
enumerable: true,
|
|
65
|
-
configurable: true,
|
|
66
|
-
writable: true,
|
|
67
|
-
value: void 0
|
|
68
|
-
});
|
|
69
|
-
Object.defineProperty(this, "workflowKey", {
|
|
70
|
-
enumerable: true,
|
|
71
|
-
configurable: true,
|
|
72
|
-
writable: true,
|
|
73
|
-
value: void 0
|
|
74
|
-
});
|
|
75
|
-
Object.defineProperty(this, "nodeInitPromise", {
|
|
76
|
-
enumerable: true,
|
|
77
|
-
configurable: true,
|
|
78
|
-
writable: true,
|
|
79
|
-
value: void 0
|
|
80
|
-
});
|
|
81
|
-
Object.defineProperty(this, "userAgentSuffix", {
|
|
82
|
-
enumerable: true,
|
|
83
|
-
configurable: true,
|
|
84
|
-
writable: true,
|
|
85
|
-
value: void 0
|
|
86
|
-
});
|
|
87
27
|
const { token } = options;
|
|
88
28
|
this.stats = options.apifyClientStats;
|
|
89
29
|
this.maxRetries = options.maxRetries;
|
|
@@ -137,10 +77,9 @@ class HttpClient {
|
|
|
137
77
|
interceptors_1.responseInterceptors.forEach((i) => this.axios.interceptors.response.use(i));
|
|
138
78
|
}
|
|
139
79
|
async ensureNodeInit() {
|
|
140
|
-
var _a;
|
|
141
80
|
if (!(0, utils_1.isNode)())
|
|
142
81
|
return;
|
|
143
|
-
|
|
82
|
+
this.nodeInitPromise ??= this.initNode();
|
|
144
83
|
return this.nodeInitPromise;
|
|
145
84
|
}
|
|
146
85
|
async initNode() {
|
|
@@ -160,7 +99,7 @@ class HttpClient {
|
|
|
160
99
|
timeout: this.timeoutMillis,
|
|
161
100
|
// Keep alive timeout for free sockets (15 seconds)
|
|
162
101
|
// Node.js will close unused sockets after this period
|
|
163
|
-
keepAliveMsecs:
|
|
102
|
+
keepAliveMsecs: 15_000,
|
|
164
103
|
// Maximum number of sockets per host
|
|
165
104
|
maxSockets: 256,
|
|
166
105
|
maxFreeSockets: 256,
|
|
@@ -210,7 +149,6 @@ class HttpClient {
|
|
|
210
149
|
*/
|
|
211
150
|
_createRequestHandler(config) {
|
|
212
151
|
const makeRequest = async (stopTrying, attempt) => {
|
|
213
|
-
var _a;
|
|
214
152
|
this.stats.requests++;
|
|
215
153
|
let response;
|
|
216
154
|
const requestIsStream = (0, utils_1.isStream)(config.data);
|
|
@@ -222,7 +160,7 @@ class HttpClient {
|
|
|
222
160
|
config = { ...config, maxRedirects: 0 };
|
|
223
161
|
}
|
|
224
162
|
// Increase timeout with each attempt. Max timeout is bounded by the client timeout.
|
|
225
|
-
config.timeout = Math.min(this.timeoutMillis, (
|
|
163
|
+
config.timeout = Math.min(this.timeoutMillis, (config.timeout ?? this.timeoutMillis) * 2 ** (attempt - 1));
|
|
226
164
|
response = await this.axios.request(config);
|
|
227
165
|
if (this._isStatusOk(response.status))
|
|
228
166
|
return response;
|
package/dist/interceptors.js
CHANGED
|
@@ -14,20 +14,10 @@ const utils_1 = require("./utils");
|
|
|
14
14
|
* The properties mimic AxiosError for easier integration in HttpClient error handling.
|
|
15
15
|
*/
|
|
16
16
|
class InvalidResponseBodyError extends Error {
|
|
17
|
+
code;
|
|
18
|
+
response;
|
|
17
19
|
constructor(response, cause) {
|
|
18
20
|
super(`Response body could not be parsed.\nCause:${cause.message}`);
|
|
19
|
-
Object.defineProperty(this, "code", {
|
|
20
|
-
enumerable: true,
|
|
21
|
-
configurable: true,
|
|
22
|
-
writable: true,
|
|
23
|
-
value: void 0
|
|
24
|
-
});
|
|
25
|
-
Object.defineProperty(this, "response", {
|
|
26
|
-
enumerable: true,
|
|
27
|
-
configurable: true,
|
|
28
|
-
writable: true,
|
|
29
|
-
value: void 0
|
|
30
|
-
});
|
|
31
21
|
this.name = this.constructor.name;
|
|
32
22
|
this.code = 'invalid-response-body';
|
|
33
23
|
this.response = response;
|
|
@@ -36,7 +26,6 @@ class InvalidResponseBodyError extends Error {
|
|
|
36
26
|
}
|
|
37
27
|
exports.InvalidResponseBodyError = InvalidResponseBodyError;
|
|
38
28
|
function serializeRequest(config) {
|
|
39
|
-
var _a, _b;
|
|
40
29
|
const [defaultTransform] = axios_1.default.defaults.transformRequest;
|
|
41
30
|
// The function not only serializes data, but it also adds correct headers.
|
|
42
31
|
const data = defaultTransform(config.data, config.headers);
|
|
@@ -46,7 +35,7 @@ function serializeRequest(config) {
|
|
|
46
35
|
// it's a small price to pay. The axios default transform does a lot
|
|
47
36
|
// of body type checks and we would have to copy all of them to the resource clients.
|
|
48
37
|
if (config.stringifyFunctions) {
|
|
49
|
-
const contentTypeHeader =
|
|
38
|
+
const contentTypeHeader = config.headers?.['Content-Type'] || config.headers?.['content-type'];
|
|
50
39
|
try {
|
|
51
40
|
const { type } = content_type_1.default.parse(contentTypeHeader);
|
|
52
41
|
if (type === 'application/json' && typeof config.data === 'object') {
|
|
@@ -81,12 +70,11 @@ function stringifyWithFunctions(obj) {
|
|
|
81
70
|
});
|
|
82
71
|
}
|
|
83
72
|
async function maybeGzipRequest(config) {
|
|
84
|
-
|
|
85
|
-
if ((_a = config.headers) === null || _a === void 0 ? void 0 : _a['content-encoding'])
|
|
73
|
+
if (config.headers?.['content-encoding'])
|
|
86
74
|
return config;
|
|
87
75
|
const maybeZippedData = await (0, utils_1.maybeGzipValue)(config.data);
|
|
88
76
|
if (maybeZippedData) {
|
|
89
|
-
|
|
77
|
+
config.headers ??= {};
|
|
90
78
|
config.headers['content-encoding'] = 'gzip';
|
|
91
79
|
config.data = maybeZippedData;
|
|
92
80
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RUN_GENERAL_ACCESS } from '@apify/consts';
|
|
2
|
-
import {
|
|
2
|
+
import { ACTOR_JOB_STATUSES, ACTOR_PERMISSION_LEVEL, META_ORIGINS } from '@apify/consts';
|
|
3
3
|
import { Log } from '@apify/log';
|
|
4
4
|
import type { ApiClientSubResourceOptions } from '../base/api_client';
|
|
5
5
|
import { ResourceClient } from '../base/resource_client';
|
|
@@ -13,6 +13,7 @@ import { RunClient } from './run';
|
|
|
13
13
|
import { RunCollectionClient } from './run_collection';
|
|
14
14
|
import type { WebhookUpdateData } from './webhook';
|
|
15
15
|
import { WebhookCollectionClient } from './webhook_collection';
|
|
16
|
+
import type { ValueOf } from 'type-fest';
|
|
16
17
|
/**
|
|
17
18
|
* Client for managing a specific Actor.
|
|
18
19
|
*
|
|
@@ -132,6 +133,37 @@ export declare class ActorClient extends ResourceClient {
|
|
|
132
133
|
* ```
|
|
133
134
|
*/
|
|
134
135
|
call(input?: unknown, options?: ActorCallOptions): Promise<ActorRun>;
|
|
136
|
+
/**
|
|
137
|
+
* Validates the provided input for the Actor against its input schema.
|
|
138
|
+
*
|
|
139
|
+
* Sends the input to the API, which validates it against the Actor's input schema without
|
|
140
|
+
* starting a run. If the input is valid, the method resolves with `true`. If the input is
|
|
141
|
+
* invalid, the API responds with an error that is thrown as an `ApifyApiError` describing the
|
|
142
|
+
* validation problem.
|
|
143
|
+
*
|
|
144
|
+
* @param input - Input to validate against the Actor's input schema. Can be any JSON-serializable
|
|
145
|
+
* value (object, array, string, number). If `contentType` is specified in options,
|
|
146
|
+
* input should be a string or Buffer.
|
|
147
|
+
* @param options - Validation options
|
|
148
|
+
* @param options.build - Tag or number of the build whose input schema the input is validated against
|
|
149
|
+
* (e.g., `'latest'` or `'1.2.345'`). If not provided, uses the default build.
|
|
150
|
+
* @param options.contentType - Content type of the input. If specified, input must be a string or Buffer.
|
|
151
|
+
* @returns `true` if the input is valid. Invalid input causes the underlying API call to throw an `ApifyApiError`.
|
|
152
|
+
* @see https://docs.apify.com/api/v2/act-input-validate-post
|
|
153
|
+
*
|
|
154
|
+
* @example
|
|
155
|
+
* ```javascript
|
|
156
|
+
* // Validate input against the default build's input schema
|
|
157
|
+
* const isValid = await client.actor('my-actor').validateInput({ url: 'https://example.com' });
|
|
158
|
+
*
|
|
159
|
+
* // Validate against a specific build
|
|
160
|
+
* const isValid = await client.actor('my-actor').validateInput(
|
|
161
|
+
* { url: 'https://example.com' },
|
|
162
|
+
* { build: 'beta' },
|
|
163
|
+
* );
|
|
164
|
+
* ```
|
|
165
|
+
*/
|
|
166
|
+
validateInput(input?: unknown, options?: ActorValidateInputOptions): Promise<boolean>;
|
|
135
167
|
/**
|
|
136
168
|
* Builds the Actor.
|
|
137
169
|
*
|
|
@@ -462,7 +494,7 @@ export interface ActorRunListItem {
|
|
|
462
494
|
actorTaskId?: string;
|
|
463
495
|
startedAt: Date;
|
|
464
496
|
finishedAt: Date;
|
|
465
|
-
status: (typeof
|
|
497
|
+
status: (typeof ACTOR_JOB_STATUSES)[keyof typeof ACTOR_JOB_STATUSES];
|
|
466
498
|
meta: ActorRunMeta;
|
|
467
499
|
buildId: string;
|
|
468
500
|
buildNumber: string;
|
|
@@ -585,6 +617,23 @@ export interface ActorRunOptions {
|
|
|
585
617
|
maxTotalChargeUsd?: number;
|
|
586
618
|
restartOnError?: boolean;
|
|
587
619
|
}
|
|
620
|
+
/**
|
|
621
|
+
* Options for validating an Actor input.
|
|
622
|
+
*/
|
|
623
|
+
export interface ActorValidateInputOptions {
|
|
624
|
+
/**
|
|
625
|
+
* Tag or number of the Actor build whose input schema the input is validated against
|
|
626
|
+
* (e.g. `beta` or `1.2.345`). If not provided, the default build is used.
|
|
627
|
+
*/
|
|
628
|
+
build?: string;
|
|
629
|
+
/**
|
|
630
|
+
* Content type for the `input`. If not specified,
|
|
631
|
+
* `input` is expected to be an object that will be stringified to JSON and content type set to
|
|
632
|
+
* `application/json; charset=utf-8`. If `options.contentType` is specified, then `input` must be a
|
|
633
|
+
* `String` or `Buffer`.
|
|
634
|
+
*/
|
|
635
|
+
contentType?: string;
|
|
636
|
+
}
|
|
588
637
|
/**
|
|
589
638
|
* Options for building an Actor.
|
|
590
639
|
*/
|
|
@@ -598,7 +647,8 @@ export interface ActorBuildOptions {
|
|
|
598
647
|
* Options for filtering the last run of an Actor.
|
|
599
648
|
*/
|
|
600
649
|
export interface ActorLastRunOptions {
|
|
601
|
-
status?:
|
|
650
|
+
status?: ValueOf<typeof ACTOR_JOB_STATUSES>;
|
|
651
|
+
origin?: ValueOf<typeof META_ORIGINS>;
|
|
602
652
|
}
|
|
603
653
|
/**
|
|
604
654
|
* Actor definition from the `.actor/actor.json` file.
|
|
@@ -140,9 +140,6 @@ class ActorClient extends resource_client_1.ResourceClient {
|
|
|
140
140
|
params: this._params(params),
|
|
141
141
|
// Apify internal property. Tells the request serialization interceptor
|
|
142
142
|
// to stringify functions to JSON, instead of omitting them.
|
|
143
|
-
// TODO: remove this ts-expect-error once we migrate HttpClient to TS and define Apify
|
|
144
|
-
// extension of Axios configs
|
|
145
|
-
// @ts-expect-error Apify extension
|
|
146
143
|
stringifyFunctions: true,
|
|
147
144
|
};
|
|
148
145
|
if (options.contentType) {
|
|
@@ -212,15 +209,69 @@ class ActorClient extends resource_client_1.ResourceClient {
|
|
|
212
209
|
// Creating a new instance of RunClient here would only allow
|
|
213
210
|
// setting it up as a nested route under actor API.
|
|
214
211
|
const newRunClient = this.apifyClient.run(id);
|
|
215
|
-
const streamedLog = await newRunClient.getStreamedLog({ toLog: options
|
|
216
|
-
streamedLog
|
|
212
|
+
const streamedLog = await newRunClient.getStreamedLog({ toLog: options?.log });
|
|
213
|
+
streamedLog?.start();
|
|
217
214
|
return this.apifyClient
|
|
218
215
|
.run(id)
|
|
219
216
|
.waitForFinish({ waitSecs })
|
|
220
217
|
.finally(async () => {
|
|
221
|
-
await
|
|
218
|
+
await streamedLog?.stop();
|
|
222
219
|
});
|
|
223
220
|
}
|
|
221
|
+
/**
|
|
222
|
+
* Validates the provided input for the Actor against its input schema.
|
|
223
|
+
*
|
|
224
|
+
* Sends the input to the API, which validates it against the Actor's input schema without
|
|
225
|
+
* starting a run. If the input is valid, the method resolves with `true`. If the input is
|
|
226
|
+
* invalid, the API responds with an error that is thrown as an `ApifyApiError` describing the
|
|
227
|
+
* validation problem.
|
|
228
|
+
*
|
|
229
|
+
* @param input - Input to validate against the Actor's input schema. Can be any JSON-serializable
|
|
230
|
+
* value (object, array, string, number). If `contentType` is specified in options,
|
|
231
|
+
* input should be a string or Buffer.
|
|
232
|
+
* @param options - Validation options
|
|
233
|
+
* @param options.build - Tag or number of the build whose input schema the input is validated against
|
|
234
|
+
* (e.g., `'latest'` or `'1.2.345'`). If not provided, uses the default build.
|
|
235
|
+
* @param options.contentType - Content type of the input. If specified, input must be a string or Buffer.
|
|
236
|
+
* @returns `true` if the input is valid. Invalid input causes the underlying API call to throw an `ApifyApiError`.
|
|
237
|
+
* @see https://docs.apify.com/api/v2/act-input-validate-post
|
|
238
|
+
*
|
|
239
|
+
* @example
|
|
240
|
+
* ```javascript
|
|
241
|
+
* // Validate input against the default build's input schema
|
|
242
|
+
* const isValid = await client.actor('my-actor').validateInput({ url: 'https://example.com' });
|
|
243
|
+
*
|
|
244
|
+
* // Validate against a specific build
|
|
245
|
+
* const isValid = await client.actor('my-actor').validateInput(
|
|
246
|
+
* { url: 'https://example.com' },
|
|
247
|
+
* { build: 'beta' },
|
|
248
|
+
* );
|
|
249
|
+
* ```
|
|
250
|
+
*/
|
|
251
|
+
async validateInput(input, options = {}) {
|
|
252
|
+
// input can be anything, so no point in validating it. E.g. if you set content-type to application/pdf
|
|
253
|
+
// then it will process input as a buffer.
|
|
254
|
+
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
255
|
+
build: ow_1.default.optional.string,
|
|
256
|
+
contentType: ow_1.default.optional.string,
|
|
257
|
+
}));
|
|
258
|
+
const request = {
|
|
259
|
+
url: this._url('validate-input'),
|
|
260
|
+
method: 'POST',
|
|
261
|
+
data: input,
|
|
262
|
+
params: this._params({ build: options.build }),
|
|
263
|
+
// Apify internal property. Tells the request serialization interceptor
|
|
264
|
+
// to stringify functions to JSON, instead of omitting them.
|
|
265
|
+
stringifyFunctions: true,
|
|
266
|
+
};
|
|
267
|
+
if (options.contentType) {
|
|
268
|
+
request.headers = {
|
|
269
|
+
'content-type': options.contentType,
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
const response = await this.httpClient.call(request);
|
|
273
|
+
return response.data.valid;
|
|
274
|
+
}
|
|
224
275
|
/**
|
|
225
276
|
* Builds the Actor.
|
|
226
277
|
*
|
|
@@ -326,7 +377,7 @@ class ActorClient extends resource_client_1.ResourceClient {
|
|
|
326
377
|
*/
|
|
327
378
|
lastRun(options = {}) {
|
|
328
379
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
329
|
-
status: ow_1.default.optional.string.oneOf(Object.values(consts_1.
|
|
380
|
+
status: ow_1.default.optional.string.oneOf(Object.values(consts_1.ACTOR_JOB_STATUSES)),
|
|
330
381
|
origin: ow_1.default.optional.string.oneOf(Object.values(consts_1.META_ORIGINS)),
|
|
331
382
|
}));
|
|
332
383
|
return new run_1.RunClient(this._subResourceOptions({
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { ACTOR_PERMISSION_LEVEL } from '@apify/consts';
|
|
1
2
|
import type { ApiClientSubResourceOptions } from '../base/api_client';
|
|
2
3
|
import { ResourceCollectionClient } from '../base/resource_collection_client';
|
|
3
4
|
import type { PaginatedIterator, PaginatedList, PaginationOptions } from '../utils';
|
|
@@ -95,4 +96,5 @@ export interface ActorCollectionCreateOptions {
|
|
|
95
96
|
actorStandby?: ActorStandby & {
|
|
96
97
|
isEnabled: boolean;
|
|
97
98
|
};
|
|
99
|
+
actorPermissionLevel?: ACTOR_PERMISSION_LEVEL;
|
|
98
100
|
}
|
|
@@ -134,14 +134,13 @@ class DatasetClient extends resource_client_1.ResourceClient {
|
|
|
134
134
|
signature: ow_1.default.optional.string,
|
|
135
135
|
}));
|
|
136
136
|
const fetchItems = async (datasetListOptions = {}) => {
|
|
137
|
-
var _a;
|
|
138
137
|
const response = await this.httpClient.call({
|
|
139
138
|
url: this._url('items'),
|
|
140
139
|
method: 'GET',
|
|
141
140
|
params: this._params(datasetListOptions),
|
|
142
141
|
timeout: resource_client_1.DEFAULT_TIMEOUT_MILLIS,
|
|
143
142
|
});
|
|
144
|
-
return this._createPaginationList(response,
|
|
143
|
+
return this._createPaginationList(response, datasetListOptions.desc ?? false);
|
|
145
144
|
};
|
|
146
145
|
return this._listPaginatedFromCallback(fetchItems, options);
|
|
147
146
|
}
|
|
@@ -339,7 +338,7 @@ class DatasetClient extends resource_client_1.ResourceClient {
|
|
|
339
338
|
const dataset = await this.get();
|
|
340
339
|
const { expiresInSecs, ...queryOptions } = options;
|
|
341
340
|
let createdItemsPublicUrl = new URL(this._publicUrl('items'));
|
|
342
|
-
if (dataset
|
|
341
|
+
if (dataset?.urlSigningSecretKey) {
|
|
343
342
|
const signature = await (0, utilities_1.createStorageContentSignatureAsync)({
|
|
344
343
|
resourceId: dataset.id,
|
|
345
344
|
urlSigningSecretKey: dataset.urlSigningSecretKey,
|
|
@@ -351,7 +350,6 @@ class DatasetClient extends resource_client_1.ResourceClient {
|
|
|
351
350
|
return createdItemsPublicUrl.toString();
|
|
352
351
|
}
|
|
353
352
|
_createPaginationList(response, userProvidedDesc) {
|
|
354
|
-
var _a;
|
|
355
353
|
return {
|
|
356
354
|
items: response.data,
|
|
357
355
|
total: Number(response.headers['x-apify-pagination-total']),
|
|
@@ -359,7 +357,7 @@ class DatasetClient extends resource_client_1.ResourceClient {
|
|
|
359
357
|
count: response.data.length, // because x-apify-pagination-count returns invalid values when hidden/empty items are skipped
|
|
360
358
|
limit: Number(response.headers['x-apify-pagination-limit']), // API returns 999999999999 when no limit is used
|
|
361
359
|
// TODO: Replace this once https://github.com/apify/apify-core/issues/3503 is solved
|
|
362
|
-
desc: JSON.parse(
|
|
360
|
+
desc: JSON.parse(response.headers['x-apify-pagination-desc'] ?? userProvidedDesc),
|
|
363
361
|
};
|
|
364
362
|
}
|
|
365
363
|
}
|
|
@@ -75,7 +75,7 @@ class DatasetCollectionClient extends resource_collection_client_1.ResourceColle
|
|
|
75
75
|
*/
|
|
76
76
|
async getOrCreate(name, options) {
|
|
77
77
|
(0, ow_1.default)(name, ow_1.default.optional.string);
|
|
78
|
-
(0, ow_1.default)(options
|
|
78
|
+
(0, ow_1.default)(options?.schema, ow_1.default.optional.object); // TODO: Add schema validatioon
|
|
79
79
|
return this._getOrCreate(name, options);
|
|
80
80
|
}
|
|
81
81
|
}
|
|
@@ -174,7 +174,7 @@ class KeyValueStoreClient extends resource_client_1.ResourceClient {
|
|
|
174
174
|
(0, ow_1.default)(key, ow_1.default.string.nonEmpty);
|
|
175
175
|
const store = await this.get();
|
|
176
176
|
const recordPublicUrl = new URL(this._publicUrl(`records/${key}`));
|
|
177
|
-
if (store
|
|
177
|
+
if (store?.urlSigningSecretKey) {
|
|
178
178
|
const signature = await (0, utilities_1.createHmacSignatureAsync)(store.urlSigningSecretKey, key);
|
|
179
179
|
recordPublicUrl.searchParams.append('signature', signature);
|
|
180
180
|
}
|
|
@@ -213,7 +213,7 @@ class KeyValueStoreClient extends resource_client_1.ResourceClient {
|
|
|
213
213
|
const store = await this.get();
|
|
214
214
|
const { expiresInSecs, ...queryOptions } = options;
|
|
215
215
|
let createdPublicKeysUrl = new URL(this._publicUrl('keys'));
|
|
216
|
-
if (store
|
|
216
|
+
if (store?.urlSigningSecretKey) {
|
|
217
217
|
const signature = await (0, utilities_1.createStorageContentSignatureAsync)({
|
|
218
218
|
resourceId: store.id,
|
|
219
219
|
urlSigningSecretKey: store.urlSigningSecretKey,
|
|
@@ -75,7 +75,7 @@ class KeyValueStoreCollectionClient extends resource_collection_client_1.Resourc
|
|
|
75
75
|
*/
|
|
76
76
|
async getOrCreate(name, options) {
|
|
77
77
|
(0, ow_1.default)(name, ow_1.default.optional.string);
|
|
78
|
-
(0, ow_1.default)(options
|
|
78
|
+
(0, ow_1.default)(options?.schema, ow_1.default.optional.object); // TODO: Add schema validatioon
|
|
79
79
|
return this._getOrCreate(name, options);
|
|
80
80
|
}
|
|
81
81
|
}
|
|
@@ -123,49 +123,14 @@ exports.LoggerActorRedirect = LoggerActorRedirect;
|
|
|
123
123
|
* Helper class for redirecting streamed Actor logs to another log.
|
|
124
124
|
*/
|
|
125
125
|
class StreamedLog {
|
|
126
|
+
destinationLog;
|
|
127
|
+
streamBuffer = [];
|
|
128
|
+
splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g;
|
|
129
|
+
relevancyTimeLimit;
|
|
130
|
+
logClient;
|
|
131
|
+
streamingTask = null;
|
|
132
|
+
stopLogging = false;
|
|
126
133
|
constructor(options) {
|
|
127
|
-
Object.defineProperty(this, "destinationLog", {
|
|
128
|
-
enumerable: true,
|
|
129
|
-
configurable: true,
|
|
130
|
-
writable: true,
|
|
131
|
-
value: void 0
|
|
132
|
-
});
|
|
133
|
-
Object.defineProperty(this, "streamBuffer", {
|
|
134
|
-
enumerable: true,
|
|
135
|
-
configurable: true,
|
|
136
|
-
writable: true,
|
|
137
|
-
value: []
|
|
138
|
-
});
|
|
139
|
-
Object.defineProperty(this, "splitMarker", {
|
|
140
|
-
enumerable: true,
|
|
141
|
-
configurable: true,
|
|
142
|
-
writable: true,
|
|
143
|
-
value: /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g
|
|
144
|
-
});
|
|
145
|
-
Object.defineProperty(this, "relevancyTimeLimit", {
|
|
146
|
-
enumerable: true,
|
|
147
|
-
configurable: true,
|
|
148
|
-
writable: true,
|
|
149
|
-
value: void 0
|
|
150
|
-
});
|
|
151
|
-
Object.defineProperty(this, "logClient", {
|
|
152
|
-
enumerable: true,
|
|
153
|
-
configurable: true,
|
|
154
|
-
writable: true,
|
|
155
|
-
value: void 0
|
|
156
|
-
});
|
|
157
|
-
Object.defineProperty(this, "streamingTask", {
|
|
158
|
-
enumerable: true,
|
|
159
|
-
configurable: true,
|
|
160
|
-
writable: true,
|
|
161
|
-
value: null
|
|
162
|
-
});
|
|
163
|
-
Object.defineProperty(this, "stopLogging", {
|
|
164
|
-
enumerable: true,
|
|
165
|
-
configurable: true,
|
|
166
|
-
writable: true,
|
|
167
|
-
value: false
|
|
168
|
-
});
|
|
169
134
|
const { toLog, logClient, fromStart = true } = options;
|
|
170
135
|
this.destinationLog = toLog;
|
|
171
136
|
this.logClient = logClient;
|