apify 4.0.0-beta.29 → 4.0.0-beta.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actor.js +9 -8
- package/dist/configuration.d.ts +1 -1
- package/dist/proxy_configuration.js +3 -3
- package/dist/utils.d.ts +1 -21
- package/dist/utils.js +1 -80
- package/package.json +9 -8
package/dist/actor.js
CHANGED
|
@@ -7,6 +7,7 @@ import { ACTOR_ENV_VARS, ACTOR_EVENT_NAMES, APIFY_ENV_VARS, INTEGER_ENV_VARS, }
|
|
|
7
7
|
import { decryptInputSecrets } from '@apify/input_secrets';
|
|
8
8
|
import log from '@apify/log';
|
|
9
9
|
import { addTimeoutToPromise } from '@apify/timeout';
|
|
10
|
+
import { parseArgument } from '@apify/validations';
|
|
10
11
|
import { ApifyStorageBackend, pushDataChargingContext, USES_PUSH_DATA_INTERCEPTION, } from './apify_storage_backend.js';
|
|
11
12
|
import { ChargingManager, pushDataAndCharge } from './charging.js';
|
|
12
13
|
import { Configuration } from './configuration.js';
|
|
@@ -14,7 +15,7 @@ import { getDefaultsFromInputSchema, noActorInputSchemaDefinedMarker, readInputS
|
|
|
14
15
|
import { PlatformEventManager } from './platform_event_manager.js';
|
|
15
16
|
import { ProxyConfiguration } from './proxy_configuration.js';
|
|
16
17
|
import { openStorage } from './storage.js';
|
|
17
|
-
import { checkCrawleeVersion, getSystemInfo, isNonEmptyObject, printOutdatedSdkWarning, snakeCaseToCamelCase,
|
|
18
|
+
import { checkCrawleeVersion, getSystemInfo, isNonEmptyObject, printOutdatedSdkWarning, snakeCaseToCamelCase, } from './utils.js';
|
|
18
19
|
/**
|
|
19
20
|
* Exit codes for the Actor process.
|
|
20
21
|
* The error codes must be in the range 1-128, to avoid collision with signal exits
|
|
@@ -546,7 +547,7 @@ export class Actor {
|
|
|
546
547
|
* @ignore
|
|
547
548
|
*/
|
|
548
549
|
async addWebhook(options) {
|
|
549
|
-
|
|
550
|
+
parseArgument(options, z
|
|
550
551
|
.object({
|
|
551
552
|
eventTypes: z.array(z.string()),
|
|
552
553
|
requestUrl: z.string(),
|
|
@@ -559,7 +560,7 @@ export class Actor {
|
|
|
559
560
|
shouldInterpolateStrings: z.boolean().optional(),
|
|
560
561
|
isApifyIntegration: z.boolean().optional(),
|
|
561
562
|
})
|
|
562
|
-
.strict()
|
|
563
|
+
.strict());
|
|
563
564
|
if (!this.isAtHome()) {
|
|
564
565
|
log.warning('Actor.addWebhook() is only supported when running on the Apify platform. The webhook will not be invoked.');
|
|
565
566
|
return undefined;
|
|
@@ -585,8 +586,8 @@ export class Actor {
|
|
|
585
586
|
*/
|
|
586
587
|
async setStatusMessage(statusMessage, options) {
|
|
587
588
|
const { isStatusMessageTerminal, level } = options || {};
|
|
588
|
-
|
|
589
|
-
|
|
589
|
+
parseArgument(statusMessage, z.string());
|
|
590
|
+
parseArgument(isStatusMessageTerminal, z.boolean().optional());
|
|
590
591
|
this._ensureActorInit('setStatusMessage');
|
|
591
592
|
const loggedStatusMessage = `[Status message]: ${statusMessage}`;
|
|
592
593
|
switch (level) {
|
|
@@ -679,7 +680,7 @@ export class Actor {
|
|
|
679
680
|
* @ignore
|
|
680
681
|
*/
|
|
681
682
|
async openDataset(datasetIdOrName, options = {}) {
|
|
682
|
-
|
|
683
|
+
parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
|
|
683
684
|
this._ensureActorInit('openDataset');
|
|
684
685
|
return this._openStorage(Dataset, datasetIdOrName, options);
|
|
685
686
|
}
|
|
@@ -826,7 +827,7 @@ export class Actor {
|
|
|
826
827
|
* @ignore
|
|
827
828
|
*/
|
|
828
829
|
async openKeyValueStore(storeIdOrName, options = {}) {
|
|
829
|
-
|
|
830
|
+
parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
|
|
830
831
|
this._ensureActorInit('openKeyValueStore');
|
|
831
832
|
return this._openStorage(KeyValueStore, storeIdOrName, options);
|
|
832
833
|
}
|
|
@@ -849,7 +850,7 @@ export class Actor {
|
|
|
849
850
|
* @ignore
|
|
850
851
|
*/
|
|
851
852
|
async openRequestQueue(queueIdOrName, options = {}) {
|
|
852
|
-
|
|
853
|
+
parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
|
|
853
854
|
this._ensureActorInit('openRequestQueue');
|
|
854
855
|
const queue = await this._openStorage(RequestQueue, queueIdOrName, options);
|
|
855
856
|
return queue;
|
package/dist/configuration.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ export declare const apifyConfigFields: {
|
|
|
5
5
|
maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
|
|
6
6
|
internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber>>>;
|
|
7
7
|
systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
|
|
8
|
-
logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof import("@
|
|
8
|
+
logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof import("@crawlee/core").LogLevel>>>>;
|
|
9
9
|
persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
|
|
10
10
|
storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
11
11
|
containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean>>>;
|
|
@@ -6,9 +6,9 @@ import { z } from 'zod';
|
|
|
6
6
|
import { APIFY_ENV_VARS, APIFY_PROXY_VALUE_REGEX } from '@apify/consts';
|
|
7
7
|
import defaultLog from '@apify/log';
|
|
8
8
|
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
9
|
+
import { parseArgument } from '@apify/validations';
|
|
9
10
|
import { Actor } from './actor.js';
|
|
10
11
|
import { Configuration } from './configuration.js';
|
|
11
|
-
import { validate } from './utils.js';
|
|
12
12
|
const CHECK_ACCESS_REQUEST_TIMEOUT_MILLIS = 4_000;
|
|
13
13
|
const CHECK_ACCESS_MAX_ATTEMPTS = 2;
|
|
14
14
|
const COUNTRY_CODE_REGEX = /^[A-Z]{2}$/;
|
|
@@ -72,7 +72,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
72
72
|
['validateRequired']: false,
|
|
73
73
|
});
|
|
74
74
|
this.configuration = configuration;
|
|
75
|
-
|
|
75
|
+
parseArgument(rest, z
|
|
76
76
|
.object({
|
|
77
77
|
groups: z.array(z.string().regex(APIFY_PROXY_VALUE_REGEX)).optional(),
|
|
78
78
|
apifyProxyGroups: z.array(z.string().regex(APIFY_PROXY_VALUE_REGEX)).optional(),
|
|
@@ -82,7 +82,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
82
82
|
apifyProxySubdivision: z.string().regex(SUBDIVISION_CODE_REGEX).optional(),
|
|
83
83
|
password: z.string().optional(),
|
|
84
84
|
})
|
|
85
|
-
.strict()
|
|
85
|
+
.strict());
|
|
86
86
|
const { groups = [], apifyProxyGroups = [], countryCode, apifyProxyCountry, subdivisionCode, apifyProxySubdivision, password = configuration.proxyPassword, } = options;
|
|
87
87
|
const groupsToUse = groups.length ? groups : apifyProxyGroups;
|
|
88
88
|
const countryCodeToUse = countryCode || apifyProxyCountry;
|
package/dist/utils.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
export { ArgumentValidationError } from '@apify/validations';
|
|
2
2
|
/**
|
|
3
3
|
* Returns `true` for a plain, non-empty object (not `null`, not an array).
|
|
4
4
|
* Mirrors the `ow.object.nonEmpty` predicate the SDK used previously.
|
|
@@ -10,26 +10,6 @@ export declare function isNonEmptyObject(value: unknown): value is Record<string
|
|
|
10
10
|
* @internal
|
|
11
11
|
*/
|
|
12
12
|
export declare function snakeCaseToCamelCase(snakeCaseStr: string): string;
|
|
13
|
-
/**
|
|
14
|
-
* Error thrown when an argument fails validation (e.g. by `Actor.addWebhook()`
|
|
15
|
-
* or the `ProxyConfiguration` constructor).
|
|
16
|
-
*
|
|
17
|
-
* Its `message` is a human-readable sentence naming the offending field and the
|
|
18
|
-
* value it received (see {@link formatZodError}) — not a raw JSON dump. The
|
|
19
|
-
* structured zod {@link https://zod.dev | zod} issues are available on `issues`
|
|
20
|
-
* (and the original `ZodError` on `cause`) for programmatic inspection.
|
|
21
|
-
*/
|
|
22
|
-
export declare class ArgumentValidationError extends Error {
|
|
23
|
-
/** Structured issues from the underlying schema check. */
|
|
24
|
-
readonly issues: z.ZodError['issues'];
|
|
25
|
-
constructor(error: z.ZodError, value: unknown);
|
|
26
|
-
}
|
|
27
|
-
/**
|
|
28
|
-
* Validates `value` against a zod `schema`, returning the parsed value, or
|
|
29
|
-
* throwing an {@link ArgumentValidationError} if it doesn't match.
|
|
30
|
-
* @internal
|
|
31
|
-
*/
|
|
32
|
-
export declare function validate<Schema extends z.ZodType>(schema: Schema, value: unknown): z.infer<Schema>;
|
|
33
13
|
/**
|
|
34
14
|
* Gets info about system, node version and apify package version.
|
|
35
15
|
* @internal
|
package/dist/utils.js
CHANGED
|
@@ -11,6 +11,7 @@ import { APIFY_ENV_VARS } from '@apify/consts';
|
|
|
11
11
|
import log from '@apify/log';
|
|
12
12
|
// @ts-ignore if we enable resolveJsonModule, we end up with `src` folder in `dist`
|
|
13
13
|
import apifyPkgJson from '../package.json' with { type: 'json' };
|
|
14
|
+
export { ArgumentValidationError } from '@apify/validations';
|
|
14
15
|
const require = createRequire(import.meta.url);
|
|
15
16
|
/**
|
|
16
17
|
* Returns `true` for a plain, non-empty object (not `null`, not an array).
|
|
@@ -31,86 +32,6 @@ export function snakeCaseToCamelCase(snakeCaseStr) {
|
|
|
31
32
|
.map((part, index) => (index > 0 ? part.charAt(0).toUpperCase() + part.slice(1) : part))
|
|
32
33
|
.join('');
|
|
33
34
|
}
|
|
34
|
-
/** Formats a zod issue path like `groups[0]` or `countryCode`. */
|
|
35
|
-
function formatIssuePath(path) {
|
|
36
|
-
let out = '';
|
|
37
|
-
for (const key of path) {
|
|
38
|
-
if (typeof key === 'number')
|
|
39
|
-
out += `[${key}]`;
|
|
40
|
-
else
|
|
41
|
-
out += out ? `.${String(key)}` : String(key);
|
|
42
|
-
}
|
|
43
|
-
return out;
|
|
44
|
-
}
|
|
45
|
-
/** Reads the value at `path` from the validated input, to include in the error. */
|
|
46
|
-
function valueAtPath(root, path) {
|
|
47
|
-
let current = root;
|
|
48
|
-
for (const key of path) {
|
|
49
|
-
if (current === null || typeof current !== 'object')
|
|
50
|
-
return undefined;
|
|
51
|
-
current = current[key];
|
|
52
|
-
}
|
|
53
|
-
return current;
|
|
54
|
-
}
|
|
55
|
-
/** Renders a primitive received value for an error; skips objects/Dates (noisy). */
|
|
56
|
-
function describeReceived(value) {
|
|
57
|
-
switch (typeof value) {
|
|
58
|
-
case 'string':
|
|
59
|
-
return value;
|
|
60
|
-
case 'number':
|
|
61
|
-
case 'boolean':
|
|
62
|
-
case 'bigint':
|
|
63
|
-
return String(value);
|
|
64
|
-
default:
|
|
65
|
-
return undefined;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
/**
|
|
69
|
-
* Formats a `ZodError` as a plain, human-readable message that names the
|
|
70
|
-
* offending field *and* the value it received (e.g. ``must match pattern
|
|
71
|
-
* /^[A-Z]{2}$/ at `countryCode`, got `CZE` ``) — closer to the old `ow` errors
|
|
72
|
-
* than zod's default, which omits the received value.
|
|
73
|
-
*/
|
|
74
|
-
function formatZodError(error, root) {
|
|
75
|
-
return error.issues
|
|
76
|
-
.map((issue) => {
|
|
77
|
-
const location = issue.path.length ? ` at \`${formatIssuePath(issue.path)}\`` : '';
|
|
78
|
-
const received = describeReceived(valueAtPath(root, issue.path));
|
|
79
|
-
const got = received === undefined ? '' : `, got \`${received}\``;
|
|
80
|
-
return `${issue.message}${location}${got}`;
|
|
81
|
-
})
|
|
82
|
-
.join('\n');
|
|
83
|
-
}
|
|
84
|
-
/**
|
|
85
|
-
* Error thrown when an argument fails validation (e.g. by `Actor.addWebhook()`
|
|
86
|
-
* or the `ProxyConfiguration` constructor).
|
|
87
|
-
*
|
|
88
|
-
* Its `message` is a human-readable sentence naming the offending field and the
|
|
89
|
-
* value it received (see {@link formatZodError}) — not a raw JSON dump. The
|
|
90
|
-
* structured zod {@link https://zod.dev | zod} issues are available on `issues`
|
|
91
|
-
* (and the original `ZodError` on `cause`) for programmatic inspection.
|
|
92
|
-
*/
|
|
93
|
-
export class ArgumentValidationError extends Error {
|
|
94
|
-
/** Structured issues from the underlying schema check. */
|
|
95
|
-
issues;
|
|
96
|
-
constructor(error, value) {
|
|
97
|
-
super(formatZodError(error, value), { cause: error });
|
|
98
|
-
this.name = 'ArgumentValidationError';
|
|
99
|
-
this.issues = error.issues;
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
/**
|
|
103
|
-
* Validates `value` against a zod `schema`, returning the parsed value, or
|
|
104
|
-
* throwing an {@link ArgumentValidationError} if it doesn't match.
|
|
105
|
-
* @internal
|
|
106
|
-
*/
|
|
107
|
-
export function validate(schema, value) {
|
|
108
|
-
const result = schema.safeParse(value);
|
|
109
|
-
if (!result.success) {
|
|
110
|
-
throw new ArgumentValidationError(result.error, value);
|
|
111
|
-
}
|
|
112
|
-
return result.data;
|
|
113
|
-
}
|
|
114
35
|
/**
|
|
115
36
|
* Gets info about system, node version and apify package version.
|
|
116
37
|
* @internal
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "apify",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.30",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -53,13 +53,14 @@
|
|
|
53
53
|
]
|
|
54
54
|
},
|
|
55
55
|
"dependencies": {
|
|
56
|
-
"@apify/consts": "^
|
|
57
|
-
"@apify/datastructures": "^
|
|
58
|
-
"@apify/input_secrets": "^
|
|
59
|
-
"@apify/log": "^
|
|
60
|
-
"@apify/pseudo_url": "^
|
|
61
|
-
"@apify/timeout": "^0.
|
|
62
|
-
"@apify/utilities": "^
|
|
56
|
+
"@apify/consts": "^3.0.1",
|
|
57
|
+
"@apify/datastructures": "^3.0.1",
|
|
58
|
+
"@apify/input_secrets": "^2.0.1",
|
|
59
|
+
"@apify/log": "^3.0.1",
|
|
60
|
+
"@apify/pseudo_url": "^3.0.1",
|
|
61
|
+
"@apify/timeout": "^1.0.1",
|
|
62
|
+
"@apify/utilities": "^3.0.1",
|
|
63
|
+
"@apify/validations": "^1.0.1",
|
|
63
64
|
"@crawlee/core": "^4.0.0-rc.0",
|
|
64
65
|
"@crawlee/types": "^4.0.0-rc.0",
|
|
65
66
|
"@crawlee/utils": "^4.0.0-rc.0",
|