apify-schema-tools 3.1.0 → 3.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.node-version +1 -1
- package/CHANGELOG.md +7 -1
- package/biome.json +8 -2
- package/dist/apify-schema-tools.js +12 -9
- package/dist/apify-schema-tools.js.map +1 -1
- package/dist/apify.d.ts +1 -1
- package/dist/apify.d.ts.map +1 -1
- package/dist/apify.js +19 -5
- package/dist/apify.js.map +1 -1
- package/dist/cli/check.d.ts +5 -0
- package/dist/cli/check.d.ts.map +1 -0
- package/dist/cli/check.js +86 -0
- package/dist/cli/check.js.map +1 -0
- package/dist/cli/init.d.ts +5 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +92 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/cli/sync.d.ts +5 -0
- package/dist/cli/sync.d.ts.map +1 -0
- package/dist/cli/sync.js +112 -0
- package/dist/cli/sync.js.map +1 -0
- package/dist/configuration.d.ts +14 -5
- package/dist/configuration.d.ts.map +1 -1
- package/dist/configuration.js +9 -5
- package/dist/configuration.js.map +1 -1
- package/dist/main.d.ts +4 -0
- package/dist/main.d.ts.map +1 -0
- package/dist/main.js +19 -0
- package/dist/main.js.map +1 -0
- package/dist/middle-schema/compare-schemas.d.ts +3 -0
- package/dist/middle-schema/compare-schemas.d.ts.map +1 -0
- package/dist/middle-schema/compare-schemas.js +90 -0
- package/dist/middle-schema/compare-schemas.js.map +1 -0
- package/dist/middle-schema/generate-typescript.d.ts +7 -0
- package/dist/middle-schema/generate-typescript.d.ts.map +1 -0
- package/dist/middle-schema/generate-typescript.js +70 -0
- package/dist/middle-schema/generate-typescript.js.map +1 -0
- package/dist/middle-schema/parse-json-schema.d.ts +4 -0
- package/dist/middle-schema/parse-json-schema.d.ts.map +1 -0
- package/dist/middle-schema/parse-json-schema.js +65 -0
- package/dist/middle-schema/parse-json-schema.js.map +1 -0
- package/dist/middle-schema/parse-typescript.d.ts +4 -0
- package/dist/middle-schema/parse-typescript.d.ts.map +1 -0
- package/dist/middle-schema/parse-typescript.js +199 -0
- package/dist/middle-schema/parse-typescript.js.map +1 -0
- package/dist/middle-schema/schema-types.d.ts +24 -0
- package/dist/middle-schema/schema-types.d.ts.map +1 -0
- package/dist/middle-schema/schema-types.js +14 -0
- package/dist/middle-schema/schema-types.js.map +1 -0
- package/dist/middle-schema/schema.d.ts +24 -0
- package/dist/middle-schema/schema.d.ts.map +1 -0
- package/dist/middle-schema/schema.js +14 -0
- package/dist/middle-schema/schema.js.map +1 -0
- package/dist/schema/entities/abstract-entity.d.ts +5 -0
- package/dist/schema/entities/abstract-entity.d.ts.map +1 -0
- package/dist/schema/entities/abstract-entity.js +3 -0
- package/dist/schema/entities/abstract-entity.js.map +1 -0
- package/dist/schema/entities/primitive-union.d.ts +12 -0
- package/dist/schema/entities/primitive-union.d.ts.map +1 -0
- package/dist/schema/entities/primitive-union.js +74 -0
- package/dist/schema/entities/primitive-union.js.map +1 -0
- package/dist/schema/entities/primitive.d.ts +15 -0
- package/dist/schema/entities/primitive.d.ts.map +1 -0
- package/dist/schema/entities/primitive.js +54 -0
- package/dist/schema/entities/primitive.js.map +1 -0
- package/dist/schema/parsers/json-schema.d.ts +4 -0
- package/dist/schema/parsers/json-schema.d.ts.map +1 -0
- package/dist/schema/parsers/json-schema.js +12 -0
- package/dist/schema/parsers/json-schema.js.map +1 -0
- package/dist/schema/parsers/typescript.d.ts +3 -0
- package/dist/schema/parsers/typescript.d.ts.map +1 -0
- package/dist/schema/parsers/typescript.js +24 -0
- package/dist/schema/parsers/typescript.js.map +1 -0
- package/dist/schemas/input.d.ts +840 -0
- package/dist/schemas/input.d.ts.map +1 -0
- package/dist/schemas/input.js +349 -0
- package/dist/schemas/input.js.map +1 -0
- package/dist/utils/filesystem.d.ts +8 -0
- package/dist/utils/filesystem.d.ts.map +1 -0
- package/dist/utils/filesystem.js +16 -0
- package/dist/utils/filesystem.js.map +1 -0
- package/dist/utils/json-schemas-interactive-conflict.d.ts +16 -0
- package/dist/utils/json-schemas-interactive-conflict.d.ts.map +1 -0
- package/dist/utils/json-schemas-interactive-conflict.js +165 -0
- package/dist/utils/json-schemas-interactive-conflict.js.map +1 -0
- package/dist/utils/json-schemas.d.ts +42 -0
- package/dist/utils/json-schemas.d.ts.map +1 -0
- package/dist/utils/json-schemas.js +162 -0
- package/dist/utils/json-schemas.js.map +1 -0
- package/dist/zod/schemas/input.d.ts +840 -0
- package/dist/zod/schemas/input.d.ts.map +1 -0
- package/dist/zod/schemas/input.js +393 -0
- package/dist/zod/schemas/input.js.map +1 -0
- package/package.json +12 -12
- package/samples/all-defaults/.actor/input_schema.json +32 -3
- package/samples/all-defaults/src-schemas/input.json +2 -1
- package/samples/deep-merged-schemas/.actor/input_schema.json +36 -3
- package/samples/merged-schemas/.actor/input_schema.json +27 -3
- package/samples/package-json-config/.actor/input_schema.json +32 -3
- package/samples/package-json-config-merged/.actor/input_schema.json +36 -3
- package/src/apify.ts +21 -6
- package/src/cli/check.ts +114 -0
- package/src/cli/init.ts +125 -0
- package/src/cli/sync.ts +164 -0
- package/src/configuration.ts +17 -7
- package/src/main.ts +25 -0
- package/src/middle-schema/compare-schemas.ts +113 -0
- package/src/middle-schema/generate-typescript.ts +88 -0
- package/src/middle-schema/parse-json-schema.ts +104 -0
- package/src/middle-schema/parse-typescript.ts +239 -0
- package/src/middle-schema/schema-types.ts +40 -0
- package/test/apify.test.ts +410 -2
- package/test/cli/check.test.ts +1571 -0
- package/test/cli/init.test.ts +459 -0
- package/test/cli/sync.test.ts +341 -0
- package/test/common.ts +68 -0
- package/test/configuration.test.ts +8 -8
- package/test/middle-schema/compare-schemas.test.ts +585 -0
- package/test/middle-schema/generate-typescript.test.ts +191 -0
- package/test/middle-schema/parse-json-schema.test.ts +178 -0
- package/test/middle-schema/parse-typescript.test.ts +143 -0
- package/test/{json-schema-conflicts.test.ts → utils/json-schemas-interactive-conflict.test.ts} +2 -2
- package/test/{json-schemas.test.ts → utils/json-schemas.test.ts} +3 -3
- package/src/apify-schema-tools.ts +0 -420
- package/src/typescript.ts +0 -563
- package/test/apify-schema-tools.test.ts +0 -2216
- package/test/typescript.test.ts +0 -1079
- /package/src/{filesystem.ts → utils/filesystem.ts} +0 -0
- /package/src/{json-schema-conflicts.ts → utils/json-schemas-interactive-conflict.ts} +0 -0
- /package/src/{json-schemas.ts → utils/json-schemas.ts} +0 -0
|
@@ -9,7 +9,15 @@
|
|
|
9
9
|
"title": "Start URLs",
|
|
10
10
|
"description": "List of URLs to scrape",
|
|
11
11
|
"default": [],
|
|
12
|
-
"editor": "requestListSources"
|
|
12
|
+
"editor": "requestListSources",
|
|
13
|
+
"items": {
|
|
14
|
+
"type": "object",
|
|
15
|
+
"properties": {
|
|
16
|
+
"url": {
|
|
17
|
+
"type": "string"
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
}
|
|
13
21
|
},
|
|
14
22
|
"searchTerm": {
|
|
15
23
|
"type": "string",
|
|
@@ -22,7 +30,21 @@
|
|
|
22
30
|
"type": "array",
|
|
23
31
|
"title": "Categories",
|
|
24
32
|
"description": "List of categories to filter results",
|
|
25
|
-
"default": []
|
|
33
|
+
"default": [],
|
|
34
|
+
"items": {
|
|
35
|
+
"type": "object",
|
|
36
|
+
"properties": {
|
|
37
|
+
"name": {
|
|
38
|
+
"type": "string",
|
|
39
|
+
"title": "Category name"
|
|
40
|
+
},
|
|
41
|
+
"id": {
|
|
42
|
+
"type": "string",
|
|
43
|
+
"title": "Category ID"
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
"additionalProperties": false
|
|
47
|
+
}
|
|
26
48
|
},
|
|
27
49
|
"maxPages": {
|
|
28
50
|
"type": "integer",
|
|
@@ -40,7 +62,14 @@
|
|
|
40
62
|
"description": "Proxy settings",
|
|
41
63
|
"default": {
|
|
42
64
|
"useApifyProxy": true
|
|
43
|
-
}
|
|
65
|
+
},
|
|
66
|
+
"properties": {
|
|
67
|
+
"useApifyProxy": {
|
|
68
|
+
"type": "boolean",
|
|
69
|
+
"default": true
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
"additionalProperties": true
|
|
44
73
|
},
|
|
45
74
|
"debugMode": {
|
|
46
75
|
"type": "boolean",
|
|
@@ -9,7 +9,15 @@
|
|
|
9
9
|
"title": "Start URLs",
|
|
10
10
|
"description": "List of URLs to scrape",
|
|
11
11
|
"default": [],
|
|
12
|
-
"editor": "requestListSources"
|
|
12
|
+
"editor": "requestListSources",
|
|
13
|
+
"items": {
|
|
14
|
+
"type": "object",
|
|
15
|
+
"properties": {
|
|
16
|
+
"url": {
|
|
17
|
+
"type": "string"
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
}
|
|
13
21
|
},
|
|
14
22
|
"searchTerm": {
|
|
15
23
|
"type": "string",
|
|
@@ -22,7 +30,21 @@
|
|
|
22
30
|
"type": "array",
|
|
23
31
|
"title": "Categories",
|
|
24
32
|
"description": "List of categories to filter results",
|
|
25
|
-
"default": []
|
|
33
|
+
"default": [],
|
|
34
|
+
"items": {
|
|
35
|
+
"type": "object",
|
|
36
|
+
"properties": {
|
|
37
|
+
"name": {
|
|
38
|
+
"type": "string",
|
|
39
|
+
"title": "Category name"
|
|
40
|
+
},
|
|
41
|
+
"id": {
|
|
42
|
+
"type": "string",
|
|
43
|
+
"title": "Category ID"
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
"additionalProperties": false
|
|
47
|
+
}
|
|
26
48
|
},
|
|
27
49
|
"maxPages": {
|
|
28
50
|
"type": "integer",
|
|
@@ -40,7 +62,18 @@
|
|
|
40
62
|
"description": "Proxy settings",
|
|
41
63
|
"default": {
|
|
42
64
|
"useApifyProxy": true
|
|
43
|
-
}
|
|
65
|
+
},
|
|
66
|
+
"properties": {
|
|
67
|
+
"useApifyProxy": {
|
|
68
|
+
"type": "boolean",
|
|
69
|
+
"default": true
|
|
70
|
+
},
|
|
71
|
+
"proxyCountry": {
|
|
72
|
+
"type": "string",
|
|
73
|
+
"default": "US"
|
|
74
|
+
}
|
|
75
|
+
},
|
|
76
|
+
"additionalProperties": true
|
|
44
77
|
},
|
|
45
78
|
"debugMode": {
|
|
46
79
|
"type": "boolean",
|
package/src/apify.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { dirname, relative } from "node:path";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { serializeMiddleSchemaToTypeScript } from "./middle-schema/generate-typescript.js";
|
|
3
|
+
import { jsonSchemaToMiddleObject } from "./middle-schema/parse-json-schema.js";
|
|
4
|
+
import type { MiddleObject } from "./middle-schema/schema-types.js";
|
|
5
|
+
import { filterValidSchemaProperties, type ObjectSchema, type OmissibleField } from "./utils/json-schemas.js";
|
|
4
6
|
|
|
5
7
|
export const ACTOR_CONFIG_PATH = ".actor/actor.json";
|
|
6
8
|
|
|
@@ -38,7 +40,18 @@ const VALID_INPUT_PROPERTY_KEYS_BY_TYPE: Record<string, string[]> = {
|
|
|
38
40
|
],
|
|
39
41
|
boolean: ["editor", "groupCaption", "groupDescription", "nullable"],
|
|
40
42
|
integer: ["editor", "maximum", "minimum", "unit", "nullable"],
|
|
41
|
-
object: [
|
|
43
|
+
object: [
|
|
44
|
+
"editor",
|
|
45
|
+
"patternKey",
|
|
46
|
+
"patternValue",
|
|
47
|
+
"maxProperties",
|
|
48
|
+
"minProperties",
|
|
49
|
+
"nullable",
|
|
50
|
+
"isSecret",
|
|
51
|
+
"properties",
|
|
52
|
+
"additionalProperties",
|
|
53
|
+
"required",
|
|
54
|
+
],
|
|
42
55
|
array: [
|
|
43
56
|
"editor",
|
|
44
57
|
"placeholderKey",
|
|
@@ -49,6 +62,8 @@ const VALID_INPUT_PROPERTY_KEYS_BY_TYPE: Record<string, string[]> = {
|
|
|
49
62
|
"minItems",
|
|
50
63
|
"uniqueItems",
|
|
51
64
|
"nullable",
|
|
65
|
+
"items",
|
|
66
|
+
"isSecret",
|
|
52
67
|
"resourceType",
|
|
53
68
|
],
|
|
54
69
|
};
|
|
@@ -76,9 +91,9 @@ export function generateInputDefaultsTsFileContent(inputSchema: ObjectSchema): s
|
|
|
76
91
|
}
|
|
77
92
|
}
|
|
78
93
|
|
|
79
|
-
const tsInterface =
|
|
94
|
+
const tsInterface = jsonSchemaToMiddleObject(inputSchema);
|
|
80
95
|
|
|
81
|
-
const defaultInterface:
|
|
96
|
+
const defaultInterface: MiddleObject = {
|
|
82
97
|
isArray: false,
|
|
83
98
|
isRequired: true,
|
|
84
99
|
properties: {},
|
|
@@ -104,7 +119,7 @@ import type { Input } from "./input.js";
|
|
|
104
119
|
${
|
|
105
120
|
areThereDefaults
|
|
106
121
|
? `\
|
|
107
|
-
interface DefaultInputs ${
|
|
122
|
+
interface DefaultInputs ${serializeMiddleSchemaToTypeScript(defaultInterface)};
|
|
108
123
|
|
|
109
124
|
export type InputWithDefaults = Input & DefaultInputs;
|
|
110
125
|
|
package/src/cli/check.ts
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/** biome-ignore-all lint/style/useNamingConvention: the package `argparse` uses snake_case names */
|
|
2
|
+
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { ArgumentDefaultsHelpFormatter, type SubParser } from "argparse";
|
|
5
|
+
import { DATASET_SCHEMA_FIELD, DESCRIPTION_FIELDS, filterValidInputSchemaProperties } from "../apify.js";
|
|
6
|
+
import { addCommonCliArgs, type CommonCliArgs, type Configuration } from "../configuration.js";
|
|
7
|
+
import { compareMiddleObjects } from "../middle-schema/compare-schemas.js";
|
|
8
|
+
import { jsonSchemaToMiddleObject } from "../middle-schema/parse-json-schema.js";
|
|
9
|
+
import { parseTypeScriptInterface, removeTypeScriptHeader } from "../middle-schema/parse-typescript.js";
|
|
10
|
+
import { readFile } from "../utils/filesystem.js";
|
|
11
|
+
import { compareSchemas, mergeObjectSchemas, readJsonSchema, readJsonSchemaField } from "../utils/json-schemas.js";
|
|
12
|
+
|
|
13
|
+
interface CheckArgs extends CommonCliArgs {
|
|
14
|
+
ignore_descriptions: boolean;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function setupCheckParser(subparsers: SubParser, configuration: Configuration): void {
|
|
18
|
+
const checkParser = subparsers.add_parser("check", {
|
|
19
|
+
help: "Check the schemas for consistency and correctness. (WARNING: input utils will not be checked!)",
|
|
20
|
+
formatter_class: ArgumentDefaultsHelpFormatter,
|
|
21
|
+
});
|
|
22
|
+
addCommonCliArgs(checkParser, configuration);
|
|
23
|
+
checkParser.add_argument("--ignore-descriptions", {
|
|
24
|
+
help: "ignore the 'title' and 'description' fields during the comparison",
|
|
25
|
+
action: "store_true",
|
|
26
|
+
default: false,
|
|
27
|
+
});
|
|
28
|
+
checkParser.set_defaults({ func: check });
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function check(args: CheckArgs): void {
|
|
32
|
+
if (args.input.includes("input")) {
|
|
33
|
+
checkInputSchemas(args);
|
|
34
|
+
}
|
|
35
|
+
if (args.input.includes("dataset")) {
|
|
36
|
+
checkDatasetSchemas(args);
|
|
37
|
+
}
|
|
38
|
+
console.log("Check passed.");
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function checkInputSchemas(args: CheckArgs): void {
|
|
42
|
+
let sourceInputSchema = readJsonSchema(args.src_input);
|
|
43
|
+
if (args.add_input) {
|
|
44
|
+
sourceInputSchema = mergeObjectSchemas(sourceInputSchema, readJsonSchema(args.add_input), args.deep_merge);
|
|
45
|
+
}
|
|
46
|
+
const generatedInputSchema = filterValidInputSchemaProperties(sourceInputSchema);
|
|
47
|
+
if (args.output.includes("json-schemas")) {
|
|
48
|
+
const outputInputSchema = readJsonSchema(args.input_schema);
|
|
49
|
+
const isInputSchemaCoherent = compareSchemas(
|
|
50
|
+
generatedInputSchema,
|
|
51
|
+
outputInputSchema,
|
|
52
|
+
args.ignore_descriptions ? DESCRIPTION_FIELDS : undefined,
|
|
53
|
+
);
|
|
54
|
+
if (!isInputSchemaCoherent) {
|
|
55
|
+
console.error("Input schema validation failed: the input schema does not match the source schema.");
|
|
56
|
+
process.exit(1);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
if (args.output.includes("ts-types")) {
|
|
60
|
+
const inputTsContent = readFile(join(args.output_ts_dir, "input.ts"));
|
|
61
|
+
const inputMiddleObject = parseTypeScriptInterface(removeTypeScriptHeader(inputTsContent));
|
|
62
|
+
const sourceInputMiddleObject = jsonSchemaToMiddleObject(sourceInputSchema);
|
|
63
|
+
const isTsInputCoherent = compareMiddleObjects(
|
|
64
|
+
sourceInputMiddleObject,
|
|
65
|
+
inputMiddleObject,
|
|
66
|
+
args.ignore_descriptions,
|
|
67
|
+
);
|
|
68
|
+
if (!isTsInputCoherent) {
|
|
69
|
+
console.error(
|
|
70
|
+
"Input TypeScript interface validation failed: the TypeScript interface does not match the source schema.",
|
|
71
|
+
);
|
|
72
|
+
process.exit(1);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function checkDatasetSchemas(args: CheckArgs): void {
|
|
78
|
+
let sourceDatasetSchema = readJsonSchema(args.src_dataset);
|
|
79
|
+
if (args.add_dataset) {
|
|
80
|
+
sourceDatasetSchema = mergeObjectSchemas(sourceDatasetSchema, readJsonSchema(args.add_dataset), args.deep_merge);
|
|
81
|
+
}
|
|
82
|
+
if (args.output.includes("json-schemas")) {
|
|
83
|
+
const outputDatasetSchema = readJsonSchemaField(args.dataset_schema, DATASET_SCHEMA_FIELD);
|
|
84
|
+
if (!outputDatasetSchema) {
|
|
85
|
+
console.error(`Dataset schema field "${DATASET_SCHEMA_FIELD}" not found in ${args.dataset_schema}.`);
|
|
86
|
+
process.exit(1);
|
|
87
|
+
}
|
|
88
|
+
const isDatasetSchemaCoherent = compareSchemas(
|
|
89
|
+
sourceDatasetSchema,
|
|
90
|
+
outputDatasetSchema,
|
|
91
|
+
args.ignore_descriptions ? DESCRIPTION_FIELDS : undefined,
|
|
92
|
+
);
|
|
93
|
+
if (!isDatasetSchemaCoherent) {
|
|
94
|
+
console.error("Dataset schema validation failed: the dataset schema does not match the source schema.");
|
|
95
|
+
process.exit(1);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
if (args.output.includes("ts-types")) {
|
|
99
|
+
const datasetTsContent = readFile(join(args.output_ts_dir, "dataset.ts"));
|
|
100
|
+
const datasetMiddleObject = parseTypeScriptInterface(removeTypeScriptHeader(datasetTsContent));
|
|
101
|
+
const sourceDatasetMiddleObject = jsonSchemaToMiddleObject(sourceDatasetSchema);
|
|
102
|
+
const isTsDatasetCoherent = compareMiddleObjects(
|
|
103
|
+
sourceDatasetMiddleObject,
|
|
104
|
+
datasetMiddleObject,
|
|
105
|
+
args.ignore_descriptions,
|
|
106
|
+
);
|
|
107
|
+
if (!isTsDatasetCoherent) {
|
|
108
|
+
console.error(
|
|
109
|
+
"Dataset TypeScript interface validation failed: the TypeScript interface does not match the source schema.",
|
|
110
|
+
);
|
|
111
|
+
process.exit(1);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
package/src/cli/init.ts
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/** biome-ignore-all lint/style/useNamingConvention: the package `argparse` uses snake_case names */
|
|
2
|
+
|
|
3
|
+
import { existsSync } from "node:fs";
|
|
4
|
+
import { ArgumentDefaultsHelpFormatter, type SubParser } from "argparse";
|
|
5
|
+
import { ACTOR_CONFIG_PATH, DATASET_SCHEMA_FIELD, getPathRelativeToActorConfig } from "../apify.js";
|
|
6
|
+
import {
|
|
7
|
+
addCommonCliArgs,
|
|
8
|
+
type CommonCliArgs,
|
|
9
|
+
type Configuration,
|
|
10
|
+
writeConfigurationToPackageJson,
|
|
11
|
+
} from "../configuration.js";
|
|
12
|
+
import { readFile, writeFile } from "../utils/filesystem.js";
|
|
13
|
+
import { readJsonSchema, readJsonSchemaField, writeJsonSchema } from "../utils/json-schemas.js";
|
|
14
|
+
|
|
15
|
+
interface InitArgs extends CommonCliArgs {
|
|
16
|
+
no_config_file: boolean;
|
|
17
|
+
only_config_file: boolean;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function setupInitParser(subparsers: SubParser, configuration: Configuration): void {
|
|
21
|
+
const initParser = subparsers.add_parser("init", {
|
|
22
|
+
help: "Initialize the Apify Schema Tools project with default settings.",
|
|
23
|
+
formatter_class: ArgumentDefaultsHelpFormatter,
|
|
24
|
+
});
|
|
25
|
+
addCommonCliArgs(initParser, configuration);
|
|
26
|
+
initParser.add_argument("--no-config-file", {
|
|
27
|
+
help: "do not create a configuration file in package.json",
|
|
28
|
+
action: "store_true",
|
|
29
|
+
default: false,
|
|
30
|
+
});
|
|
31
|
+
initParser.add_argument("--only-config-file", {
|
|
32
|
+
help: "create only the configuration file in package.json, without initializing schemas",
|
|
33
|
+
action: "store_true",
|
|
34
|
+
default: false,
|
|
35
|
+
});
|
|
36
|
+
initParser.set_defaults({ func: init });
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function init(args: InitArgs): void {
|
|
40
|
+
if (args.only_config_file && args.no_config_file) {
|
|
41
|
+
throw new Error("The options --only-config-file and --no-config-file were defined together: doing nothing.");
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
console.log("Initializing Apify Schema Tools in the current project...");
|
|
45
|
+
|
|
46
|
+
if (!args.no_config_file) {
|
|
47
|
+
writeConfigurationToPackageJson(args);
|
|
48
|
+
console.log("Configuration written to package.json");
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
if (args.only_config_file) {
|
|
52
|
+
console.log("Only configuration file created, skipping schema initialization.");
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
if (args.input.includes("input")) {
|
|
57
|
+
initializeInputSchema(args);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
if (args.input.includes("dataset")) {
|
|
61
|
+
initializeDatasetSchema(args);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function initializeInputSchema(args: InitArgs): void {
|
|
66
|
+
if (existsSync(args.src_input)) {
|
|
67
|
+
console.log(`Input schema already exists at ${args.src_input}, skipping initialization.`);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
if (!existsSync(args.input_schema)) {
|
|
71
|
+
throw new Error("The current Actor does not have an input schema.");
|
|
72
|
+
}
|
|
73
|
+
const inputSchema = readJsonSchema(args.input_schema);
|
|
74
|
+
writeJsonSchema(args.src_input, inputSchema);
|
|
75
|
+
console.log(`Input schema initialized at ${args.src_input}`);
|
|
76
|
+
if (args.add_input) {
|
|
77
|
+
writeJsonSchema(args.add_input, { type: "object", properties: {} });
|
|
78
|
+
console.log(`Additional input schema initialized at ${args.add_input}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function initializeDatasetSchema(args: InitArgs): void {
|
|
83
|
+
if (!existsSync(ACTOR_CONFIG_PATH)) {
|
|
84
|
+
throw new Error(`The current Actor does not have an ${ACTOR_CONFIG_PATH} configuration file.`);
|
|
85
|
+
}
|
|
86
|
+
const actorConfig = JSON.parse(readFile(ACTOR_CONFIG_PATH));
|
|
87
|
+
const relativeDatasetPath = getPathRelativeToActorConfig(args.dataset_schema);
|
|
88
|
+
if (actorConfig.storages?.dataset !== relativeDatasetPath) {
|
|
89
|
+
writeFile(
|
|
90
|
+
ACTOR_CONFIG_PATH,
|
|
91
|
+
JSON.stringify(
|
|
92
|
+
{
|
|
93
|
+
...actorConfig,
|
|
94
|
+
storages: {
|
|
95
|
+
...actorConfig.storages,
|
|
96
|
+
dataset: relativeDatasetPath,
|
|
97
|
+
},
|
|
98
|
+
},
|
|
99
|
+
null,
|
|
100
|
+
4,
|
|
101
|
+
),
|
|
102
|
+
);
|
|
103
|
+
console.log(`Updated ${ACTOR_CONFIG_PATH} to use the dataset schema at ${args.dataset_schema}`);
|
|
104
|
+
}
|
|
105
|
+
if (!existsSync(args.dataset_schema)) {
|
|
106
|
+
writeFile(
|
|
107
|
+
args.dataset_schema,
|
|
108
|
+
JSON.stringify(
|
|
109
|
+
{
|
|
110
|
+
actorSpecification: 1,
|
|
111
|
+
[DATASET_SCHEMA_FIELD]: { type: "object", properties: {} },
|
|
112
|
+
},
|
|
113
|
+
null,
|
|
114
|
+
4,
|
|
115
|
+
),
|
|
116
|
+
);
|
|
117
|
+
console.log(`Dataset schema initialized at ${args.dataset_schema}`);
|
|
118
|
+
}
|
|
119
|
+
if (existsSync(args.src_dataset)) {
|
|
120
|
+
console.log(`Dataset schema already exists at ${args.src_dataset}, skipping initialization.`);
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
const datasetItemSchema = readJsonSchemaField(args.dataset_schema, DATASET_SCHEMA_FIELD);
|
|
124
|
+
writeJsonSchema(args.src_dataset, datasetItemSchema);
|
|
125
|
+
}
|
package/src/cli/sync.ts
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/** biome-ignore-all lint/style/useNamingConvention: the package `argparse` uses snake_case names */
|
|
2
|
+
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { ArgumentDefaultsHelpFormatter, type SubParser } from "argparse";
|
|
5
|
+
import {
|
|
6
|
+
DATASET_SCHEMA_FIELD,
|
|
7
|
+
DESCRIPTION_FIELDS,
|
|
8
|
+
filterValidInputSchemaProperties,
|
|
9
|
+
generateInputDefaultsTsFileContent,
|
|
10
|
+
} from "../apify.js";
|
|
11
|
+
import { addCommonCliArgs, type CommonCliArgs, type Configuration } from "../configuration.js";
|
|
12
|
+
import { serializeMiddleObjectToTypeScript, writeTypeScriptFile } from "../middle-schema/generate-typescript.js";
|
|
13
|
+
import { jsonSchemaToMiddleObject } from "../middle-schema/parse-json-schema.js";
|
|
14
|
+
import {
|
|
15
|
+
mergeObjectSchemas,
|
|
16
|
+
type ObjectSchema,
|
|
17
|
+
readJsonSchema,
|
|
18
|
+
readJsonSchemaField,
|
|
19
|
+
writeJsonSchema,
|
|
20
|
+
writeSchemaToField,
|
|
21
|
+
} from "../utils/json-schemas.js";
|
|
22
|
+
import { type ConflictResolutionStrategy, checkConflicts } from "../utils/json-schemas-interactive-conflict.js";
|
|
23
|
+
|
|
24
|
+
interface SyncArgs extends CommonCliArgs {
|
|
25
|
+
include_input_utils: string;
|
|
26
|
+
force: boolean;
|
|
27
|
+
fail_on_conflict: boolean;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function setupSyncParser(subparsers: SubParser, configuration: Configuration): void {
|
|
31
|
+
const syncParser = subparsers.add_parser("sync", {
|
|
32
|
+
help: `Generate JSON schemas and TypeScript files from the source schemas. \
|
|
33
|
+
By default, if conflicts are detected, the user will be prompted to resolve them.`,
|
|
34
|
+
formatter_class: ArgumentDefaultsHelpFormatter,
|
|
35
|
+
});
|
|
36
|
+
addCommonCliArgs(syncParser, configuration);
|
|
37
|
+
syncParser.add_argument("--include-input-utils", {
|
|
38
|
+
help: "include input utilities in the generated TypeScript files: 'input' input and 'ts-types' output are required",
|
|
39
|
+
choices: ["true", "false"],
|
|
40
|
+
default: "true",
|
|
41
|
+
});
|
|
42
|
+
syncParser.add_argument("--force", {
|
|
43
|
+
help: "force the sync operation, even if conflicts are detected",
|
|
44
|
+
action: "store_true",
|
|
45
|
+
default: false,
|
|
46
|
+
});
|
|
47
|
+
syncParser.add_argument("--fail-on-conflict", {
|
|
48
|
+
help: "fail the sync operation if conflicts are detected",
|
|
49
|
+
action: "store_true",
|
|
50
|
+
default: false,
|
|
51
|
+
});
|
|
52
|
+
syncParser.set_defaults({ func: sync });
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function sync(args: SyncArgs): void {
|
|
56
|
+
console.log("Syncing schemas...");
|
|
57
|
+
|
|
58
|
+
if (args.input.includes("input")) {
|
|
59
|
+
handleInputSync(args);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
if (args.input.includes("dataset")) {
|
|
63
|
+
handleDatasetSync(args);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async function handleInputSync(args: SyncArgs): Promise<void> {
|
|
68
|
+
let inputSchemas = {
|
|
69
|
+
sourceSchema: readJsonSchema(args.src_input),
|
|
70
|
+
additionalSchema: args.add_input ? readJsonSchema(args.add_input) : undefined,
|
|
71
|
+
};
|
|
72
|
+
let resultingInputSchema: ObjectSchema | undefined;
|
|
73
|
+
if (args.output.includes("json-schemas")) {
|
|
74
|
+
const conflictResolutionStrategy = getConflictResolutionStrategy(args);
|
|
75
|
+
const existingInputSchema = readJsonSchema(args.input_schema);
|
|
76
|
+
inputSchemas = await checkConflicts(
|
|
77
|
+
inputSchemas.sourceSchema,
|
|
78
|
+
args.src_input,
|
|
79
|
+
inputSchemas.additionalSchema,
|
|
80
|
+
args.add_input,
|
|
81
|
+
existingInputSchema,
|
|
82
|
+
args.deep_merge,
|
|
83
|
+
DESCRIPTION_FIELDS,
|
|
84
|
+
conflictResolutionStrategy,
|
|
85
|
+
);
|
|
86
|
+
resultingInputSchema = inputSchemas.additionalSchema
|
|
87
|
+
? mergeObjectSchemas(inputSchemas.sourceSchema, inputSchemas.additionalSchema, args.deep_merge)
|
|
88
|
+
: inputSchemas.sourceSchema;
|
|
89
|
+
writeJsonSchema(args.input_schema, filterValidInputSchemaProperties(resultingInputSchema));
|
|
90
|
+
}
|
|
91
|
+
if (args.output.includes("ts-types")) {
|
|
92
|
+
resultingInputSchema ??= inputSchemas.additionalSchema
|
|
93
|
+
? mergeObjectSchemas(inputSchemas.sourceSchema, inputSchemas.additionalSchema, args.deep_merge)
|
|
94
|
+
: inputSchemas.sourceSchema;
|
|
95
|
+
writeTypeScriptFile(
|
|
96
|
+
join(args.output_ts_dir, "input.ts"),
|
|
97
|
+
serializeMiddleObjectToTypeScript("Input", jsonSchemaToMiddleObject(resultingInputSchema)),
|
|
98
|
+
);
|
|
99
|
+
if (args.include_input_utils === "true") {
|
|
100
|
+
writeTypeScriptFile(
|
|
101
|
+
join(args.output_ts_dir, "input-utils.ts"),
|
|
102
|
+
generateInputDefaultsTsFileContent(resultingInputSchema),
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
// Update the source schemas, which may have been modified by the conflict resolution
|
|
107
|
+
writeJsonSchema(args.src_input, inputSchemas.sourceSchema);
|
|
108
|
+
if (args.add_input && inputSchemas.additionalSchema) {
|
|
109
|
+
writeJsonSchema(args.add_input, inputSchemas.additionalSchema);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async function handleDatasetSync(args: SyncArgs): Promise<void> {
|
|
114
|
+
let datasetSchemas = {
|
|
115
|
+
sourceSchema: readJsonSchema(args.src_dataset),
|
|
116
|
+
additionalSchema: args.add_dataset ? readJsonSchema(args.add_dataset) : undefined,
|
|
117
|
+
};
|
|
118
|
+
let resultingDatasetSchema: ObjectSchema | undefined;
|
|
119
|
+
if (args.output.includes("json-schemas")) {
|
|
120
|
+
const conflictResolutionStrategy = getConflictResolutionStrategy(args);
|
|
121
|
+
const existingDatasetSchema = readJsonSchemaField(args.dataset_schema, DATASET_SCHEMA_FIELD);
|
|
122
|
+
datasetSchemas = await checkConflicts(
|
|
123
|
+
datasetSchemas.sourceSchema,
|
|
124
|
+
args.src_dataset,
|
|
125
|
+
datasetSchemas.additionalSchema,
|
|
126
|
+
args.add_dataset,
|
|
127
|
+
existingDatasetSchema,
|
|
128
|
+
args.deep_merge,
|
|
129
|
+
DESCRIPTION_FIELDS,
|
|
130
|
+
conflictResolutionStrategy,
|
|
131
|
+
);
|
|
132
|
+
resultingDatasetSchema = datasetSchemas.additionalSchema
|
|
133
|
+
? mergeObjectSchemas(datasetSchemas.sourceSchema, datasetSchemas.additionalSchema, args.deep_merge)
|
|
134
|
+
: datasetSchemas.sourceSchema;
|
|
135
|
+
writeSchemaToField(args.dataset_schema, resultingDatasetSchema, DATASET_SCHEMA_FIELD);
|
|
136
|
+
}
|
|
137
|
+
if (args.output.includes("ts-types")) {
|
|
138
|
+
resultingDatasetSchema ??= datasetSchemas.additionalSchema
|
|
139
|
+
? mergeObjectSchemas(datasetSchemas.sourceSchema, datasetSchemas.additionalSchema, args.deep_merge)
|
|
140
|
+
: datasetSchemas.sourceSchema;
|
|
141
|
+
writeTypeScriptFile(
|
|
142
|
+
join(args.output_ts_dir, "dataset.ts"),
|
|
143
|
+
serializeMiddleObjectToTypeScript("DatasetItem", jsonSchemaToMiddleObject(resultingDatasetSchema)),
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
// Update the source schemas, which may have been modified by the conflict resolution
|
|
147
|
+
writeJsonSchema(args.src_dataset, datasetSchemas.sourceSchema);
|
|
148
|
+
if (args.add_dataset && datasetSchemas.additionalSchema) {
|
|
149
|
+
writeJsonSchema(args.add_dataset, datasetSchemas.additionalSchema);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function getConflictResolutionStrategy(args: SyncArgs): ConflictResolutionStrategy {
|
|
154
|
+
if (args.fail_on_conflict && args.force) {
|
|
155
|
+
throw new Error("The options --force and --fail-on-conflict cannot be defined together.");
|
|
156
|
+
}
|
|
157
|
+
if (args.force) {
|
|
158
|
+
return "log";
|
|
159
|
+
}
|
|
160
|
+
if (args.fail_on_conflict) {
|
|
161
|
+
return "error";
|
|
162
|
+
}
|
|
163
|
+
return "interactive";
|
|
164
|
+
}
|
package/src/configuration.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
import { existsSync } from "node:fs";
|
|
4
4
|
import type { ArgumentParser } from "argparse";
|
|
5
5
|
import { array, boolean, enum as enum_, object, string, type infer as zodInfer } from "zod/v4";
|
|
6
|
-
import { readFile, writeFile } from "./filesystem.js";
|
|
6
|
+
import { readFile, writeFile } from "./utils/filesystem.js";
|
|
7
7
|
|
|
8
8
|
const zod = {
|
|
9
9
|
object,
|
|
@@ -19,6 +19,10 @@ const OUTPUTS = ["json-schemas", "ts-types"] as const;
|
|
|
19
19
|
type Input = (typeof INPUTS)[number];
|
|
20
20
|
type Output = (typeof OUTPUTS)[number];
|
|
21
21
|
|
|
22
|
+
/**
|
|
23
|
+
* These CLI arguments are also available in the configuration file.
|
|
24
|
+
* They are in snake_case because `argparse` uses it for argument names.
|
|
25
|
+
*/
|
|
22
26
|
export interface CommonCliArgs {
|
|
23
27
|
input: Input[];
|
|
24
28
|
output: Output[];
|
|
@@ -32,7 +36,11 @@ export interface CommonCliArgs {
|
|
|
32
36
|
output_ts_dir: string;
|
|
33
37
|
}
|
|
34
38
|
|
|
35
|
-
|
|
39
|
+
/**
|
|
40
|
+
* Zod schema for the configuration file.
|
|
41
|
+
* Since the configuration is in JSON format, we use camelCase.
|
|
42
|
+
*/
|
|
43
|
+
export const CONFIGURATION_SCHEMA = zod.object({
|
|
36
44
|
input: zod.array(zod.enum(INPUTS)).default([...INPUTS]),
|
|
37
45
|
output: zod.array(zod.enum(OUTPUTS)).default([...OUTPUTS]),
|
|
38
46
|
srcInput: zod.string().default("src-schemas/input.json"),
|
|
@@ -45,10 +53,12 @@ export const Configuration = zod.object({
|
|
|
45
53
|
outputTSDir: zod.string().default("src/generated"),
|
|
46
54
|
});
|
|
47
55
|
|
|
48
|
-
export
|
|
56
|
+
export type Configuration = zodInfer<typeof CONFIGURATION_SCHEMA>;
|
|
57
|
+
|
|
58
|
+
export function parseConfigurationFromFileOrDefault(): Configuration {
|
|
49
59
|
const packageJsonContent = existsSync("package.json") ? readFile("package.json") : undefined;
|
|
50
60
|
const rawConfig = packageJsonContent ? (JSON.parse(packageJsonContent)["apify-schema-tools"] ?? {}) : {};
|
|
51
|
-
return
|
|
61
|
+
return CONFIGURATION_SCHEMA.parse(rawConfig);
|
|
52
62
|
}
|
|
53
63
|
|
|
54
64
|
export function writeConfigurationToPackageJson(args: CommonCliArgs): void {
|
|
@@ -73,10 +83,10 @@ export function writeConfigurationToPackageJson(args: CommonCliArgs): void {
|
|
|
73
83
|
}
|
|
74
84
|
|
|
75
85
|
/**
|
|
76
|
-
* This function will set as default values the configuration
|
|
77
|
-
* In this way, the CLI arguments can override the configuration.
|
|
86
|
+
* This function will set as default values the configuration file, or the default one.
|
|
87
|
+
* In this way, the CLI arguments can override the configuration file.
|
|
78
88
|
*/
|
|
79
|
-
export function addCommonCliArgs(parser: ArgumentParser, configuration: zodInfer<typeof
|
|
89
|
+
export function addCommonCliArgs(parser: ArgumentParser, configuration: zodInfer<typeof CONFIGURATION_SCHEMA>): void {
|
|
80
90
|
parser.add_argument("-i", "--input", {
|
|
81
91
|
help: "specify which sources to use for generation",
|
|
82
92
|
choices: [...INPUTS],
|
package/src/main.ts
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/** biome-ignore-all lint/style/useNamingConvention: the package `argparse` uses snake_case names */
|
|
4
|
+
|
|
5
|
+
import { ArgumentDefaultsHelpFormatter, ArgumentParser } from "argparse";
|
|
6
|
+
import { setupCheckParser } from "./cli/check.js";
|
|
7
|
+
import { setupInitParser } from "./cli/init.js";
|
|
8
|
+
import { setupSyncParser } from "./cli/sync.js";
|
|
9
|
+
import { parseConfigurationFromFileOrDefault } from "./configuration.js";
|
|
10
|
+
|
|
11
|
+
const configuration = parseConfigurationFromFileOrDefault();
|
|
12
|
+
|
|
13
|
+
const rootParser = new ArgumentParser({
|
|
14
|
+
description: "Apify Schema Tools - Generate JSON schemas and TypeScript files for Actor input and output dataset.",
|
|
15
|
+
formatter_class: ArgumentDefaultsHelpFormatter,
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
const subparsers = rootParser.add_subparsers();
|
|
19
|
+
|
|
20
|
+
setupInitParser(subparsers, configuration);
|
|
21
|
+
setupSyncParser(subparsers, configuration);
|
|
22
|
+
setupCheckParser(subparsers, configuration);
|
|
23
|
+
|
|
24
|
+
const parsedArgs = rootParser.parse_args();
|
|
25
|
+
parsedArgs.func(parsedArgs);
|