apify-schema-tools 3.0.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cspell/custom-dictionary.txt +1 -0
- package/.node-version +1 -1
- package/CHANGELOG.md +13 -1
- package/biome.json +24 -18
- package/check-samples.sh +4 -0
- package/dist/apify-schema-tools.js +16 -21
- package/dist/apify-schema-tools.js.map +1 -1
- package/dist/apify.d.ts +1 -1
- package/dist/apify.d.ts.map +1 -1
- package/dist/apify.js +22 -8
- package/dist/apify.js.map +1 -1
- package/dist/cli/check.d.ts +5 -0
- package/dist/cli/check.d.ts.map +1 -0
- package/dist/cli/check.js +86 -0
- package/dist/cli/check.js.map +1 -0
- package/dist/cli/init.d.ts +5 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +92 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/cli/sync.d.ts +5 -0
- package/dist/cli/sync.d.ts.map +1 -0
- package/dist/cli/sync.js +112 -0
- package/dist/cli/sync.js.map +1 -0
- package/dist/configuration.d.ts +16 -5
- package/dist/configuration.d.ts.map +1 -1
- package/dist/configuration.js +19 -7
- package/dist/configuration.js.map +1 -1
- package/dist/json-schema-conflicts.js +3 -3
- package/dist/json-schema-conflicts.js.map +1 -1
- package/dist/json-schemas.d.ts +1 -1
- package/dist/json-schemas.d.ts.map +1 -1
- package/dist/json-schemas.js +1 -1
- package/dist/json-schemas.js.map +1 -1
- package/dist/main.d.ts +4 -0
- package/dist/main.d.ts.map +1 -0
- package/dist/main.js +19 -0
- package/dist/main.js.map +1 -0
- package/dist/middle-schema/compare-schemas.d.ts +3 -0
- package/dist/middle-schema/compare-schemas.d.ts.map +1 -0
- package/dist/middle-schema/compare-schemas.js +90 -0
- package/dist/middle-schema/compare-schemas.js.map +1 -0
- package/dist/middle-schema/generate-typescript.d.ts +7 -0
- package/dist/middle-schema/generate-typescript.d.ts.map +1 -0
- package/dist/middle-schema/generate-typescript.js +70 -0
- package/dist/middle-schema/generate-typescript.js.map +1 -0
- package/dist/middle-schema/parse-json-schema.d.ts +4 -0
- package/dist/middle-schema/parse-json-schema.d.ts.map +1 -0
- package/dist/middle-schema/parse-json-schema.js +65 -0
- package/dist/middle-schema/parse-json-schema.js.map +1 -0
- package/dist/middle-schema/parse-typescript.d.ts +4 -0
- package/dist/middle-schema/parse-typescript.d.ts.map +1 -0
- package/dist/middle-schema/parse-typescript.js +199 -0
- package/dist/middle-schema/parse-typescript.js.map +1 -0
- package/dist/middle-schema/schema-types.d.ts +24 -0
- package/dist/middle-schema/schema-types.d.ts.map +1 -0
- package/dist/middle-schema/schema-types.js +14 -0
- package/dist/middle-schema/schema-types.js.map +1 -0
- package/dist/middle-schema/schema.d.ts +24 -0
- package/dist/middle-schema/schema.d.ts.map +1 -0
- package/dist/middle-schema/schema.js +14 -0
- package/dist/middle-schema/schema.js.map +1 -0
- package/dist/schema/entities/abstract-entity.d.ts +5 -0
- package/dist/schema/entities/abstract-entity.d.ts.map +1 -0
- package/dist/schema/entities/abstract-entity.js +3 -0
- package/dist/schema/entities/abstract-entity.js.map +1 -0
- package/dist/schema/entities/primitive-union.d.ts +12 -0
- package/dist/schema/entities/primitive-union.d.ts.map +1 -0
- package/dist/schema/entities/primitive-union.js +74 -0
- package/dist/schema/entities/primitive-union.js.map +1 -0
- package/dist/schema/entities/primitive.d.ts +15 -0
- package/dist/schema/entities/primitive.d.ts.map +1 -0
- package/dist/schema/entities/primitive.js +54 -0
- package/dist/schema/entities/primitive.js.map +1 -0
- package/dist/schema/parsers/json-schema.d.ts +4 -0
- package/dist/schema/parsers/json-schema.d.ts.map +1 -0
- package/dist/schema/parsers/json-schema.js +12 -0
- package/dist/schema/parsers/json-schema.js.map +1 -0
- package/dist/schema/parsers/typescript.d.ts +3 -0
- package/dist/schema/parsers/typescript.d.ts.map +1 -0
- package/dist/schema/parsers/typescript.js +24 -0
- package/dist/schema/parsers/typescript.js.map +1 -0
- package/dist/schemas/input.d.ts +840 -0
- package/dist/schemas/input.d.ts.map +1 -0
- package/dist/schemas/input.js +349 -0
- package/dist/schemas/input.js.map +1 -0
- package/dist/utils/filesystem.d.ts +8 -0
- package/dist/utils/filesystem.d.ts.map +1 -0
- package/dist/utils/filesystem.js +16 -0
- package/dist/utils/filesystem.js.map +1 -0
- package/dist/utils/json-schemas-interactive-conflict.d.ts +16 -0
- package/dist/utils/json-schemas-interactive-conflict.d.ts.map +1 -0
- package/dist/utils/json-schemas-interactive-conflict.js +165 -0
- package/dist/utils/json-schemas-interactive-conflict.js.map +1 -0
- package/dist/utils/json-schemas.d.ts +42 -0
- package/dist/utils/json-schemas.d.ts.map +1 -0
- package/dist/utils/json-schemas.js +162 -0
- package/dist/utils/json-schemas.js.map +1 -0
- package/dist/zod/schemas/input.d.ts +840 -0
- package/dist/zod/schemas/input.d.ts.map +1 -0
- package/dist/zod/schemas/input.js +393 -0
- package/dist/zod/schemas/input.js.map +1 -0
- package/package.json +12 -12
- package/samples/all-defaults/.actor/input_schema.json +32 -3
- package/samples/all-defaults/src-schemas/input.json +2 -1
- package/samples/deep-merged-schemas/.actor/input_schema.json +36 -3
- package/samples/merged-schemas/.actor/input_schema.json +27 -3
- package/samples/package-json-config/.actor/input_schema.json +32 -3
- package/samples/package-json-config-merged/.actor/actor.json +15 -0
- package/samples/package-json-config-merged/.actor/dataset_schema.json +32 -0
- package/samples/package-json-config-merged/.actor/input_schema.json +91 -0
- package/samples/package-json-config-merged/custom-add-schemas/add-input.json +21 -0
- package/samples/package-json-config-merged/custom-src-schemas/dataset-item.json +28 -0
- package/samples/package-json-config-merged/custom-src-schemas/input.json +89 -0
- package/samples/package-json-config-merged/package.json +19 -0
- package/samples/package-json-config-merged/src/custom-generated/dataset.ts +25 -0
- package/samples/package-json-config-merged/src/custom-generated/input-utils.ts +73 -0
- package/samples/package-json-config-merged/src/custom-generated/input.ts +49 -0
- package/src/apify.ts +24 -9
- package/src/cli/check.ts +114 -0
- package/src/cli/init.ts +125 -0
- package/src/cli/sync.ts +164 -0
- package/src/configuration.ts +27 -8
- package/src/main.ts +25 -0
- package/src/middle-schema/compare-schemas.ts +113 -0
- package/src/middle-schema/generate-typescript.ts +88 -0
- package/src/middle-schema/parse-json-schema.ts +104 -0
- package/src/middle-schema/parse-typescript.ts +239 -0
- package/src/middle-schema/schema-types.ts +40 -0
- package/src/{json-schema-conflicts.ts → utils/json-schemas-interactive-conflict.ts} +3 -3
- package/src/{json-schemas.ts → utils/json-schemas.ts} +1 -1
- package/test/apify.test.ts +413 -5
- package/test/cli/check.test.ts +1571 -0
- package/test/cli/init.test.ts +459 -0
- package/test/cli/sync.test.ts +341 -0
- package/test/common.ts +68 -0
- package/test/configuration.test.ts +59 -26
- package/test/middle-schema/compare-schemas.test.ts +585 -0
- package/test/middle-schema/generate-typescript.test.ts +191 -0
- package/test/middle-schema/parse-json-schema.test.ts +178 -0
- package/test/middle-schema/parse-typescript.test.ts +143 -0
- package/test/{json-schema-conflicts.test.ts → utils/json-schemas-interactive-conflict.test.ts} +3 -3
- package/test/{json-schemas.test.ts → utils/json-schemas.test.ts} +4 -4
- package/update-samples.sh +4 -0
- package/src/apify-schema-tools.ts +0 -431
- package/src/typescript.ts +0 -563
- package/test/apify-schema-tools.test.ts +0 -2216
- package/test/typescript.test.ts +0 -1079
- /package/src/{filesystem.ts → utils/filesystem.ts} +0 -0
package/src/cli/init.ts
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/** biome-ignore-all lint/style/useNamingConvention: the package `argparse` uses snake_case names */
|
|
2
|
+
|
|
3
|
+
import { existsSync } from "node:fs";
|
|
4
|
+
import { ArgumentDefaultsHelpFormatter, type SubParser } from "argparse";
|
|
5
|
+
import { ACTOR_CONFIG_PATH, DATASET_SCHEMA_FIELD, getPathRelativeToActorConfig } from "../apify.js";
|
|
6
|
+
import {
|
|
7
|
+
addCommonCliArgs,
|
|
8
|
+
type CommonCliArgs,
|
|
9
|
+
type Configuration,
|
|
10
|
+
writeConfigurationToPackageJson,
|
|
11
|
+
} from "../configuration.js";
|
|
12
|
+
import { readFile, writeFile } from "../utils/filesystem.js";
|
|
13
|
+
import { readJsonSchema, readJsonSchemaField, writeJsonSchema } from "../utils/json-schemas.js";
|
|
14
|
+
|
|
15
|
+
interface InitArgs extends CommonCliArgs {
|
|
16
|
+
no_config_file: boolean;
|
|
17
|
+
only_config_file: boolean;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function setupInitParser(subparsers: SubParser, configuration: Configuration): void {
|
|
21
|
+
const initParser = subparsers.add_parser("init", {
|
|
22
|
+
help: "Initialize the Apify Schema Tools project with default settings.",
|
|
23
|
+
formatter_class: ArgumentDefaultsHelpFormatter,
|
|
24
|
+
});
|
|
25
|
+
addCommonCliArgs(initParser, configuration);
|
|
26
|
+
initParser.add_argument("--no-config-file", {
|
|
27
|
+
help: "do not create a configuration file in package.json",
|
|
28
|
+
action: "store_true",
|
|
29
|
+
default: false,
|
|
30
|
+
});
|
|
31
|
+
initParser.add_argument("--only-config-file", {
|
|
32
|
+
help: "create only the configuration file in package.json, without initializing schemas",
|
|
33
|
+
action: "store_true",
|
|
34
|
+
default: false,
|
|
35
|
+
});
|
|
36
|
+
initParser.set_defaults({ func: init });
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function init(args: InitArgs): void {
|
|
40
|
+
if (args.only_config_file && args.no_config_file) {
|
|
41
|
+
throw new Error("The options --only-config-file and --no-config-file were defined together: doing nothing.");
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
console.log("Initializing Apify Schema Tools in the current project...");
|
|
45
|
+
|
|
46
|
+
if (!args.no_config_file) {
|
|
47
|
+
writeConfigurationToPackageJson(args);
|
|
48
|
+
console.log("Configuration written to package.json");
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
if (args.only_config_file) {
|
|
52
|
+
console.log("Only configuration file created, skipping schema initialization.");
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
if (args.input.includes("input")) {
|
|
57
|
+
initializeInputSchema(args);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
if (args.input.includes("dataset")) {
|
|
61
|
+
initializeDatasetSchema(args);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function initializeInputSchema(args: InitArgs): void {
|
|
66
|
+
if (existsSync(args.src_input)) {
|
|
67
|
+
console.log(`Input schema already exists at ${args.src_input}, skipping initialization.`);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
if (!existsSync(args.input_schema)) {
|
|
71
|
+
throw new Error("The current Actor does not have an input schema.");
|
|
72
|
+
}
|
|
73
|
+
const inputSchema = readJsonSchema(args.input_schema);
|
|
74
|
+
writeJsonSchema(args.src_input, inputSchema);
|
|
75
|
+
console.log(`Input schema initialized at ${args.src_input}`);
|
|
76
|
+
if (args.add_input) {
|
|
77
|
+
writeJsonSchema(args.add_input, { type: "object", properties: {} });
|
|
78
|
+
console.log(`Additional input schema initialized at ${args.add_input}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function initializeDatasetSchema(args: InitArgs): void {
|
|
83
|
+
if (!existsSync(ACTOR_CONFIG_PATH)) {
|
|
84
|
+
throw new Error(`The current Actor does not have an ${ACTOR_CONFIG_PATH} configuration file.`);
|
|
85
|
+
}
|
|
86
|
+
const actorConfig = JSON.parse(readFile(ACTOR_CONFIG_PATH));
|
|
87
|
+
const relativeDatasetPath = getPathRelativeToActorConfig(args.dataset_schema);
|
|
88
|
+
if (actorConfig.storages?.dataset !== relativeDatasetPath) {
|
|
89
|
+
writeFile(
|
|
90
|
+
ACTOR_CONFIG_PATH,
|
|
91
|
+
JSON.stringify(
|
|
92
|
+
{
|
|
93
|
+
...actorConfig,
|
|
94
|
+
storages: {
|
|
95
|
+
...actorConfig.storages,
|
|
96
|
+
dataset: relativeDatasetPath,
|
|
97
|
+
},
|
|
98
|
+
},
|
|
99
|
+
null,
|
|
100
|
+
4,
|
|
101
|
+
),
|
|
102
|
+
);
|
|
103
|
+
console.log(`Updated ${ACTOR_CONFIG_PATH} to use the dataset schema at ${args.dataset_schema}`);
|
|
104
|
+
}
|
|
105
|
+
if (!existsSync(args.dataset_schema)) {
|
|
106
|
+
writeFile(
|
|
107
|
+
args.dataset_schema,
|
|
108
|
+
JSON.stringify(
|
|
109
|
+
{
|
|
110
|
+
actorSpecification: 1,
|
|
111
|
+
[DATASET_SCHEMA_FIELD]: { type: "object", properties: {} },
|
|
112
|
+
},
|
|
113
|
+
null,
|
|
114
|
+
4,
|
|
115
|
+
),
|
|
116
|
+
);
|
|
117
|
+
console.log(`Dataset schema initialized at ${args.dataset_schema}`);
|
|
118
|
+
}
|
|
119
|
+
if (existsSync(args.src_dataset)) {
|
|
120
|
+
console.log(`Dataset schema already exists at ${args.src_dataset}, skipping initialization.`);
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
const datasetItemSchema = readJsonSchemaField(args.dataset_schema, DATASET_SCHEMA_FIELD);
|
|
124
|
+
writeJsonSchema(args.src_dataset, datasetItemSchema);
|
|
125
|
+
}
|
package/src/cli/sync.ts
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/** biome-ignore-all lint/style/useNamingConvention: the package `argparse` uses snake_case names */
|
|
2
|
+
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { ArgumentDefaultsHelpFormatter, type SubParser } from "argparse";
|
|
5
|
+
import {
|
|
6
|
+
DATASET_SCHEMA_FIELD,
|
|
7
|
+
DESCRIPTION_FIELDS,
|
|
8
|
+
filterValidInputSchemaProperties,
|
|
9
|
+
generateInputDefaultsTsFileContent,
|
|
10
|
+
} from "../apify.js";
|
|
11
|
+
import { addCommonCliArgs, type CommonCliArgs, type Configuration } from "../configuration.js";
|
|
12
|
+
import { serializeMiddleObjectToTypeScript, writeTypeScriptFile } from "../middle-schema/generate-typescript.js";
|
|
13
|
+
import { jsonSchemaToMiddleObject } from "../middle-schema/parse-json-schema.js";
|
|
14
|
+
import {
|
|
15
|
+
mergeObjectSchemas,
|
|
16
|
+
type ObjectSchema,
|
|
17
|
+
readJsonSchema,
|
|
18
|
+
readJsonSchemaField,
|
|
19
|
+
writeJsonSchema,
|
|
20
|
+
writeSchemaToField,
|
|
21
|
+
} from "../utils/json-schemas.js";
|
|
22
|
+
import { type ConflictResolutionStrategy, checkConflicts } from "../utils/json-schemas-interactive-conflict.js";
|
|
23
|
+
|
|
24
|
+
interface SyncArgs extends CommonCliArgs {
|
|
25
|
+
include_input_utils: string;
|
|
26
|
+
force: boolean;
|
|
27
|
+
fail_on_conflict: boolean;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function setupSyncParser(subparsers: SubParser, configuration: Configuration): void {
|
|
31
|
+
const syncParser = subparsers.add_parser("sync", {
|
|
32
|
+
help: `Generate JSON schemas and TypeScript files from the source schemas. \
|
|
33
|
+
By default, if conflicts are detected, the user will be prompted to resolve them.`,
|
|
34
|
+
formatter_class: ArgumentDefaultsHelpFormatter,
|
|
35
|
+
});
|
|
36
|
+
addCommonCliArgs(syncParser, configuration);
|
|
37
|
+
syncParser.add_argument("--include-input-utils", {
|
|
38
|
+
help: "include input utilities in the generated TypeScript files: 'input' input and 'ts-types' output are required",
|
|
39
|
+
choices: ["true", "false"],
|
|
40
|
+
default: "true",
|
|
41
|
+
});
|
|
42
|
+
syncParser.add_argument("--force", {
|
|
43
|
+
help: "force the sync operation, even if conflicts are detected",
|
|
44
|
+
action: "store_true",
|
|
45
|
+
default: false,
|
|
46
|
+
});
|
|
47
|
+
syncParser.add_argument("--fail-on-conflict", {
|
|
48
|
+
help: "fail the sync operation if conflicts are detected",
|
|
49
|
+
action: "store_true",
|
|
50
|
+
default: false,
|
|
51
|
+
});
|
|
52
|
+
syncParser.set_defaults({ func: sync });
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function sync(args: SyncArgs): void {
|
|
56
|
+
console.log("Syncing schemas...");
|
|
57
|
+
|
|
58
|
+
if (args.input.includes("input")) {
|
|
59
|
+
handleInputSync(args);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
if (args.input.includes("dataset")) {
|
|
63
|
+
handleDatasetSync(args);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async function handleInputSync(args: SyncArgs): Promise<void> {
|
|
68
|
+
let inputSchemas = {
|
|
69
|
+
sourceSchema: readJsonSchema(args.src_input),
|
|
70
|
+
additionalSchema: args.add_input ? readJsonSchema(args.add_input) : undefined,
|
|
71
|
+
};
|
|
72
|
+
let resultingInputSchema: ObjectSchema | undefined;
|
|
73
|
+
if (args.output.includes("json-schemas")) {
|
|
74
|
+
const conflictResolutionStrategy = getConflictResolutionStrategy(args);
|
|
75
|
+
const existingInputSchema = readJsonSchema(args.input_schema);
|
|
76
|
+
inputSchemas = await checkConflicts(
|
|
77
|
+
inputSchemas.sourceSchema,
|
|
78
|
+
args.src_input,
|
|
79
|
+
inputSchemas.additionalSchema,
|
|
80
|
+
args.add_input,
|
|
81
|
+
existingInputSchema,
|
|
82
|
+
args.deep_merge,
|
|
83
|
+
DESCRIPTION_FIELDS,
|
|
84
|
+
conflictResolutionStrategy,
|
|
85
|
+
);
|
|
86
|
+
resultingInputSchema = inputSchemas.additionalSchema
|
|
87
|
+
? mergeObjectSchemas(inputSchemas.sourceSchema, inputSchemas.additionalSchema, args.deep_merge)
|
|
88
|
+
: inputSchemas.sourceSchema;
|
|
89
|
+
writeJsonSchema(args.input_schema, filterValidInputSchemaProperties(resultingInputSchema));
|
|
90
|
+
}
|
|
91
|
+
if (args.output.includes("ts-types")) {
|
|
92
|
+
resultingInputSchema ??= inputSchemas.additionalSchema
|
|
93
|
+
? mergeObjectSchemas(inputSchemas.sourceSchema, inputSchemas.additionalSchema, args.deep_merge)
|
|
94
|
+
: inputSchemas.sourceSchema;
|
|
95
|
+
writeTypeScriptFile(
|
|
96
|
+
join(args.output_ts_dir, "input.ts"),
|
|
97
|
+
serializeMiddleObjectToTypeScript("Input", jsonSchemaToMiddleObject(resultingInputSchema)),
|
|
98
|
+
);
|
|
99
|
+
if (args.include_input_utils === "true") {
|
|
100
|
+
writeTypeScriptFile(
|
|
101
|
+
join(args.output_ts_dir, "input-utils.ts"),
|
|
102
|
+
generateInputDefaultsTsFileContent(resultingInputSchema),
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
// Update the source schemas, which may have been modified by the conflict resolution
|
|
107
|
+
writeJsonSchema(args.src_input, inputSchemas.sourceSchema);
|
|
108
|
+
if (args.add_input && inputSchemas.additionalSchema) {
|
|
109
|
+
writeJsonSchema(args.add_input, inputSchemas.additionalSchema);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async function handleDatasetSync(args: SyncArgs): Promise<void> {
|
|
114
|
+
let datasetSchemas = {
|
|
115
|
+
sourceSchema: readJsonSchema(args.src_dataset),
|
|
116
|
+
additionalSchema: args.add_dataset ? readJsonSchema(args.add_dataset) : undefined,
|
|
117
|
+
};
|
|
118
|
+
let resultingDatasetSchema: ObjectSchema | undefined;
|
|
119
|
+
if (args.output.includes("json-schemas")) {
|
|
120
|
+
const conflictResolutionStrategy = getConflictResolutionStrategy(args);
|
|
121
|
+
const existingDatasetSchema = readJsonSchemaField(args.dataset_schema, DATASET_SCHEMA_FIELD);
|
|
122
|
+
datasetSchemas = await checkConflicts(
|
|
123
|
+
datasetSchemas.sourceSchema,
|
|
124
|
+
args.src_dataset,
|
|
125
|
+
datasetSchemas.additionalSchema,
|
|
126
|
+
args.add_dataset,
|
|
127
|
+
existingDatasetSchema,
|
|
128
|
+
args.deep_merge,
|
|
129
|
+
DESCRIPTION_FIELDS,
|
|
130
|
+
conflictResolutionStrategy,
|
|
131
|
+
);
|
|
132
|
+
resultingDatasetSchema = datasetSchemas.additionalSchema
|
|
133
|
+
? mergeObjectSchemas(datasetSchemas.sourceSchema, datasetSchemas.additionalSchema, args.deep_merge)
|
|
134
|
+
: datasetSchemas.sourceSchema;
|
|
135
|
+
writeSchemaToField(args.dataset_schema, resultingDatasetSchema, DATASET_SCHEMA_FIELD);
|
|
136
|
+
}
|
|
137
|
+
if (args.output.includes("ts-types")) {
|
|
138
|
+
resultingDatasetSchema ??= datasetSchemas.additionalSchema
|
|
139
|
+
? mergeObjectSchemas(datasetSchemas.sourceSchema, datasetSchemas.additionalSchema, args.deep_merge)
|
|
140
|
+
: datasetSchemas.sourceSchema;
|
|
141
|
+
writeTypeScriptFile(
|
|
142
|
+
join(args.output_ts_dir, "dataset.ts"),
|
|
143
|
+
serializeMiddleObjectToTypeScript("DatasetItem", jsonSchemaToMiddleObject(resultingDatasetSchema)),
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
// Update the source schemas, which may have been modified by the conflict resolution
|
|
147
|
+
writeJsonSchema(args.src_dataset, datasetSchemas.sourceSchema);
|
|
148
|
+
if (args.add_dataset && datasetSchemas.additionalSchema) {
|
|
149
|
+
writeJsonSchema(args.add_dataset, datasetSchemas.additionalSchema);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function getConflictResolutionStrategy(args: SyncArgs): ConflictResolutionStrategy {
|
|
154
|
+
if (args.fail_on_conflict && args.force) {
|
|
155
|
+
throw new Error("The options --force and --fail-on-conflict cannot be defined together.");
|
|
156
|
+
}
|
|
157
|
+
if (args.force) {
|
|
158
|
+
return "log";
|
|
159
|
+
}
|
|
160
|
+
if (args.fail_on_conflict) {
|
|
161
|
+
return "error";
|
|
162
|
+
}
|
|
163
|
+
return "interactive";
|
|
164
|
+
}
|
package/src/configuration.ts
CHANGED
|
@@ -2,12 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
import { existsSync } from "node:fs";
|
|
4
4
|
import type { ArgumentParser } from "argparse";
|
|
5
|
-
import { array, enum as enum_, object, string, type infer as zodInfer } from "zod/v4";
|
|
6
|
-
import { readFile, writeFile } from "./filesystem.js";
|
|
5
|
+
import { array, boolean, enum as enum_, object, string, type infer as zodInfer } from "zod/v4";
|
|
6
|
+
import { readFile, writeFile } from "./utils/filesystem.js";
|
|
7
7
|
|
|
8
8
|
const zod = {
|
|
9
9
|
object,
|
|
10
10
|
array,
|
|
11
|
+
boolean,
|
|
11
12
|
enum: enum_,
|
|
12
13
|
string,
|
|
13
14
|
};
|
|
@@ -18,6 +19,10 @@ const OUTPUTS = ["json-schemas", "ts-types"] as const;
|
|
|
18
19
|
type Input = (typeof INPUTS)[number];
|
|
19
20
|
type Output = (typeof OUTPUTS)[number];
|
|
20
21
|
|
|
22
|
+
/**
|
|
23
|
+
* These CLI arguments are also available in the configuration file.
|
|
24
|
+
* They are in snake_case because `argparse` uses it for argument names.
|
|
25
|
+
*/
|
|
21
26
|
export interface CommonCliArgs {
|
|
22
27
|
input: Input[];
|
|
23
28
|
output: Output[];
|
|
@@ -25,27 +30,35 @@ export interface CommonCliArgs {
|
|
|
25
30
|
src_dataset: string;
|
|
26
31
|
add_input?: string;
|
|
27
32
|
add_dataset?: string;
|
|
33
|
+
deep_merge: boolean;
|
|
28
34
|
input_schema: string;
|
|
29
35
|
dataset_schema: string;
|
|
30
36
|
output_ts_dir: string;
|
|
31
37
|
}
|
|
32
38
|
|
|
33
|
-
|
|
39
|
+
/**
|
|
40
|
+
* Zod schema for the configuration file.
|
|
41
|
+
* Since the configuration is in JSON format, we use camelCase.
|
|
42
|
+
*/
|
|
43
|
+
export const CONFIGURATION_SCHEMA = zod.object({
|
|
34
44
|
input: zod.array(zod.enum(INPUTS)).default([...INPUTS]),
|
|
35
45
|
output: zod.array(zod.enum(OUTPUTS)).default([...OUTPUTS]),
|
|
36
46
|
srcInput: zod.string().default("src-schemas/input.json"),
|
|
37
47
|
srcDataset: zod.string().default("src-schemas/dataset-item.json"),
|
|
38
48
|
addInput: zod.string().optional(),
|
|
39
49
|
addDataset: zod.string().optional(),
|
|
50
|
+
deepMerge: zod.boolean().default(false),
|
|
40
51
|
inputSchema: zod.string().default(".actor/input_schema.json"),
|
|
41
52
|
datasetSchema: zod.string().default(".actor/dataset_schema.json"),
|
|
42
53
|
outputTSDir: zod.string().default("src/generated"),
|
|
43
54
|
});
|
|
44
55
|
|
|
45
|
-
export
|
|
56
|
+
export type Configuration = zodInfer<typeof CONFIGURATION_SCHEMA>;
|
|
57
|
+
|
|
58
|
+
export function parseConfigurationFromFileOrDefault(): Configuration {
|
|
46
59
|
const packageJsonContent = existsSync("package.json") ? readFile("package.json") : undefined;
|
|
47
60
|
const rawConfig = packageJsonContent ? (JSON.parse(packageJsonContent)["apify-schema-tools"] ?? {}) : {};
|
|
48
|
-
return
|
|
61
|
+
return CONFIGURATION_SCHEMA.parse(rawConfig);
|
|
49
62
|
}
|
|
50
63
|
|
|
51
64
|
export function writeConfigurationToPackageJson(args: CommonCliArgs): void {
|
|
@@ -61,6 +74,7 @@ export function writeConfigurationToPackageJson(args: CommonCliArgs): void {
|
|
|
61
74
|
srcDataset: args.src_dataset,
|
|
62
75
|
addInput: args.add_input,
|
|
63
76
|
addDataset: args.add_dataset,
|
|
77
|
+
deepMerge: args.deep_merge,
|
|
64
78
|
inputSchema: args.input_schema,
|
|
65
79
|
datasetSchema: args.dataset_schema,
|
|
66
80
|
outputTSDir: args.output_ts_dir,
|
|
@@ -69,10 +83,10 @@ export function writeConfigurationToPackageJson(args: CommonCliArgs): void {
|
|
|
69
83
|
}
|
|
70
84
|
|
|
71
85
|
/**
|
|
72
|
-
* This function will set as default values the configuration
|
|
73
|
-
* In this way, the CLI arguments can override the configuration.
|
|
86
|
+
* This function will set as default values the configuration file, or the default one.
|
|
87
|
+
* In this way, the CLI arguments can override the configuration file.
|
|
74
88
|
*/
|
|
75
|
-
export function
|
|
89
|
+
export function addCommonCliArgs(parser: ArgumentParser, configuration: zodInfer<typeof CONFIGURATION_SCHEMA>): void {
|
|
76
90
|
parser.add_argument("-i", "--input", {
|
|
77
91
|
help: "specify which sources to use for generation",
|
|
78
92
|
choices: [...INPUTS],
|
|
@@ -103,6 +117,11 @@ export function addCommonCLIArgs(parser: ArgumentParser, configuration: zodInfer
|
|
|
103
117
|
help: "path to an additional schema to merge into the dataset schema",
|
|
104
118
|
default: configuration.addDataset,
|
|
105
119
|
});
|
|
120
|
+
parser.add_argument("--deep-merge", {
|
|
121
|
+
help: "whether to deep merge additional schemas into the main schema",
|
|
122
|
+
action: "store_true",
|
|
123
|
+
default: configuration.deepMerge,
|
|
124
|
+
});
|
|
106
125
|
|
|
107
126
|
parser.add_argument("--input-schema", {
|
|
108
127
|
help: "the path of the destination input schema file",
|
package/src/main.ts
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/** biome-ignore-all lint/style/useNamingConvention: the package `argparse` uses snake_case names */
|
|
4
|
+
|
|
5
|
+
import { ArgumentDefaultsHelpFormatter, ArgumentParser } from "argparse";
|
|
6
|
+
import { setupCheckParser } from "./cli/check.js";
|
|
7
|
+
import { setupInitParser } from "./cli/init.js";
|
|
8
|
+
import { setupSyncParser } from "./cli/sync.js";
|
|
9
|
+
import { parseConfigurationFromFileOrDefault } from "./configuration.js";
|
|
10
|
+
|
|
11
|
+
const configuration = parseConfigurationFromFileOrDefault();
|
|
12
|
+
|
|
13
|
+
const rootParser = new ArgumentParser({
|
|
14
|
+
description: "Apify Schema Tools - Generate JSON schemas and TypeScript files for Actor input and output dataset.",
|
|
15
|
+
formatter_class: ArgumentDefaultsHelpFormatter,
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
const subparsers = rootParser.add_subparsers();
|
|
19
|
+
|
|
20
|
+
setupInitParser(subparsers, configuration);
|
|
21
|
+
setupSyncParser(subparsers, configuration);
|
|
22
|
+
setupCheckParser(subparsers, configuration);
|
|
23
|
+
|
|
24
|
+
const parsedArgs = rootParser.parse_args();
|
|
25
|
+
parsedArgs.func(parsedArgs);
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { equals } from "ramda";
|
|
2
|
+
import {
|
|
3
|
+
isMiddleBasicVar,
|
|
4
|
+
isMiddleEnum,
|
|
5
|
+
isMiddleObject,
|
|
6
|
+
type MiddleBasicVar,
|
|
7
|
+
type MiddleEnum,
|
|
8
|
+
type MiddleObject,
|
|
9
|
+
type MiddleSchema,
|
|
10
|
+
} from "./schema-types.js";
|
|
11
|
+
|
|
12
|
+
function compareObjectProperties(key: string, propA: MiddleObject, propB: MiddleSchema, ignoreDocs?: boolean): boolean {
|
|
13
|
+
if (!isMiddleObject(propB)) {
|
|
14
|
+
console.error(`Property "${key}" is an interface in one schema but not in the other.`);
|
|
15
|
+
return false;
|
|
16
|
+
}
|
|
17
|
+
if (!compareMiddleObjects(propA, propB, ignoreDocs)) {
|
|
18
|
+
console.error(`Property "${key}" interfaces do not match.`);
|
|
19
|
+
return false;
|
|
20
|
+
}
|
|
21
|
+
return true;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function compareEnumProperties(key: string, propA: MiddleSchema, propB: MiddleSchema): boolean {
|
|
25
|
+
if (!isMiddleEnum(propB)) {
|
|
26
|
+
console.error(`Property "${key}" is an enum in one schema but not in the other.`);
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
if (JSON.stringify((propA as MiddleEnum).enum) !== JSON.stringify((propB as MiddleEnum).enum)) {
|
|
30
|
+
console.error(
|
|
31
|
+
`Property "${key}" enums do not match: ${JSON.stringify((propA as MiddleEnum).enum)} vs ${JSON.stringify((propB as MiddleEnum).enum)}`,
|
|
32
|
+
);
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function compareBasicVarProperties(key: string, propA: MiddleSchema, propB: MiddleSchema): boolean {
|
|
39
|
+
if (!isMiddleBasicVar(propB)) {
|
|
40
|
+
console.error(`Property "${key}" is a basic var in one schema but not in the other.`);
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
if (!equals((propA as MiddleBasicVar).type, (propB as MiddleBasicVar).type)) {
|
|
44
|
+
console.error(
|
|
45
|
+
`Property "${key}" types do not match: ${JSON.stringify((propA as MiddleBasicVar).type)} vs ${JSON.stringify((propB as MiddleBasicVar).type)}`,
|
|
46
|
+
);
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
return true;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function compareProperties(key: string, propA: MiddleSchema, propB: MiddleSchema, ignoreDocs?: boolean): boolean {
|
|
53
|
+
if (propA.isArray !== propB.isArray) {
|
|
54
|
+
console.error(`Property "${key}" has different array status: ${propA.isArray} vs ${propB.isArray}`);
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (propA.isRequired !== propB.isRequired) {
|
|
59
|
+
console.error(`Property "${key}" has different required status: ${propA.isRequired} vs ${propB.isRequired}`);
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (!ignoreDocs && propA.doc !== propB.doc) {
|
|
64
|
+
console.error(`Property "${key}" has different documentation: "${propA.doc}" vs "${propB.doc}"`);
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (isMiddleObject(propA)) {
|
|
69
|
+
return compareObjectProperties(key, propA, propB, ignoreDocs);
|
|
70
|
+
}
|
|
71
|
+
if (isMiddleEnum(propA)) {
|
|
72
|
+
return compareEnumProperties(key, propA, propB);
|
|
73
|
+
}
|
|
74
|
+
return compareBasicVarProperties(key, propA, propB);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function compareAdditionalProperties(a: MiddleObject, b: MiddleObject, ignoreDocs?: boolean): boolean {
|
|
78
|
+
if (!(a.additionalProperties || b.additionalProperties)) {
|
|
79
|
+
return true; // Both have no additional properties
|
|
80
|
+
}
|
|
81
|
+
if (!(a.additionalProperties && b.additionalProperties)) {
|
|
82
|
+
console.error(`One interface has additionalProperties defined, but the other does not.
|
|
83
|
+
If you generated an interface from a JSON schema, pay attention that "additionalProperties" is set to an empty schema by default.`);
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
return compareProperties("[key: string]", a.additionalProperties, b.additionalProperties, ignoreDocs);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function compareMiddleObjects(a: MiddleObject, b: MiddleObject, ignoreDocs?: boolean): boolean {
|
|
90
|
+
if (Object.keys(a.properties).length !== Object.keys(b.properties).length) {
|
|
91
|
+
console.error("Interfaces have different number of properties.");
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
for (const key in a.properties) {
|
|
96
|
+
if (Object.hasOwn(a.properties, key)) {
|
|
97
|
+
if (!(key in b.properties)) {
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
100
|
+
const propA = a.properties[key];
|
|
101
|
+
const propB = b.properties[key];
|
|
102
|
+
if (!compareProperties(key, propA, propB, ignoreDocs)) {
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (!compareAdditionalProperties(a, b, ignoreDocs)) {
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
return true;
|
|
113
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { readFile, writeFile } from "../utils/filesystem.js";
|
|
2
|
+
import {
|
|
3
|
+
isMiddleBasicVar,
|
|
4
|
+
isMiddleEnum,
|
|
5
|
+
isMiddleObject,
|
|
6
|
+
type MiddleBasicVar,
|
|
7
|
+
type MiddleEnum,
|
|
8
|
+
type MiddleObject,
|
|
9
|
+
type MiddleSchema,
|
|
10
|
+
} from "./schema-types.js";
|
|
11
|
+
|
|
12
|
+
export const TYPESCRIPT_FILE_HEADER = `\
|
|
13
|
+
/**
|
|
14
|
+
* This file was automatically generated by apify-schema-tools.
|
|
15
|
+
* DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
|
|
16
|
+
* and run apify-schema-tools' "sync" command to regenerate this file.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
`;
|
|
20
|
+
|
|
21
|
+
function serializeInterfaceSchema(schema: MiddleObject, indentLevel: number): string {
|
|
22
|
+
const indent = "\t".repeat(indentLevel);
|
|
23
|
+
const properties = Object.entries(schema.properties);
|
|
24
|
+
if (properties.length === 0 && !schema.additionalProperties) {
|
|
25
|
+
return "{}";
|
|
26
|
+
}
|
|
27
|
+
const innerIndentLevel = indentLevel + 1;
|
|
28
|
+
const innerIndent = "\t".repeat(innerIndentLevel);
|
|
29
|
+
let result = "{";
|
|
30
|
+
for (const [key, value] of properties) {
|
|
31
|
+
if (value.doc) {
|
|
32
|
+
result += `\n${innerIndent}/**\n${innerIndent} * ${value.doc}\n${innerIndent} */`;
|
|
33
|
+
}
|
|
34
|
+
result += `\n${innerIndent}${key}${
|
|
35
|
+
value.isRequired ? "" : "?"
|
|
36
|
+
}: ${serializeMiddleSchemaToTypeScript(value, innerIndentLevel)};`;
|
|
37
|
+
}
|
|
38
|
+
if (schema.additionalProperties) {
|
|
39
|
+
result += `\n${innerIndent}[key: string]: ${serializeMiddleSchemaToTypeScript(
|
|
40
|
+
schema.additionalProperties,
|
|
41
|
+
innerIndentLevel,
|
|
42
|
+
)};`;
|
|
43
|
+
}
|
|
44
|
+
result += `\n${indent}}`;
|
|
45
|
+
return result;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function serializeEnumSchema(schema: MiddleEnum): string {
|
|
49
|
+
const serializedEnum = JSON.stringify(schema.enum).replace(/\[|\]/g, "").replace(/,/g, " | ");
|
|
50
|
+
return schema.isArray && schema.enum.length > 1 ? `(${serializedEnum})` : serializedEnum;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function serializeBasicVarSchema(schema: MiddleBasicVar): string {
|
|
54
|
+
if (Array.isArray(schema.type)) {
|
|
55
|
+
return schema.isArray ? `(${schema.type.join(" | ")})` : schema.type.join(" | ");
|
|
56
|
+
}
|
|
57
|
+
return schema.type;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function serializeMiddleSchemaToTypeScript(schema: MiddleSchema, indentLevel = 0): string {
|
|
61
|
+
let result = "";
|
|
62
|
+
if (isMiddleObject(schema)) {
|
|
63
|
+
result = serializeInterfaceSchema(schema, indentLevel);
|
|
64
|
+
} else if (isMiddleEnum(schema)) {
|
|
65
|
+
result = serializeEnumSchema(schema);
|
|
66
|
+
} else if (isMiddleBasicVar(schema)) {
|
|
67
|
+
result = serializeBasicVarSchema(schema);
|
|
68
|
+
}
|
|
69
|
+
if (schema.isArray) {
|
|
70
|
+
result += "[]";
|
|
71
|
+
}
|
|
72
|
+
return result;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function serializeMiddleObjectToTypeScript(name: string, schema: MiddleObject): string {
|
|
76
|
+
const serializedSchema = serializeMiddleSchemaToTypeScript(schema);
|
|
77
|
+
const docComment = schema.doc ? `/**\n * ${schema.doc}\n */\n` : "";
|
|
78
|
+
return `${docComment}export interface ${name} ${serializedSchema}${schema.isRequired ? "" : " | undefined"}`;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function writeTypeScriptFile(filePath: string, content: string): void {
|
|
82
|
+
writeFile(filePath, TYPESCRIPT_FILE_HEADER + content);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function readGeneratedTypeScriptFile(filePath: string): string {
|
|
86
|
+
const content = readFile(filePath);
|
|
87
|
+
return content.replace(TYPESCRIPT_FILE_HEADER, "");
|
|
88
|
+
}
|