apify-schema-tools 3.0.0 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/.cspell/custom-dictionary.txt +1 -0
  2. package/.node-version +1 -1
  3. package/CHANGELOG.md +13 -1
  4. package/biome.json +24 -18
  5. package/check-samples.sh +4 -0
  6. package/dist/apify-schema-tools.js +16 -21
  7. package/dist/apify-schema-tools.js.map +1 -1
  8. package/dist/apify.d.ts +1 -1
  9. package/dist/apify.d.ts.map +1 -1
  10. package/dist/apify.js +22 -8
  11. package/dist/apify.js.map +1 -1
  12. package/dist/cli/check.d.ts +5 -0
  13. package/dist/cli/check.d.ts.map +1 -0
  14. package/dist/cli/check.js +86 -0
  15. package/dist/cli/check.js.map +1 -0
  16. package/dist/cli/init.d.ts +5 -0
  17. package/dist/cli/init.d.ts.map +1 -0
  18. package/dist/cli/init.js +92 -0
  19. package/dist/cli/init.js.map +1 -0
  20. package/dist/cli/sync.d.ts +5 -0
  21. package/dist/cli/sync.d.ts.map +1 -0
  22. package/dist/cli/sync.js +112 -0
  23. package/dist/cli/sync.js.map +1 -0
  24. package/dist/configuration.d.ts +16 -5
  25. package/dist/configuration.d.ts.map +1 -1
  26. package/dist/configuration.js +19 -7
  27. package/dist/configuration.js.map +1 -1
  28. package/dist/json-schema-conflicts.js +3 -3
  29. package/dist/json-schema-conflicts.js.map +1 -1
  30. package/dist/json-schemas.d.ts +1 -1
  31. package/dist/json-schemas.d.ts.map +1 -1
  32. package/dist/json-schemas.js +1 -1
  33. package/dist/json-schemas.js.map +1 -1
  34. package/dist/main.d.ts +4 -0
  35. package/dist/main.d.ts.map +1 -0
  36. package/dist/main.js +19 -0
  37. package/dist/main.js.map +1 -0
  38. package/dist/middle-schema/compare-schemas.d.ts +3 -0
  39. package/dist/middle-schema/compare-schemas.d.ts.map +1 -0
  40. package/dist/middle-schema/compare-schemas.js +90 -0
  41. package/dist/middle-schema/compare-schemas.js.map +1 -0
  42. package/dist/middle-schema/generate-typescript.d.ts +7 -0
  43. package/dist/middle-schema/generate-typescript.d.ts.map +1 -0
  44. package/dist/middle-schema/generate-typescript.js +70 -0
  45. package/dist/middle-schema/generate-typescript.js.map +1 -0
  46. package/dist/middle-schema/parse-json-schema.d.ts +4 -0
  47. package/dist/middle-schema/parse-json-schema.d.ts.map +1 -0
  48. package/dist/middle-schema/parse-json-schema.js +65 -0
  49. package/dist/middle-schema/parse-json-schema.js.map +1 -0
  50. package/dist/middle-schema/parse-typescript.d.ts +4 -0
  51. package/dist/middle-schema/parse-typescript.d.ts.map +1 -0
  52. package/dist/middle-schema/parse-typescript.js +199 -0
  53. package/dist/middle-schema/parse-typescript.js.map +1 -0
  54. package/dist/middle-schema/schema-types.d.ts +24 -0
  55. package/dist/middle-schema/schema-types.d.ts.map +1 -0
  56. package/dist/middle-schema/schema-types.js +14 -0
  57. package/dist/middle-schema/schema-types.js.map +1 -0
  58. package/dist/middle-schema/schema.d.ts +24 -0
  59. package/dist/middle-schema/schema.d.ts.map +1 -0
  60. package/dist/middle-schema/schema.js +14 -0
  61. package/dist/middle-schema/schema.js.map +1 -0
  62. package/dist/schema/entities/abstract-entity.d.ts +5 -0
  63. package/dist/schema/entities/abstract-entity.d.ts.map +1 -0
  64. package/dist/schema/entities/abstract-entity.js +3 -0
  65. package/dist/schema/entities/abstract-entity.js.map +1 -0
  66. package/dist/schema/entities/primitive-union.d.ts +12 -0
  67. package/dist/schema/entities/primitive-union.d.ts.map +1 -0
  68. package/dist/schema/entities/primitive-union.js +74 -0
  69. package/dist/schema/entities/primitive-union.js.map +1 -0
  70. package/dist/schema/entities/primitive.d.ts +15 -0
  71. package/dist/schema/entities/primitive.d.ts.map +1 -0
  72. package/dist/schema/entities/primitive.js +54 -0
  73. package/dist/schema/entities/primitive.js.map +1 -0
  74. package/dist/schema/parsers/json-schema.d.ts +4 -0
  75. package/dist/schema/parsers/json-schema.d.ts.map +1 -0
  76. package/dist/schema/parsers/json-schema.js +12 -0
  77. package/dist/schema/parsers/json-schema.js.map +1 -0
  78. package/dist/schema/parsers/typescript.d.ts +3 -0
  79. package/dist/schema/parsers/typescript.d.ts.map +1 -0
  80. package/dist/schema/parsers/typescript.js +24 -0
  81. package/dist/schema/parsers/typescript.js.map +1 -0
  82. package/dist/schemas/input.d.ts +840 -0
  83. package/dist/schemas/input.d.ts.map +1 -0
  84. package/dist/schemas/input.js +349 -0
  85. package/dist/schemas/input.js.map +1 -0
  86. package/dist/utils/filesystem.d.ts +8 -0
  87. package/dist/utils/filesystem.d.ts.map +1 -0
  88. package/dist/utils/filesystem.js +16 -0
  89. package/dist/utils/filesystem.js.map +1 -0
  90. package/dist/utils/json-schemas-interactive-conflict.d.ts +16 -0
  91. package/dist/utils/json-schemas-interactive-conflict.d.ts.map +1 -0
  92. package/dist/utils/json-schemas-interactive-conflict.js +165 -0
  93. package/dist/utils/json-schemas-interactive-conflict.js.map +1 -0
  94. package/dist/utils/json-schemas.d.ts +42 -0
  95. package/dist/utils/json-schemas.d.ts.map +1 -0
  96. package/dist/utils/json-schemas.js +162 -0
  97. package/dist/utils/json-schemas.js.map +1 -0
  98. package/dist/zod/schemas/input.d.ts +840 -0
  99. package/dist/zod/schemas/input.d.ts.map +1 -0
  100. package/dist/zod/schemas/input.js +393 -0
  101. package/dist/zod/schemas/input.js.map +1 -0
  102. package/package.json +12 -12
  103. package/samples/all-defaults/.actor/input_schema.json +32 -3
  104. package/samples/all-defaults/src-schemas/input.json +2 -1
  105. package/samples/deep-merged-schemas/.actor/input_schema.json +36 -3
  106. package/samples/merged-schemas/.actor/input_schema.json +27 -3
  107. package/samples/package-json-config/.actor/input_schema.json +32 -3
  108. package/samples/package-json-config-merged/.actor/actor.json +15 -0
  109. package/samples/package-json-config-merged/.actor/dataset_schema.json +32 -0
  110. package/samples/package-json-config-merged/.actor/input_schema.json +91 -0
  111. package/samples/package-json-config-merged/custom-add-schemas/add-input.json +21 -0
  112. package/samples/package-json-config-merged/custom-src-schemas/dataset-item.json +28 -0
  113. package/samples/package-json-config-merged/custom-src-schemas/input.json +89 -0
  114. package/samples/package-json-config-merged/package.json +19 -0
  115. package/samples/package-json-config-merged/src/custom-generated/dataset.ts +25 -0
  116. package/samples/package-json-config-merged/src/custom-generated/input-utils.ts +73 -0
  117. package/samples/package-json-config-merged/src/custom-generated/input.ts +49 -0
  118. package/src/apify.ts +24 -9
  119. package/src/cli/check.ts +114 -0
  120. package/src/cli/init.ts +125 -0
  121. package/src/cli/sync.ts +164 -0
  122. package/src/configuration.ts +27 -8
  123. package/src/main.ts +25 -0
  124. package/src/middle-schema/compare-schemas.ts +113 -0
  125. package/src/middle-schema/generate-typescript.ts +88 -0
  126. package/src/middle-schema/parse-json-schema.ts +104 -0
  127. package/src/middle-schema/parse-typescript.ts +239 -0
  128. package/src/middle-schema/schema-types.ts +40 -0
  129. package/src/{json-schema-conflicts.ts → utils/json-schemas-interactive-conflict.ts} +3 -3
  130. package/src/{json-schemas.ts → utils/json-schemas.ts} +1 -1
  131. package/test/apify.test.ts +413 -5
  132. package/test/cli/check.test.ts +1571 -0
  133. package/test/cli/init.test.ts +459 -0
  134. package/test/cli/sync.test.ts +341 -0
  135. package/test/common.ts +68 -0
  136. package/test/configuration.test.ts +59 -26
  137. package/test/middle-schema/compare-schemas.test.ts +585 -0
  138. package/test/middle-schema/generate-typescript.test.ts +191 -0
  139. package/test/middle-schema/parse-json-schema.test.ts +178 -0
  140. package/test/middle-schema/parse-typescript.test.ts +143 -0
  141. package/test/{json-schema-conflicts.test.ts → utils/json-schemas-interactive-conflict.test.ts} +3 -3
  142. package/test/{json-schemas.test.ts → utils/json-schemas.test.ts} +4 -4
  143. package/update-samples.sh +4 -0
  144. package/src/apify-schema-tools.ts +0 -431
  145. package/src/typescript.ts +0 -563
  146. package/test/apify-schema-tools.test.ts +0 -2216
  147. package/test/typescript.test.ts +0 -1079
  148. /package/src/{filesystem.ts → utils/filesystem.ts} +0 -0
@@ -0,0 +1,125 @@
1
+ /** biome-ignore-all lint/style/useNamingConvention: the package `argparse` uses snake_case names */
2
+
3
+ import { existsSync } from "node:fs";
4
+ import { ArgumentDefaultsHelpFormatter, type SubParser } from "argparse";
5
+ import { ACTOR_CONFIG_PATH, DATASET_SCHEMA_FIELD, getPathRelativeToActorConfig } from "../apify.js";
6
+ import {
7
+ addCommonCliArgs,
8
+ type CommonCliArgs,
9
+ type Configuration,
10
+ writeConfigurationToPackageJson,
11
+ } from "../configuration.js";
12
+ import { readFile, writeFile } from "../utils/filesystem.js";
13
+ import { readJsonSchema, readJsonSchemaField, writeJsonSchema } from "../utils/json-schemas.js";
14
+
15
+ interface InitArgs extends CommonCliArgs {
16
+ no_config_file: boolean;
17
+ only_config_file: boolean;
18
+ }
19
+
20
+ export function setupInitParser(subparsers: SubParser, configuration: Configuration): void {
21
+ const initParser = subparsers.add_parser("init", {
22
+ help: "Initialize the Apify Schema Tools project with default settings.",
23
+ formatter_class: ArgumentDefaultsHelpFormatter,
24
+ });
25
+ addCommonCliArgs(initParser, configuration);
26
+ initParser.add_argument("--no-config-file", {
27
+ help: "do not create a configuration file in package.json",
28
+ action: "store_true",
29
+ default: false,
30
+ });
31
+ initParser.add_argument("--only-config-file", {
32
+ help: "create only the configuration file in package.json, without initializing schemas",
33
+ action: "store_true",
34
+ default: false,
35
+ });
36
+ initParser.set_defaults({ func: init });
37
+ }
38
+
39
+ function init(args: InitArgs): void {
40
+ if (args.only_config_file && args.no_config_file) {
41
+ throw new Error("The options --only-config-file and --no-config-file were defined together: doing nothing.");
42
+ }
43
+
44
+ console.log("Initializing Apify Schema Tools in the current project...");
45
+
46
+ if (!args.no_config_file) {
47
+ writeConfigurationToPackageJson(args);
48
+ console.log("Configuration written to package.json");
49
+ }
50
+
51
+ if (args.only_config_file) {
52
+ console.log("Only configuration file created, skipping schema initialization.");
53
+ return;
54
+ }
55
+
56
+ if (args.input.includes("input")) {
57
+ initializeInputSchema(args);
58
+ }
59
+
60
+ if (args.input.includes("dataset")) {
61
+ initializeDatasetSchema(args);
62
+ }
63
+ }
64
+
65
+ function initializeInputSchema(args: InitArgs): void {
66
+ if (existsSync(args.src_input)) {
67
+ console.log(`Input schema already exists at ${args.src_input}, skipping initialization.`);
68
+ return;
69
+ }
70
+ if (!existsSync(args.input_schema)) {
71
+ throw new Error("The current Actor does not have an input schema.");
72
+ }
73
+ const inputSchema = readJsonSchema(args.input_schema);
74
+ writeJsonSchema(args.src_input, inputSchema);
75
+ console.log(`Input schema initialized at ${args.src_input}`);
76
+ if (args.add_input) {
77
+ writeJsonSchema(args.add_input, { type: "object", properties: {} });
78
+ console.log(`Additional input schema initialized at ${args.add_input}`);
79
+ }
80
+ }
81
+
82
+ function initializeDatasetSchema(args: InitArgs): void {
83
+ if (!existsSync(ACTOR_CONFIG_PATH)) {
84
+ throw new Error(`The current Actor does not have an ${ACTOR_CONFIG_PATH} configuration file.`);
85
+ }
86
+ const actorConfig = JSON.parse(readFile(ACTOR_CONFIG_PATH));
87
+ const relativeDatasetPath = getPathRelativeToActorConfig(args.dataset_schema);
88
+ if (actorConfig.storages?.dataset !== relativeDatasetPath) {
89
+ writeFile(
90
+ ACTOR_CONFIG_PATH,
91
+ JSON.stringify(
92
+ {
93
+ ...actorConfig,
94
+ storages: {
95
+ ...actorConfig.storages,
96
+ dataset: relativeDatasetPath,
97
+ },
98
+ },
99
+ null,
100
+ 4,
101
+ ),
102
+ );
103
+ console.log(`Updated ${ACTOR_CONFIG_PATH} to use the dataset schema at ${args.dataset_schema}`);
104
+ }
105
+ if (!existsSync(args.dataset_schema)) {
106
+ writeFile(
107
+ args.dataset_schema,
108
+ JSON.stringify(
109
+ {
110
+ actorSpecification: 1,
111
+ [DATASET_SCHEMA_FIELD]: { type: "object", properties: {} },
112
+ },
113
+ null,
114
+ 4,
115
+ ),
116
+ );
117
+ console.log(`Dataset schema initialized at ${args.dataset_schema}`);
118
+ }
119
+ if (existsSync(args.src_dataset)) {
120
+ console.log(`Dataset schema already exists at ${args.src_dataset}, skipping initialization.`);
121
+ return;
122
+ }
123
+ const datasetItemSchema = readJsonSchemaField(args.dataset_schema, DATASET_SCHEMA_FIELD);
124
+ writeJsonSchema(args.src_dataset, datasetItemSchema);
125
+ }
@@ -0,0 +1,164 @@
1
+ /** biome-ignore-all lint/style/useNamingConvention: the package `argparse` uses snake_case names */
2
+
3
+ import { join } from "node:path";
4
+ import { ArgumentDefaultsHelpFormatter, type SubParser } from "argparse";
5
+ import {
6
+ DATASET_SCHEMA_FIELD,
7
+ DESCRIPTION_FIELDS,
8
+ filterValidInputSchemaProperties,
9
+ generateInputDefaultsTsFileContent,
10
+ } from "../apify.js";
11
+ import { addCommonCliArgs, type CommonCliArgs, type Configuration } from "../configuration.js";
12
+ import { serializeMiddleObjectToTypeScript, writeTypeScriptFile } from "../middle-schema/generate-typescript.js";
13
+ import { jsonSchemaToMiddleObject } from "../middle-schema/parse-json-schema.js";
14
+ import {
15
+ mergeObjectSchemas,
16
+ type ObjectSchema,
17
+ readJsonSchema,
18
+ readJsonSchemaField,
19
+ writeJsonSchema,
20
+ writeSchemaToField,
21
+ } from "../utils/json-schemas.js";
22
+ import { type ConflictResolutionStrategy, checkConflicts } from "../utils/json-schemas-interactive-conflict.js";
23
+
24
+ interface SyncArgs extends CommonCliArgs {
25
+ include_input_utils: string;
26
+ force: boolean;
27
+ fail_on_conflict: boolean;
28
+ }
29
+
30
+ export function setupSyncParser(subparsers: SubParser, configuration: Configuration): void {
31
+ const syncParser = subparsers.add_parser("sync", {
32
+ help: `Generate JSON schemas and TypeScript files from the source schemas. \
33
+ By default, if conflicts are detected, the user will be prompted to resolve them.`,
34
+ formatter_class: ArgumentDefaultsHelpFormatter,
35
+ });
36
+ addCommonCliArgs(syncParser, configuration);
37
+ syncParser.add_argument("--include-input-utils", {
38
+ help: "include input utilities in the generated TypeScript files: 'input' input and 'ts-types' output are required",
39
+ choices: ["true", "false"],
40
+ default: "true",
41
+ });
42
+ syncParser.add_argument("--force", {
43
+ help: "force the sync operation, even if conflicts are detected",
44
+ action: "store_true",
45
+ default: false,
46
+ });
47
+ syncParser.add_argument("--fail-on-conflict", {
48
+ help: "fail the sync operation if conflicts are detected",
49
+ action: "store_true",
50
+ default: false,
51
+ });
52
+ syncParser.set_defaults({ func: sync });
53
+ }
54
+
55
+ function sync(args: SyncArgs): void {
56
+ console.log("Syncing schemas...");
57
+
58
+ if (args.input.includes("input")) {
59
+ handleInputSync(args);
60
+ }
61
+
62
+ if (args.input.includes("dataset")) {
63
+ handleDatasetSync(args);
64
+ }
65
+ }
66
+
67
+ async function handleInputSync(args: SyncArgs): Promise<void> {
68
+ let inputSchemas = {
69
+ sourceSchema: readJsonSchema(args.src_input),
70
+ additionalSchema: args.add_input ? readJsonSchema(args.add_input) : undefined,
71
+ };
72
+ let resultingInputSchema: ObjectSchema | undefined;
73
+ if (args.output.includes("json-schemas")) {
74
+ const conflictResolutionStrategy = getConflictResolutionStrategy(args);
75
+ const existingInputSchema = readJsonSchema(args.input_schema);
76
+ inputSchemas = await checkConflicts(
77
+ inputSchemas.sourceSchema,
78
+ args.src_input,
79
+ inputSchemas.additionalSchema,
80
+ args.add_input,
81
+ existingInputSchema,
82
+ args.deep_merge,
83
+ DESCRIPTION_FIELDS,
84
+ conflictResolutionStrategy,
85
+ );
86
+ resultingInputSchema = inputSchemas.additionalSchema
87
+ ? mergeObjectSchemas(inputSchemas.sourceSchema, inputSchemas.additionalSchema, args.deep_merge)
88
+ : inputSchemas.sourceSchema;
89
+ writeJsonSchema(args.input_schema, filterValidInputSchemaProperties(resultingInputSchema));
90
+ }
91
+ if (args.output.includes("ts-types")) {
92
+ resultingInputSchema ??= inputSchemas.additionalSchema
93
+ ? mergeObjectSchemas(inputSchemas.sourceSchema, inputSchemas.additionalSchema, args.deep_merge)
94
+ : inputSchemas.sourceSchema;
95
+ writeTypeScriptFile(
96
+ join(args.output_ts_dir, "input.ts"),
97
+ serializeMiddleObjectToTypeScript("Input", jsonSchemaToMiddleObject(resultingInputSchema)),
98
+ );
99
+ if (args.include_input_utils === "true") {
100
+ writeTypeScriptFile(
101
+ join(args.output_ts_dir, "input-utils.ts"),
102
+ generateInputDefaultsTsFileContent(resultingInputSchema),
103
+ );
104
+ }
105
+ }
106
+ // Update the source schemas, which may have been modified by the conflict resolution
107
+ writeJsonSchema(args.src_input, inputSchemas.sourceSchema);
108
+ if (args.add_input && inputSchemas.additionalSchema) {
109
+ writeJsonSchema(args.add_input, inputSchemas.additionalSchema);
110
+ }
111
+ }
112
+
113
+ async function handleDatasetSync(args: SyncArgs): Promise<void> {
114
+ let datasetSchemas = {
115
+ sourceSchema: readJsonSchema(args.src_dataset),
116
+ additionalSchema: args.add_dataset ? readJsonSchema(args.add_dataset) : undefined,
117
+ };
118
+ let resultingDatasetSchema: ObjectSchema | undefined;
119
+ if (args.output.includes("json-schemas")) {
120
+ const conflictResolutionStrategy = getConflictResolutionStrategy(args);
121
+ const existingDatasetSchema = readJsonSchemaField(args.dataset_schema, DATASET_SCHEMA_FIELD);
122
+ datasetSchemas = await checkConflicts(
123
+ datasetSchemas.sourceSchema,
124
+ args.src_dataset,
125
+ datasetSchemas.additionalSchema,
126
+ args.add_dataset,
127
+ existingDatasetSchema,
128
+ args.deep_merge,
129
+ DESCRIPTION_FIELDS,
130
+ conflictResolutionStrategy,
131
+ );
132
+ resultingDatasetSchema = datasetSchemas.additionalSchema
133
+ ? mergeObjectSchemas(datasetSchemas.sourceSchema, datasetSchemas.additionalSchema, args.deep_merge)
134
+ : datasetSchemas.sourceSchema;
135
+ writeSchemaToField(args.dataset_schema, resultingDatasetSchema, DATASET_SCHEMA_FIELD);
136
+ }
137
+ if (args.output.includes("ts-types")) {
138
+ resultingDatasetSchema ??= datasetSchemas.additionalSchema
139
+ ? mergeObjectSchemas(datasetSchemas.sourceSchema, datasetSchemas.additionalSchema, args.deep_merge)
140
+ : datasetSchemas.sourceSchema;
141
+ writeTypeScriptFile(
142
+ join(args.output_ts_dir, "dataset.ts"),
143
+ serializeMiddleObjectToTypeScript("DatasetItem", jsonSchemaToMiddleObject(resultingDatasetSchema)),
144
+ );
145
+ }
146
+ // Update the source schemas, which may have been modified by the conflict resolution
147
+ writeJsonSchema(args.src_dataset, datasetSchemas.sourceSchema);
148
+ if (args.add_dataset && datasetSchemas.additionalSchema) {
149
+ writeJsonSchema(args.add_dataset, datasetSchemas.additionalSchema);
150
+ }
151
+ }
152
+
153
+ function getConflictResolutionStrategy(args: SyncArgs): ConflictResolutionStrategy {
154
+ if (args.fail_on_conflict && args.force) {
155
+ throw new Error("The options --force and --fail-on-conflict cannot be defined together.");
156
+ }
157
+ if (args.force) {
158
+ return "log";
159
+ }
160
+ if (args.fail_on_conflict) {
161
+ return "error";
162
+ }
163
+ return "interactive";
164
+ }
@@ -2,12 +2,13 @@
2
2
 
3
3
  import { existsSync } from "node:fs";
4
4
  import type { ArgumentParser } from "argparse";
5
- import { array, enum as enum_, object, string, type infer as zodInfer } from "zod/v4";
6
- import { readFile, writeFile } from "./filesystem.js";
5
+ import { array, boolean, enum as enum_, object, string, type infer as zodInfer } from "zod/v4";
6
+ import { readFile, writeFile } from "./utils/filesystem.js";
7
7
 
8
8
  const zod = {
9
9
  object,
10
10
  array,
11
+ boolean,
11
12
  enum: enum_,
12
13
  string,
13
14
  };
@@ -18,6 +19,10 @@ const OUTPUTS = ["json-schemas", "ts-types"] as const;
18
19
  type Input = (typeof INPUTS)[number];
19
20
  type Output = (typeof OUTPUTS)[number];
20
21
 
22
+ /**
23
+ * These CLI arguments are also available in the configuration file.
24
+ * They are in snake_case because `argparse` uses it for argument names.
25
+ */
21
26
  export interface CommonCliArgs {
22
27
  input: Input[];
23
28
  output: Output[];
@@ -25,27 +30,35 @@ export interface CommonCliArgs {
25
30
  src_dataset: string;
26
31
  add_input?: string;
27
32
  add_dataset?: string;
33
+ deep_merge: boolean;
28
34
  input_schema: string;
29
35
  dataset_schema: string;
30
36
  output_ts_dir: string;
31
37
  }
32
38
 
33
- export const Configuration = zod.object({
39
+ /**
40
+ * Zod schema for the configuration file.
41
+ * Since the configuration is in JSON format, we use camelCase.
42
+ */
43
+ export const CONFIGURATION_SCHEMA = zod.object({
34
44
  input: zod.array(zod.enum(INPUTS)).default([...INPUTS]),
35
45
  output: zod.array(zod.enum(OUTPUTS)).default([...OUTPUTS]),
36
46
  srcInput: zod.string().default("src-schemas/input.json"),
37
47
  srcDataset: zod.string().default("src-schemas/dataset-item.json"),
38
48
  addInput: zod.string().optional(),
39
49
  addDataset: zod.string().optional(),
50
+ deepMerge: zod.boolean().default(false),
40
51
  inputSchema: zod.string().default(".actor/input_schema.json"),
41
52
  datasetSchema: zod.string().default(".actor/dataset_schema.json"),
42
53
  outputTSDir: zod.string().default("src/generated"),
43
54
  });
44
55
 
45
- export function parseConfigurationFromFileOrDefault(): zodInfer<typeof Configuration> {
56
+ export type Configuration = zodInfer<typeof CONFIGURATION_SCHEMA>;
57
+
58
+ export function parseConfigurationFromFileOrDefault(): Configuration {
46
59
  const packageJsonContent = existsSync("package.json") ? readFile("package.json") : undefined;
47
60
  const rawConfig = packageJsonContent ? (JSON.parse(packageJsonContent)["apify-schema-tools"] ?? {}) : {};
48
- return Configuration.parse(rawConfig);
61
+ return CONFIGURATION_SCHEMA.parse(rawConfig);
49
62
  }
50
63
 
51
64
  export function writeConfigurationToPackageJson(args: CommonCliArgs): void {
@@ -61,6 +74,7 @@ export function writeConfigurationToPackageJson(args: CommonCliArgs): void {
61
74
  srcDataset: args.src_dataset,
62
75
  addInput: args.add_input,
63
76
  addDataset: args.add_dataset,
77
+ deepMerge: args.deep_merge,
64
78
  inputSchema: args.input_schema,
65
79
  datasetSchema: args.dataset_schema,
66
80
  outputTSDir: args.output_ts_dir,
@@ -69,10 +83,10 @@ export function writeConfigurationToPackageJson(args: CommonCliArgs): void {
69
83
  }
70
84
 
71
85
  /**
72
- * This function will set as default values the configuration from package.json, or the default one.
73
- * In this way, the CLI arguments can override the configuration.
86
+ * This function will set as default values the configuration file, or the default one.
87
+ * In this way, the CLI arguments can override the configuration file.
74
88
  */
75
- export function addCommonCLIArgs(parser: ArgumentParser, configuration: zodInfer<typeof Configuration>): void {
89
+ export function addCommonCliArgs(parser: ArgumentParser, configuration: zodInfer<typeof CONFIGURATION_SCHEMA>): void {
76
90
  parser.add_argument("-i", "--input", {
77
91
  help: "specify which sources to use for generation",
78
92
  choices: [...INPUTS],
@@ -103,6 +117,11 @@ export function addCommonCLIArgs(parser: ArgumentParser, configuration: zodInfer
103
117
  help: "path to an additional schema to merge into the dataset schema",
104
118
  default: configuration.addDataset,
105
119
  });
120
+ parser.add_argument("--deep-merge", {
121
+ help: "whether to deep merge additional schemas into the main schema",
122
+ action: "store_true",
123
+ default: configuration.deepMerge,
124
+ });
106
125
 
107
126
  parser.add_argument("--input-schema", {
108
127
  help: "the path of the destination input schema file",
package/src/main.ts ADDED
@@ -0,0 +1,25 @@
1
+ #!/usr/bin/env node
2
+
3
+ /** biome-ignore-all lint/style/useNamingConvention: the package `argparse` uses snake_case names */
4
+
5
+ import { ArgumentDefaultsHelpFormatter, ArgumentParser } from "argparse";
6
+ import { setupCheckParser } from "./cli/check.js";
7
+ import { setupInitParser } from "./cli/init.js";
8
+ import { setupSyncParser } from "./cli/sync.js";
9
+ import { parseConfigurationFromFileOrDefault } from "./configuration.js";
10
+
11
+ const configuration = parseConfigurationFromFileOrDefault();
12
+
13
+ const rootParser = new ArgumentParser({
14
+ description: "Apify Schema Tools - Generate JSON schemas and TypeScript files for Actor input and output dataset.",
15
+ formatter_class: ArgumentDefaultsHelpFormatter,
16
+ });
17
+
18
+ const subparsers = rootParser.add_subparsers();
19
+
20
+ setupInitParser(subparsers, configuration);
21
+ setupSyncParser(subparsers, configuration);
22
+ setupCheckParser(subparsers, configuration);
23
+
24
+ const parsedArgs = rootParser.parse_args();
25
+ parsedArgs.func(parsedArgs);
@@ -0,0 +1,113 @@
1
+ import { equals } from "ramda";
2
+ import {
3
+ isMiddleBasicVar,
4
+ isMiddleEnum,
5
+ isMiddleObject,
6
+ type MiddleBasicVar,
7
+ type MiddleEnum,
8
+ type MiddleObject,
9
+ type MiddleSchema,
10
+ } from "./schema-types.js";
11
+
12
+ function compareObjectProperties(key: string, propA: MiddleObject, propB: MiddleSchema, ignoreDocs?: boolean): boolean {
13
+ if (!isMiddleObject(propB)) {
14
+ console.error(`Property "${key}" is an interface in one schema but not in the other.`);
15
+ return false;
16
+ }
17
+ if (!compareMiddleObjects(propA, propB, ignoreDocs)) {
18
+ console.error(`Property "${key}" interfaces do not match.`);
19
+ return false;
20
+ }
21
+ return true;
22
+ }
23
+
24
+ function compareEnumProperties(key: string, propA: MiddleSchema, propB: MiddleSchema): boolean {
25
+ if (!isMiddleEnum(propB)) {
26
+ console.error(`Property "${key}" is an enum in one schema but not in the other.`);
27
+ return false;
28
+ }
29
+ if (JSON.stringify((propA as MiddleEnum).enum) !== JSON.stringify((propB as MiddleEnum).enum)) {
30
+ console.error(
31
+ `Property "${key}" enums do not match: ${JSON.stringify((propA as MiddleEnum).enum)} vs ${JSON.stringify((propB as MiddleEnum).enum)}`,
32
+ );
33
+ return false;
34
+ }
35
+ return true;
36
+ }
37
+
38
+ function compareBasicVarProperties(key: string, propA: MiddleSchema, propB: MiddleSchema): boolean {
39
+ if (!isMiddleBasicVar(propB)) {
40
+ console.error(`Property "${key}" is a basic var in one schema but not in the other.`);
41
+ return false;
42
+ }
43
+ if (!equals((propA as MiddleBasicVar).type, (propB as MiddleBasicVar).type)) {
44
+ console.error(
45
+ `Property "${key}" types do not match: ${JSON.stringify((propA as MiddleBasicVar).type)} vs ${JSON.stringify((propB as MiddleBasicVar).type)}`,
46
+ );
47
+ return false;
48
+ }
49
+ return true;
50
+ }
51
+
52
+ function compareProperties(key: string, propA: MiddleSchema, propB: MiddleSchema, ignoreDocs?: boolean): boolean {
53
+ if (propA.isArray !== propB.isArray) {
54
+ console.error(`Property "${key}" has different array status: ${propA.isArray} vs ${propB.isArray}`);
55
+ return false;
56
+ }
57
+
58
+ if (propA.isRequired !== propB.isRequired) {
59
+ console.error(`Property "${key}" has different required status: ${propA.isRequired} vs ${propB.isRequired}`);
60
+ return false;
61
+ }
62
+
63
+ if (!ignoreDocs && propA.doc !== propB.doc) {
64
+ console.error(`Property "${key}" has different documentation: "${propA.doc}" vs "${propB.doc}"`);
65
+ return false;
66
+ }
67
+
68
+ if (isMiddleObject(propA)) {
69
+ return compareObjectProperties(key, propA, propB, ignoreDocs);
70
+ }
71
+ if (isMiddleEnum(propA)) {
72
+ return compareEnumProperties(key, propA, propB);
73
+ }
74
+ return compareBasicVarProperties(key, propA, propB);
75
+ }
76
+
77
+ function compareAdditionalProperties(a: MiddleObject, b: MiddleObject, ignoreDocs?: boolean): boolean {
78
+ if (!(a.additionalProperties || b.additionalProperties)) {
79
+ return true; // Both have no additional properties
80
+ }
81
+ if (!(a.additionalProperties && b.additionalProperties)) {
82
+ console.error(`One interface has additionalProperties defined, but the other does not.
83
+ If you generated an interface from a JSON schema, pay attention that "additionalProperties" is set to an empty schema by default.`);
84
+ return false;
85
+ }
86
+ return compareProperties("[key: string]", a.additionalProperties, b.additionalProperties, ignoreDocs);
87
+ }
88
+
89
+ export function compareMiddleObjects(a: MiddleObject, b: MiddleObject, ignoreDocs?: boolean): boolean {
90
+ if (Object.keys(a.properties).length !== Object.keys(b.properties).length) {
91
+ console.error("Interfaces have different number of properties.");
92
+ return false;
93
+ }
94
+
95
+ for (const key in a.properties) {
96
+ if (Object.hasOwn(a.properties, key)) {
97
+ if (!(key in b.properties)) {
98
+ return false;
99
+ }
100
+ const propA = a.properties[key];
101
+ const propB = b.properties[key];
102
+ if (!compareProperties(key, propA, propB, ignoreDocs)) {
103
+ return false;
104
+ }
105
+ }
106
+ }
107
+
108
+ if (!compareAdditionalProperties(a, b, ignoreDocs)) {
109
+ return false;
110
+ }
111
+
112
+ return true;
113
+ }
@@ -0,0 +1,88 @@
1
+ import { readFile, writeFile } from "../utils/filesystem.js";
2
+ import {
3
+ isMiddleBasicVar,
4
+ isMiddleEnum,
5
+ isMiddleObject,
6
+ type MiddleBasicVar,
7
+ type MiddleEnum,
8
+ type MiddleObject,
9
+ type MiddleSchema,
10
+ } from "./schema-types.js";
11
+
12
+ export const TYPESCRIPT_FILE_HEADER = `\
13
+ /**
14
+ * This file was automatically generated by apify-schema-tools.
15
+ * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
16
+ * and run apify-schema-tools' "sync" command to regenerate this file.
17
+ */
18
+
19
+ `;
20
+
21
+ function serializeInterfaceSchema(schema: MiddleObject, indentLevel: number): string {
22
+ const indent = "\t".repeat(indentLevel);
23
+ const properties = Object.entries(schema.properties);
24
+ if (properties.length === 0 && !schema.additionalProperties) {
25
+ return "{}";
26
+ }
27
+ const innerIndentLevel = indentLevel + 1;
28
+ const innerIndent = "\t".repeat(innerIndentLevel);
29
+ let result = "{";
30
+ for (const [key, value] of properties) {
31
+ if (value.doc) {
32
+ result += `\n${innerIndent}/**\n${innerIndent} * ${value.doc}\n${innerIndent} */`;
33
+ }
34
+ result += `\n${innerIndent}${key}${
35
+ value.isRequired ? "" : "?"
36
+ }: ${serializeMiddleSchemaToTypeScript(value, innerIndentLevel)};`;
37
+ }
38
+ if (schema.additionalProperties) {
39
+ result += `\n${innerIndent}[key: string]: ${serializeMiddleSchemaToTypeScript(
40
+ schema.additionalProperties,
41
+ innerIndentLevel,
42
+ )};`;
43
+ }
44
+ result += `\n${indent}}`;
45
+ return result;
46
+ }
47
+
48
+ function serializeEnumSchema(schema: MiddleEnum): string {
49
+ const serializedEnum = JSON.stringify(schema.enum).replace(/\[|\]/g, "").replace(/,/g, " | ");
50
+ return schema.isArray && schema.enum.length > 1 ? `(${serializedEnum})` : serializedEnum;
51
+ }
52
+
53
+ function serializeBasicVarSchema(schema: MiddleBasicVar): string {
54
+ if (Array.isArray(schema.type)) {
55
+ return schema.isArray ? `(${schema.type.join(" | ")})` : schema.type.join(" | ");
56
+ }
57
+ return schema.type;
58
+ }
59
+
60
+ export function serializeMiddleSchemaToTypeScript(schema: MiddleSchema, indentLevel = 0): string {
61
+ let result = "";
62
+ if (isMiddleObject(schema)) {
63
+ result = serializeInterfaceSchema(schema, indentLevel);
64
+ } else if (isMiddleEnum(schema)) {
65
+ result = serializeEnumSchema(schema);
66
+ } else if (isMiddleBasicVar(schema)) {
67
+ result = serializeBasicVarSchema(schema);
68
+ }
69
+ if (schema.isArray) {
70
+ result += "[]";
71
+ }
72
+ return result;
73
+ }
74
+
75
+ export function serializeMiddleObjectToTypeScript(name: string, schema: MiddleObject): string {
76
+ const serializedSchema = serializeMiddleSchemaToTypeScript(schema);
77
+ const docComment = schema.doc ? `/**\n * ${schema.doc}\n */\n` : "";
78
+ return `${docComment}export interface ${name} ${serializedSchema}${schema.isRequired ? "" : " | undefined"}`;
79
+ }
80
+
81
+ export function writeTypeScriptFile(filePath: string, content: string): void {
82
+ writeFile(filePath, TYPESCRIPT_FILE_HEADER + content);
83
+ }
84
+
85
+ export function readGeneratedTypeScriptFile(filePath: string): string {
86
+ const content = readFile(filePath);
87
+ return content.replace(TYPESCRIPT_FILE_HEADER, "");
88
+ }