apify-schema-tools 3.1.0 → 3.2.1

package/src/apify-schema-tools.ts

@@ -1,420 +0,0 @@
-#!/usr/bin/env node
-
-/** biome-ignore-all lint/style/useNamingConvention: the package `argparse` uses snake_case names */
-
-import { existsSync } from "node:fs";
-import { join } from "node:path";
-import { ArgumentDefaultsHelpFormatter, ArgumentParser } from "argparse";
-import {
-	ACTOR_CONFIG_PATH,
-	DATASET_SCHEMA_FIELD,
-	DESCRIPTION_FIELDS,
-	filterValidInputSchemaProperties,
-	generateInputDefaultsTsFileContent,
-	getPathRelativeToActorConfig,
-} from "./apify.js";
-import {
-	addCommonCliArgs,
-	type CommonCliArgs,
-	parseConfigurationFromFileOrDefault,
-	writeConfigurationToPackageJson,
-} from "./configuration.js";
-import { readFile, writeFile } from "./filesystem.js";
-import { type ConflictResolutionStrategy, checkConflicts } from "./json-schema-conflicts.js";
-import {
-	compareSchemas,
-	mergeObjectSchemas,
-	type ObjectSchema,
-	readJsonSchema,
-	readJsonSchemaField,
-	writeJsonSchema,
-	writeSchemaToField,
-} from "./json-schemas.js";
-import {
-	compareTypescriptInterfaces,
-	jsonSchemaToTypeScriptInterface,
-	parseTypeScriptInterface,
-	removeTypeScriptHeader,
-	serializeTypeScriptInterface,
-	writeTypeScriptFile,
-} from "./typescript.js";
-
-const configuration = parseConfigurationFromFileOrDefault();
-
-const rootParser = new ArgumentParser({
-	description: "Apify Schema Tools - Generate JSON schemas and TypeScript files for Actor input and output dataset.",
-	formatter_class: ArgumentDefaultsHelpFormatter,
-});
-
-// Create subparsers for commands, and define arguments shared by multiple commands
-
-const subparsers = rootParser.add_subparsers();
-
-// init command
-
-interface InitArgs extends CommonCliArgs {
-	no_config_file: boolean;
-	only_config_file: boolean;
-}
-
-const initParser = subparsers.add_parser("init", {
-	help: "Initialize the Apify Schema Tools project with default settings.",
-	formatter_class: ArgumentDefaultsHelpFormatter,
-});
-addCommonCliArgs(initParser, configuration);
-initParser.add_argument("--no-config-file", {
-	help: "do not create a configuration file in package.json",
-	action: "store_true",
-	default: false,
-});
-initParser.add_argument("--only-config-file", {
-	help: "create only the configuration file in package.json, without initializing schemas",
-	action: "store_true",
-	default: false,
-});
-
-function initializeInputSchema(args: InitArgs): void {
-	if (existsSync(args.src_input)) {
-		console.log(`Input schema already exists at ${args.src_input}, skipping initialization.`);
-		return;
-	}
-	if (!existsSync(args.input_schema)) {
-		throw new Error("The current Actor does not have an input schema.");
-	}
-	const inputSchema = readJsonSchema(args.input_schema);
-	writeJsonSchema(args.src_input, inputSchema);
-	console.log(`Input schema initialized at ${args.src_input}`);
-	if (args.add_input) {
-		writeJsonSchema(args.add_input, { type: "object", properties: {} });
-		console.log(`Additional input schema initialized at ${args.add_input}`);
-	}
-}
-
-function initializeDatasetSchema(args: InitArgs): void {
-	if (!existsSync(ACTOR_CONFIG_PATH)) {
-		throw new Error(`The current Actor does not have an ${ACTOR_CONFIG_PATH} configuration file.`);
-	}
-	const actorConfig = JSON.parse(readFile(ACTOR_CONFIG_PATH));
-	const relativeDatasetPath = getPathRelativeToActorConfig(args.dataset_schema);
-	if (actorConfig.storages?.dataset !== relativeDatasetPath) {
-		writeFile(
-			ACTOR_CONFIG_PATH,
-			JSON.stringify(
-				{
-					...actorConfig,
-					storages: {
-						...actorConfig.storages,
-						dataset: relativeDatasetPath,
-					},
-				},
-				null,
-				4,
-			),
-		);
-		console.log(`Updated ${ACTOR_CONFIG_PATH} to use the dataset schema at ${args.dataset_schema}`);
-	}
-	if (!existsSync(args.dataset_schema)) {
-		writeFile(
-			args.dataset_schema,
-			JSON.stringify(
-				{
-					actorSpecification: 1,
-					[DATASET_SCHEMA_FIELD]: { type: "object", properties: {} },
-				},
-				null,
-				4,
-			),
-		);
-		console.log(`Dataset schema initialized at ${args.dataset_schema}`);
-	}
-	if (existsSync(args.src_dataset)) {
-		console.log(`Dataset schema already exists at ${args.src_dataset}, skipping initialization.`);
-		return;
-	}
-	const datasetItemSchema = readJsonSchemaField(args.dataset_schema, DATASET_SCHEMA_FIELD);
-	writeJsonSchema(args.src_dataset, datasetItemSchema);
-}
-
-function init(args: InitArgs): void {
-	if (args.only_config_file && args.no_config_file) {
-		throw new Error("The options --only-config-file and --no-config-file were defined together: doing nothing.");
-	}
-
-	console.log("Initializing Apify Schema Tools in the current project...");
-
-	if (!args.no_config_file) {
-		writeConfigurationToPackageJson(args);
-		console.log("Configuration written to package.json");
-	}
-
-	if (args.only_config_file) {
-		console.log("Only configuration file created, skipping schema initialization.");
-		return;
-	}
-
-	if (args.input.includes("input")) {
-		initializeInputSchema(args);
-	}
-
-	if (args.input.includes("dataset")) {
-		initializeDatasetSchema(args);
-	}
-}
-
-initParser.set_defaults({ func: init });
-
-// sync command
-
-interface SyncArgs extends CommonCliArgs {
-	include_input_utils: string;
-	force: boolean;
-	fail_on_conflict: boolean;
-}
-
-const syncParser = subparsers.add_parser("sync", {
-	help: `Generate JSON schemas and TypeScript files from the source schemas. \
-By default, if conflicts are detected, the user will be prompted to resolve them.`,
-	formatter_class: ArgumentDefaultsHelpFormatter,
-});
-addCommonCliArgs(syncParser, configuration);
-syncParser.add_argument("--include-input-utils", {
-	help: "include input utilities in the generated TypeScript files: 'input' input and 'ts-types' output are required",
-	choices: ["true", "false"],
-	default: "true",
-});
-syncParser.add_argument("--force", {
-	help: "force the sync operation, even if conflicts are detected",
-	action: "store_true",
-	default: false,
-});
-syncParser.add_argument("--fail-on-conflict", {
-	help: "fail the sync operation if conflicts are detected",
-	action: "store_true",
-	default: false,
-});
-
-function getConflictResolutionStrategy(args: SyncArgs): ConflictResolutionStrategy {
-	if (args.fail_on_conflict && args.force) {
-		throw new Error("The options --force and --fail-on-conflict cannot be defined together.");
-	}
-	if (args.force) {
-		return "log";
-	}
-	if (args.fail_on_conflict) {
-		return "error";
-	}
-	return "interactive";
-}
-
-async function handleInputSync(args: SyncArgs): Promise<void> {
-	let inputSchemas = {
-		sourceSchema: readJsonSchema(args.src_input),
-		additionalSchema: args.add_input ? readJsonSchema(args.add_input) : undefined,
-	};
-	let resultingInputSchema: ObjectSchema | undefined;
-	if (args.output.includes("json-schemas")) {
-		const conflictResolutionStrategy = getConflictResolutionStrategy(args);
-		const existingInputSchema = readJsonSchema(args.input_schema);
-		inputSchemas = await checkConflicts(
-			inputSchemas.sourceSchema,
-			args.src_input,
-			inputSchemas.additionalSchema,
-			args.add_input,
-			existingInputSchema,
-			args.deep_merge,
-			DESCRIPTION_FIELDS,
-			conflictResolutionStrategy,
-		);
-		resultingInputSchema = inputSchemas.additionalSchema
-			? mergeObjectSchemas(inputSchemas.sourceSchema, inputSchemas.additionalSchema, args.deep_merge)
-			: inputSchemas.sourceSchema;
-		writeJsonSchema(args.input_schema, filterValidInputSchemaProperties(resultingInputSchema));
-	}
-	if (args.output.includes("ts-types")) {
-		if (!resultingInputSchema) {
-			resultingInputSchema = inputSchemas.additionalSchema
-				? mergeObjectSchemas(inputSchemas.sourceSchema, inputSchemas.additionalSchema, args.deep_merge)
-				: inputSchemas.sourceSchema;
-		}
-		writeTypeScriptFile(
-			join(args.output_ts_dir, "input.ts"),
-			serializeTypeScriptInterface("Input", jsonSchemaToTypeScriptInterface(resultingInputSchema)),
-		);
-		if (args.include_input_utils === "true") {
-			writeTypeScriptFile(
-				join(args.output_ts_dir, "input-utils.ts"),
-				generateInputDefaultsTsFileContent(resultingInputSchema),
-			);
-		}
-	}
-	// Update the source schemas, which may have been modified by the conflict resolution
-	writeJsonSchema(args.src_input, inputSchemas.sourceSchema);
-	if (args.add_input && inputSchemas.additionalSchema) {
-		writeJsonSchema(args.add_input, inputSchemas.additionalSchema);
-	}
-}
-
-async function handleDatasetSync(args: SyncArgs): Promise<void> {
-	let datasetSchemas = {
-		sourceSchema: readJsonSchema(args.src_dataset),
-		additionalSchema: args.add_dataset ? readJsonSchema(args.add_dataset) : undefined,
-	};
-	let resultingDatasetSchema: ObjectSchema | undefined;
-	if (args.output.includes("json-schemas")) {
-		const conflictResolutionStrategy = getConflictResolutionStrategy(args);
-		const existingDatasetSchema = readJsonSchemaField(args.dataset_schema, DATASET_SCHEMA_FIELD);
-		datasetSchemas = await checkConflicts(
-			datasetSchemas.sourceSchema,
-			args.src_dataset,
-			datasetSchemas.additionalSchema,
-			args.add_dataset,
-			existingDatasetSchema,
-			args.deep_merge,
-			DESCRIPTION_FIELDS,
-			conflictResolutionStrategy,
-		);
-		resultingDatasetSchema = datasetSchemas.additionalSchema
-			? mergeObjectSchemas(datasetSchemas.sourceSchema, datasetSchemas.additionalSchema, args.deep_merge)
-			: datasetSchemas.sourceSchema;
-		writeSchemaToField(args.dataset_schema, resultingDatasetSchema, DATASET_SCHEMA_FIELD);
-	}
-	if (args.output.includes("ts-types")) {
-		if (!resultingDatasetSchema) {
-			resultingDatasetSchema = datasetSchemas.additionalSchema
-				? mergeObjectSchemas(datasetSchemas.sourceSchema, datasetSchemas.additionalSchema, args.deep_merge)
-				: datasetSchemas.sourceSchema;
-		}
-		writeTypeScriptFile(
-			join(args.output_ts_dir, "dataset.ts"),
-			serializeTypeScriptInterface("DatasetItem", jsonSchemaToTypeScriptInterface(resultingDatasetSchema)),
-		);
-	}
-	// Update the source schemas, which may have been modified by the conflict resolution
-	writeJsonSchema(args.src_dataset, datasetSchemas.sourceSchema);
-	if (args.add_dataset && datasetSchemas.additionalSchema) {
-		writeJsonSchema(args.add_dataset, datasetSchemas.additionalSchema);
-	}
-}
-
-function sync(args: SyncArgs): void {
-	console.log("Syncing schemas...");
-
-	if (args.input.includes("input")) {
-		handleInputSync(args);
-	}
-
-	if (args.input.includes("dataset")) {
-		handleDatasetSync(args);
-	}
-}
-
-syncParser.set_defaults({ func: sync });
-
-// check command
-
-interface CheckArgs extends CommonCliArgs {
-	ignore_descriptions: boolean;
-}
-
-const checkParser = subparsers.add_parser("check", {
-	help: "Check the schemas for consistency and correctness. (WARNING: input utils will not be checked!)",
-	formatter_class: ArgumentDefaultsHelpFormatter,
-});
-addCommonCliArgs(checkParser, configuration);
-checkParser.add_argument("--ignore-descriptions", {
-	help: "ignore the 'title' and 'description' fields during the comparison",
-	action: "store_true",
-	default: false,
-});
-
-function checkInputSchemas(args: CheckArgs): void {
-	let sourceInputSchema = readJsonSchema(args.src_input);
-	if (args.add_input) {
-		sourceInputSchema = mergeObjectSchemas(sourceInputSchema, readJsonSchema(args.add_input), args.deep_merge);
-	}
-	const generatedInputSchema = filterValidInputSchemaProperties(sourceInputSchema);
-	if (args.output.includes("json-schemas")) {
-		const outputInputSchema = readJsonSchema(args.input_schema);
-		const isInputSchemaCoherent = compareSchemas(
-			generatedInputSchema,
-			outputInputSchema,
-			args.ignore_descriptions ? DESCRIPTION_FIELDS : undefined,
-		);
-		if (!isInputSchemaCoherent) {
-			console.error("Input schema validation failed: the input schema does not match the source schema.");
-			process.exit(1);
-		}
-	}
-	if (args.output.includes("ts-types")) {
-		const inputTsContent = readFile(join(args.output_ts_dir, "input.ts"));
-		const inputTsInterface = parseTypeScriptInterface(removeTypeScriptHeader(inputTsContent));
-		const sourceInputTsInterface = jsonSchemaToTypeScriptInterface(sourceInputSchema);
-		const isTsInputCoherent = compareTypescriptInterfaces(
-			sourceInputTsInterface,
-			inputTsInterface,
-			args.ignore_descriptions,
-		);
-		if (!isTsInputCoherent) {
-			console.error(
-				"Input TypeScript interface validation failed: the TypeScript interface does not match the source schema.",
-			);
-			process.exit(1);
-		}
-	}
-}
-
-function checkDatasetSchemas(args: CheckArgs): void {
-	let sourceDatasetSchema = readJsonSchema(args.src_dataset);
-	if (args.add_dataset) {
-		sourceDatasetSchema = mergeObjectSchemas(sourceDatasetSchema, readJsonSchema(args.add_dataset), args.deep_merge);
-	}
-	if (args.output.includes("json-schemas")) {
-		const outputDatasetSchema = readJsonSchemaField(args.dataset_schema, DATASET_SCHEMA_FIELD);
-		if (!outputDatasetSchema) {
-			console.error(`Dataset schema field "${DATASET_SCHEMA_FIELD}" not found in ${args.dataset_schema}.`);
-			process.exit(1);
-		}
-		const isDatasetSchemaCoherent = compareSchemas(
-			sourceDatasetSchema,
-			outputDatasetSchema,
-			args.ignore_descriptions ? DESCRIPTION_FIELDS : undefined,
-		);
-		if (!isDatasetSchemaCoherent) {
-			console.error("Dataset schema validation failed: the dataset schema does not match the source schema.");
-			process.exit(1);
-		}
-	}
-	if (args.output.includes("ts-types")) {
-		const datasetTsContent = readFile(join(args.output_ts_dir, "dataset.ts"));
-		const datasetTsInterface = parseTypeScriptInterface(removeTypeScriptHeader(datasetTsContent));
-		const sourceDatasetTsInterface = jsonSchemaToTypeScriptInterface(sourceDatasetSchema);
-		const isTsDatasetCoherent = compareTypescriptInterfaces(
-			sourceDatasetTsInterface,
-			datasetTsInterface,
-			args.ignore_descriptions,
-		);
-		if (!isTsDatasetCoherent) {
-			console.error(
-				"Dataset TypeScript interface validation failed: the TypeScript interface does not match the source schema.",
-			);
-			process.exit(1);
-		}
-	}
-}
-
-function check(args: CheckArgs): void {
-	if (args.input.includes("input")) {
-		checkInputSchemas(args);
-	}
-	if (args.input.includes("dataset")) {
-		checkDatasetSchemas(args);
-	}
-	console.log("Check passed.");
-}
-
-checkParser.set_defaults({ func: check });
-
-// Parse and execute
-
-const parsedArgs = rootParser.parse_args();
-parsedArgs.func(parsedArgs);