apify-schema-tools 2.0.1

package/samples/package-json-config/src/custom-generated/dataset.ts

@@ -0,0 +1,24 @@
+/**
+ * This file was automatically generated by apify-schema-tools.
+ * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
+ * and run apify-schema-tools' "sync" command to regenerate this file.
+ */
+
+export interface DatasetItem {
+	/**
+	 * Page title
+	 */
+	title: string;
+	/**
+	 * Page URL
+	 */
+	url: string;
+	/**
+	 * Extracted text
+	 */
+	text?: string;
+	/**
+	 * When the data was scraped
+	 */
+	timestamp?: string;
+}

package/samples/package-json-config/src/custom-generated/input-utils.ts

@@ -0,0 +1,60 @@
+/**
+ * This file was automatically generated by apify-schema-tools.
+ * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
+ * and run apify-schema-tools' "sync" command to regenerate this file.
+ */
+
+import { Actor } from "apify";
+import type { Input } from "./input.js";
+
+export const DEFAULT_INPUT_VALUES = {
+	startUrls: [],
+	categories: [],
+	maxPages: 10,
+	proxy: {
+		useApifyProxy: true,
+	},
+	debugMode: false,
+};
+
+export const REQUIRED_INPUT_FIELDS_WITHOUT_DEFAULT = ["searchTerm"];
+
+export type InputWithDefaults = Input & {
+	/**
+	 * List of categories to filter results
+	 */
+	categories: {
+		name?: string;
+		id?: string;
+	}[];
+	/**
+	 * Maximum number of pages to scrape
+	 */
+	maxPages: number;
+	/**
+	 * Proxy settings
+	 */
+	proxy: {
+		useApifyProxy?: boolean;
+	};
+	/**
+	 * Enable debug logging
+	 */
+	debugMode: boolean;
+};
+
+export function getInputWithDefaultValues(input?: Input | null): InputWithDefaults {
+	if (Actor.isAtHome()) {
+		// The platform is supposed to fill in the default values
+		return input as InputWithDefaults;
+	}
+	if (!input) {
+		throw new Error(
+			`Input is required, because the following fields are required: ${REQUIRED_INPUT_FIELDS_WITHOUT_DEFAULT.join(", ")}`,
+		);
+	}
+	return {
+		...DEFAULT_INPUT_VALUES,
+		...(input ?? ({} as Input)),
+	};
+}

package/samples/package-json-config/src/custom-generated/input.ts

@@ -0,0 +1,42 @@
+/**
+ * This file was automatically generated by apify-schema-tools.
+ * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
+ * and run apify-schema-tools' "sync" command to regenerate this file.
+ */
+
+/**
+ * startUrls and searchTerm are required.
+ */
+export interface Input {
+	/**
+	 * List of URLs to scrape
+	 */
+	startUrls: {
+		url?: string;
+	}[];
+	/**
+	 * Term to search for
+	 */
+	searchTerm: string;
+	/**
+	 * List of categories to filter results
+	 */
+	categories?: {
+		name?: string;
+		id?: string;
+	}[];
+	/**
+	 * Maximum number of pages to scrape
+	 */
+	maxPages?: number;
+	/**
+	 * Proxy settings
+	 */
+	proxy?: {
+		useApifyProxy?: boolean;
+	};
+	/**
+	 * Enable debug logging
+	 */
+	debugMode?: boolean;
+}

package/src/apify-schema-tools.ts

@@ -0,0 +1,302 @@
+#!/usr/bin/env node
+
+import { existsSync, read } from "node:fs";
+import { join } from "node:path";
+import { ArgumentDefaultsHelpFormatter, type ArgumentOptions, ArgumentParser } from "argparse";
+import {
+	ACTOR_CONFIG_PATH,
+	DATASET_SCHEMA_FIELD,
+	filterValidInputSchemaProperties,
+	generateInputDefaultsTSFileContent,
+} from "./apify.js";
+import {
+	type CommonCLIArgs,
+	addCommonCLIArgs,
+	parseConfigurationFromFileOrDefault,
+	writeConfigurationToPackageJson,
+} from "./configuration.js";
+import { readFile, writeFile } from "./filesystem.js";
+import {
+	compareSchemas,
+	mergeObjectSchemas,
+	readJsonSchema,
+	readJsonSchemaField,
+	writeJsonSchema,
+	writeSchemaToField,
+} from "./json-schemas.js";
+import {
+	compareTypescriptInterfaces,
+	jsonSchemaToTypeScriptInterface,
+	parseTypeScriptInterface,
+	removeTypeScriptHeader,
+	serializeTypeScriptInterface,
+	writeTypeScriptFile,
+} from "./typescript.js";
+
+const configuration = parseConfigurationFromFileOrDefault();
+
+const rootParser = new ArgumentParser({
+	description: "Apify Schema Tools - Generate JSON schemas and TypeScript files for Actor input and output dataset.",
+	formatter_class: ArgumentDefaultsHelpFormatter,
+});
+
+// Create subparsers for commands, and define arguments shared by multiple commands
+
+const subparsers = rootParser.add_subparsers();
+
+const DEEP_MERGE_PARAM = "--deep-merge" as const;
+const DEEP_MERGE_OPTIONS: ArgumentOptions = {
+	help: "whether to deep merge additional schemas into the main schema",
+	action: "store_true",
+	default: false,
+};
+
+// init command
+
+interface InitArgs extends CommonCLIArgs {
+	no_config_file: boolean;
+	only_config_file: boolean;
+}
+
+const initParser = subparsers.add_parser("init", {
+	help: "Initialize the Apify Schema Tools project with default settings.",
+	formatter_class: ArgumentDefaultsHelpFormatter,
+});
+addCommonCLIArgs(initParser, configuration);
+initParser.add_argument("--no-config-file", {
+	help: "do not create a configuration file in package.json",
+	action: "store_true",
+	default: false,
+});
+initParser.add_argument("--only-config-file", {
+	help: "create only the configuration file in package.json, without initializing schemas",
+	action: "store_true",
+	default: false,
+});
+
+function init(args: InitArgs) {
+	if (args.only_config_file && args.no_config_file) {
+		throw new Error("The options --only-config-file and --no-config-file were defined together: doing nothing.");
+	}
+
+	console.log("Initializing Apify Schema Tools in the current project...");
+
+	if (!args.no_config_file) {
+		writeConfigurationToPackageJson(args);
+		console.log("Configuration written to package.json");
+	}
+
+	if (args.only_config_file) {
+		console.log("Only configuration file created, skipping schema initialization.");
+		return;
+	}
+
+	if (args.input.includes("input")) {
+		if (!existsSync(args.input_schema)) {
+			throw new Error("The current Actor does not have an input schema.");
+		}
+		const inputSchema = readJsonSchema(args.input_schema);
+		writeJsonSchema(args.src_input, inputSchema);
+		console.log(`Input schema initialized at ${args.src_input}`);
+		if (args.add_input) {
+			writeJsonSchema(args.add_input, { type: "object", properties: {} });
+			console.log(`Additional input schema initialized at ${args.add_input}`);
+		}
+	}
+
+	if (args.input.includes("dataset")) {
+		if (!existsSync(ACTOR_CONFIG_PATH)) {
+			throw new Error(`The current Actor does not have an ${ACTOR_CONFIG_PATH} configuration file.`);
+		}
+		const actorConfig = JSON.parse(readFile(ACTOR_CONFIG_PATH));
+		if (actorConfig.storages?.dataset !== args.dataset_schema) {
+			writeFile(
+				ACTOR_CONFIG_PATH,
+				JSON.stringify(
+					{ ...actorConfig, storages: { ...actorConfig.storages, dataset: args.dataset_schema } },
+					null,
+					4,
+				),
+			);
+			console.log(`Updated ${ACTOR_CONFIG_PATH} to use the dataset schema at ${args.dataset_schema}`);
+		}
+		if (!existsSync(args.dataset_schema)) {
+			writeFile(
+				args.dataset_schema,
+				JSON.stringify(
+					{
+						actorSpecification: 1,
+						[DATASET_SCHEMA_FIELD]: { type: "object", properties: {} },
+					},
+					null,
+					4,
+				),
+			);
+			console.log(`Dataset schema initialized at ${args.dataset_schema}`);
+		}
+		const datasetItemSchema = readJsonSchemaField(args.dataset_schema, DATASET_SCHEMA_FIELD);
+		writeJsonSchema(args.src_dataset, datasetItemSchema);
+	}
+}
+initParser.set_defaults({ func: init });
+
+// sync command
+
+interface SyncArgs extends CommonCLIArgs {
+	deep_merge: boolean;
+	include_input_utils: string;
+}
+
+const syncParser = subparsers.add_parser("sync", {
+	help: "Generate JSON schemas and TypeScript files from the source schemas.",
+	formatter_class: ArgumentDefaultsHelpFormatter,
+});
+addCommonCLIArgs(syncParser, configuration);
+syncParser.add_argument(DEEP_MERGE_PARAM, DEEP_MERGE_OPTIONS);
+syncParser.add_argument("--include-input-utils", {
+	help:
+		"include input utilities in the generated TypeScript files:" + " 'input' input and 'ts-types' output are required",
+	choices: ["true", "false"],
+	default: "true",
+});
+
+function sync(args: SyncArgs) {
+	console.log("Syncing schemas...");
+
+	if (args.input.includes("input")) {
+		let inputSchema = readJsonSchema(args.src_input);
+		if (args.add_input) {
+			inputSchema = mergeObjectSchemas(inputSchema, readJsonSchema(args.add_input), args.deep_merge);
+		}
+		if (args.output.includes("json-schemas")) {
+			writeJsonSchema(args.input_schema, filterValidInputSchemaProperties(inputSchema));
+		}
+		if (args.output.includes("ts-types")) {
+			writeTypeScriptFile(
+				join(args.output_ts_dir, "input.ts"),
+				serializeTypeScriptInterface("Input", jsonSchemaToTypeScriptInterface(inputSchema)),
+			);
+			if (args.include_input_utils === "true") {
+				writeTypeScriptFile(
+					join(args.output_ts_dir, "input-utils.ts"),
+					generateInputDefaultsTSFileContent(inputSchema),
+				);
+			}
+		}
+	}
+
+	if (args.input.includes("dataset")) {
+		let datasetSchema = readJsonSchema(args.src_dataset);
+		if (args.add_dataset) {
+			datasetSchema = mergeObjectSchemas(datasetSchema, readJsonSchema(args.add_dataset), args.deep_merge);
+		}
+		if (args.output.includes("json-schemas")) {
+			writeSchemaToField(args.dataset_schema, datasetSchema, DATASET_SCHEMA_FIELD);
+		}
+		if (args.output.includes("ts-types")) {
+			writeTypeScriptFile(
+				join(args.output_ts_dir, "dataset.ts"),
+				serializeTypeScriptInterface("DatasetItem", jsonSchemaToTypeScriptInterface(datasetSchema)),
+			);
+		}
+	}
+}
+syncParser.set_defaults({ func: sync });
+
+// check command
+
+interface CheckArgs extends CommonCLIArgs {
+	deep_merge: boolean;
+	ignore_descriptions: boolean;
+}
+
+const checkParser = subparsers.add_parser("check", {
+	help: "Check the schemas for consistency and correctness. (WARNING: input utils will not be checked!)",
+	formatter_class: ArgumentDefaultsHelpFormatter,
+});
+addCommonCLIArgs(checkParser, configuration);
+checkParser.add_argument(DEEP_MERGE_PARAM, DEEP_MERGE_OPTIONS);
+checkParser.add_argument("--ignore-descriptions", {
+	help: "ignore the 'title' and 'description' fields during the comparison",
+	action: "store_true",
+	default: false,
+});
+
+function check(args: CheckArgs) {
+	if (args.input.includes("input")) {
+		let sourceInputSchema = readJsonSchema(args.src_input);
+		if (args.add_input) {
+			sourceInputSchema = mergeObjectSchemas(sourceInputSchema, readJsonSchema(args.add_input), args.deep_merge);
+		}
+		const generatedInputSchema = filterValidInputSchemaProperties(sourceInputSchema);
+		if (args.output.includes("json-schemas")) {
+			const outputInputSchema = readJsonSchema(args.input_schema);
+			const isInputSchemaCoherent = compareSchemas(generatedInputSchema, outputInputSchema, args.ignore_descriptions);
+			if (!isInputSchemaCoherent) {
+				console.error("Input schema validation failed: the input schema does not match the source schema.");
+				process.exit(1);
+			}
+		}
+		if (args.output.includes("ts-types")) {
+			const inputTSContent = readFile(join(args.output_ts_dir, "input.ts"));
+			const inputTSInterface = parseTypeScriptInterface(removeTypeScriptHeader(inputTSContent));
+			const sourceInputTSInterface = jsonSchemaToTypeScriptInterface(sourceInputSchema);
+			const isTSInputCoherent = compareTypescriptInterfaces(
+				sourceInputTSInterface,
+				inputTSInterface,
+				args.ignore_descriptions,
+			);
+			if (!isTSInputCoherent) {
+				console.error(
+					"Input TypeScript interface validation failed: the TypeScript interface does not match the source schema.",
+				);
+				process.exit(1);
+			}
+		}
+	}
+	if (args.input.includes("dataset")) {
+		let sourceDatasetSchema = readJsonSchema(args.src_dataset);
+		if (args.add_dataset) {
+			sourceDatasetSchema = mergeObjectSchemas(sourceDatasetSchema, readJsonSchema(args.add_dataset), args.deep_merge);
+		}
+		if (args.output.includes("json-schemas")) {
+			const outputDatasetSchema = readJsonSchemaField(args.dataset_schema, DATASET_SCHEMA_FIELD);
+			if (!outputDatasetSchema) {
+				console.error(`Dataset schema field "${DATASET_SCHEMA_FIELD}" not found in ${args.dataset_schema}.`);
+				process.exit(1);
+			}
+			const isDatasetSchemaCoherent = compareSchemas(
+				sourceDatasetSchema,
+				outputDatasetSchema,
+				args.ignore_descriptions,
+			);
+			if (!isDatasetSchemaCoherent) {
+				console.error("Dataset schema validation failed: the dataset schema does not match the source schema.");
+				process.exit(1);
+			}
+		}
+		if (args.output.includes("ts-types")) {
+			const datasetTSContent = readFile(join(args.output_ts_dir, "dataset.ts"));
+			const datasetTSInterface = parseTypeScriptInterface(removeTypeScriptHeader(datasetTSContent));
+			const sourceDatasetTSInterface = jsonSchemaToTypeScriptInterface(sourceDatasetSchema);
+			const isTSDatasetCoherent = compareTypescriptInterfaces(
+				sourceDatasetTSInterface,
+				datasetTSInterface,
+				args.ignore_descriptions,
+			);
+			if (!isTSDatasetCoherent) {
+				console.error(
+					"Dataset TypeScript interface validation failed: the TypeScript interface does not match the source schema.",
+				);
+				process.exit(1);
+			}
+		}
+	}
+	console.log("Check passed.");
+}
+checkParser.set_defaults({ func: check });
+
+// Parse and execute
+
+const parsedArgs = rootParser.parse_args();
+parsedArgs.func(parsedArgs);

package/src/apify.ts

@@ -0,0 +1,124 @@
+import { type ObjectSchema, filterValidSchemaProperties } from "./json-schemas.js";
+import { type TypeScriptInterface, jsonSchemaToTypeScriptInterface, serializeTypeScriptSchema } from "./typescript.js";
+
+export const ACTOR_CONFIG_PATH = ".actor/actor.json";
+
+const VALID_INPUT_ROOT_KEYS = ["title", "description", "type", "schemaVersion", "properties", "required"];
+
+const VALID_INPUT_PROPERTY_KEYS = [
+	"type",
+	"title",
+	"description",
+	"default",
+	"prefill",
+	"example",
+	"sectionCaption",
+	"sectionDescription",
+];
+
+const VALID_INPUT_PROPERTY_TYPES = ["string", "boolean", "integer", "object", "array"];
+
+const VALID_INPUT_PROPERTY_KEYS_BY_TYPE: Record<string, string[]> = {
+	string: [
+		"editor",
+		"pattern",
+		"minLength",
+		"maxLength",
+		"enum",
+		"enumTitles",
+		"nullable",
+		"isSecret",
+		"dateType",
+		"resourceType",
+	],
+	boolean: ["editor", "groupCaption", "groupDescription", "nullable"],
+	integer: ["editor", "maximum", "minimum", "unit", "nullable"],
+	object: ["editor", "patternKey", "patternValue", "maxProperties", "minProperties", "nullable"],
+	array: [
+		"editor",
+		"placeholderKey",
+		"placeholderValue",
+		"patternKey",
+		"patternValue",
+		"maxItems",
+		"minItems",
+		"uniqueItems",
+		"nullable",
+		"resourceType",
+	],
+};
+
+export function filterValidInputSchemaProperties(schema: ObjectSchema): ObjectSchema {
+	return filterValidSchemaProperties(
+		schema,
+		VALID_INPUT_ROOT_KEYS,
+		VALID_INPUT_PROPERTY_KEYS,
+		VALID_INPUT_PROPERTY_TYPES,
+		VALID_INPUT_PROPERTY_KEYS_BY_TYPE,
+	);
+}
+
+/**
+ * Generates the content of a TypeScript file that exports a function to fill
+ * the default values for the input schema, and the interface for the input
+ * with the default values filled in.
+ */
+export function generateInputDefaultsTSFileContent(inputSchema: ObjectSchema) {
+	const defaultValues: Record<string, unknown> = {};
+
+	for (const [property, definition] of Object.entries(inputSchema.properties ?? {})) {
+		if ("default" in definition) {
+			defaultValues[property] = definition.default;
+		}
+	}
+	const tsInterfaceSchema = jsonSchemaToTypeScriptInterface(inputSchema);
+
+	const optionalParamsWithDefaults: TypeScriptInterface = {
+		isArray: false,
+		isRequired: true,
+		properties: {},
+	};
+	const requiredParamsWithoutDefaults: string[] = [];
+	for (const [key, schema] of Object.entries(tsInterfaceSchema.properties)) {
+		if (!schema.isRequired && key in defaultValues) {
+			optionalParamsWithDefaults.properties[key] = {
+				...schema,
+				isRequired: true,
+			};
+		} else if (schema.isRequired && !(key in defaultValues)) {
+			requiredParamsWithoutDefaults.push(key);
+		}
+	}
+
+	return `\
+import { Actor } from "apify";
+import type { Input } from "./input.js";
+
+export const DEFAULT_INPUT_VALUES = ${JSON.stringify(defaultValues, null, "\t")};
+
+export const REQUIRED_INPUT_FIELDS_WITHOUT_DEFAULT = ${JSON.stringify(requiredParamsWithoutDefaults, null, "\t")};
+
+export type InputWithDefaults = Input & ${serializeTypeScriptSchema(optionalParamsWithDefaults)}
+
+export function getInputWithDefaultValues(input?: Input | null): InputWithDefaults {
+	if (Actor.isAtHome()) {
+		// The platform is supposed to fill in the default values
+		return input as InputWithDefaults;
+	}
+${
+	requiredParamsWithoutDefaults.length > 0
+		? `\
+	if (!input) {
+		throw new Error(\`Input is required, because the following fields are required: $\{REQUIRED_INPUT_FIELDS_WITHOUT_DEFAULT.join(", ")\}\`);
+	}`
+		: ""
+}
+	return {
+		...DEFAULT_INPUT_VALUES,
+		...(input ?? {} as Input),
+	};
+}
+`;
+}
+
+export const DATASET_SCHEMA_FIELD = "fields";

package/src/configuration.ts

@@ -0,0 +1,110 @@
+import { existsSync } from "node:fs";
+import type { ArgumentParser } from "argparse";
+import * as zod from "zod/v4";
+import { readFile, writeFile } from "./filesystem.js";
+
+const INPUTS = ["input", "dataset"] as const;
+const OUTPUTS = ["json-schemas", "ts-types"] as const;
+
+type Input = (typeof INPUTS)[number];
+type Output = (typeof OUTPUTS)[number];
+
+export interface CommonCLIArgs {
+	input: Input[];
+	output: Output[];
+	src_input: string;
+	src_dataset: string;
+	add_input?: string;
+	add_dataset?: string;
+	input_schema: string;
+	dataset_schema: string;
+	output_ts_dir: string;
+}
+
+export const Configuration = zod.object({
+	input: zod.array(zod.enum(INPUTS)).default([...INPUTS]),
+	output: zod.array(zod.enum(OUTPUTS)).default([...OUTPUTS]),
+	srcInput: zod.string().default("src-schemas/input.json"),
+	srcDataset: zod.string().default("src-schemas/dataset-item.json"),
+	addInput: zod.string().optional(),
+	addDataset: zod.string().optional(),
+	inputSchema: zod.string().default(".actor/input_schema.json"),
+	datasetSchema: zod.string().default(".actor/dataset_schema.json"),
+	outputTSDir: zod.string().default("src/generated"),
+});
+
+export function parseConfigurationFromFileOrDefault(): zod.infer<typeof Configuration> {
+	const packageJsonContent = existsSync("package.json") ? readFile("package.json") : undefined;
+	const rawConfig = packageJsonContent ? (JSON.parse(packageJsonContent)["apify-schema-tools"] ?? {}) : {};
+	return Configuration.parse(rawConfig);
+}
+
+export function writeConfigurationToPackageJson(args: CommonCLIArgs): void {
+	if (!existsSync("package.json")) {
+		throw new Error("package.json does not exist. Please run this command in the root of your project.");
+	}
+	const packageJsonContent = readFile("package.json");
+	const packageJson = JSON.parse(packageJsonContent);
+	packageJson["apify-schema-tools"] = {
+		input: args.input,
+		output: args.output,
+		srcInput: args.src_input,
+		srcDataset: args.src_dataset,
+		addInput: args.add_input,
+		addDataset: args.add_dataset,
+		inputSchema: args.input_schema,
+		datasetSchema: args.dataset_schema,
+		outputTSDir: args.output_ts_dir,
+	};
+	writeFile("package.json", JSON.stringify(packageJson, null, 2));
+}
+
+/**
+ * This function will set as default values the configuration from package.json, or the default one.
+ * In this way, the CLI arguments can override the configuration.
+ */
+export function addCommonCLIArgs(parser: ArgumentParser, configuration: zod.infer<typeof Configuration>): void {
+	parser.add_argument("-i", "--input", {
+		help: "specify which sources to use for generation",
+		choices: [...INPUTS],
+		default: configuration.input,
+		nargs: "*",
+	});
+	parser.add_argument("-o", "--output", {
+		help: "specify what to generate",
+		choices: [...OUTPUTS],
+		default: configuration.output,
+		nargs: "*",
+	});
+
+	parser.add_argument("--src-input", {
+		help: "path to the input schema source file",
+		default: configuration.srcInput,
+	});
+	parser.add_argument("--src-dataset", {
+		help: "path to the dataset schema source file",
+		default: configuration.srcDataset,
+	});
+
+	parser.add_argument("--add-input", {
+		help: "path to an additional schema to merge into the input schema",
+		default: configuration.addInput,
+	});
+	parser.add_argument("--add-dataset", {
+		help: "path to an additional schema to merge into the dataset schema",
+		default: configuration.addDataset,
+	});
+
+	parser.add_argument("--input-schema", {
+		help: "the path of the destination input schema file",
+		default: configuration.inputSchema,
+	});
+	parser.add_argument("--dataset-schema", {
+		help: "the path of the destination dataset schema file",
+		default: configuration.datasetSchema,
+	});
+	parser.add_argument("--output-ts-dir", {
+		help: "path where to save generated TypeScript files",
+		default: configuration.outputTSDir,
+	});
+}

package/src/filesystem.ts

@@ -0,0 +1,18 @@
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+
+const DEFAULT_ENCODING: BufferEncoding = "utf-8";
+
+export function readFile(filePath: string): string {
+	return readFileSync(filePath, DEFAULT_ENCODING);
+}
+
+/**
+ * Writes content to a file, creating the directory structure if it doesn't exist.
+ * @param filePath - The path to the file to write.
+ * @param content - The content to write to the file.
+ */
+export function writeFile(filePath: string, content: string): void {
+	mkdirSync(dirname(filePath), { recursive: true });
+	writeFileSync(filePath, content, DEFAULT_ENCODING);
+}