apify-schema-tools 3.0.0 → 3.1.0

package/samples/package-json-config-merged/custom-add-schemas/add-input.json

@@ -0,0 +1,21 @@
+{
+	"type": "object",
+	"properties": {
+		"proxy": {
+			"title": "Proxy configuration",
+			"description": "Proxy settings",
+			"type": "object",
+			"properties": {
+				"proxyCountry": {
+					"type": "string",
+					"default": "US"
+				}
+			}
+		},
+		"extraField": {
+			"type": "string",
+			"title": "An extra field from add-input.json",
+			"description": "This field was added from the additional schema and merged into the main schema."
+		}
+	}
+}

package/samples/package-json-config-merged/custom-src-schemas/dataset-item.json

@@ -0,0 +1,28 @@
+{
+	"$schema": "http://json-schema.org/draft-07/schema#",
+	"title": "Dataset schema for Web Scraper",
+	"type": "object",
+	"properties": {
+		"title": {
+			"type": "string",
+			"title": "Title",
+			"description": "Page title"
+		},
+		"url": {
+			"type": "string",
+			"title": "URL",
+			"description": "Page URL"
+		},
+		"text": {
+			"type": "string",
+			"title": "Text content",
+			"description": "Extracted text"
+		},
+		"timestamp": {
+			"type": "string",
+			"title": "Timestamp",
+			"description": "When the data was scraped"
+		}
+	},
+	"required": ["title", "url"]
+}

package/samples/package-json-config-merged/custom-src-schemas/input.json

@@ -0,0 +1,89 @@
+{
+	"title": "Input schema for Web Scraper",
+	"description": "startUrls and searchTerm are required.",
+	"type": "object",
+	"schemaVersion": 1,
+	"properties": {
+		"startUrls": {
+			"position": 10,
+			"type": "array",
+			"title": "Start URLs",
+			"description": "List of URLs to scrape",
+			"default": [],
+			"editor": "requestListSources",
+			"items": {
+				"type": "object",
+				"properties": {
+					"url": {
+						"type": "string"
+					}
+				}
+			}
+		},
+		"searchTerm": {
+			"position": 20,
+			"type": "string",
+			"title": "Search term",
+			"description": "Term to search for",
+			"minLength": 1,
+			"maxLength": 100
+		},
+		"categories": {
+			"position": 30,
+			"type": "array",
+			"title": "Categories",
+			"description": "List of categories to filter results",
+			"default": [],
+			"items": {
+				"type": "object",
+				"properties": {
+					"name": {
+						"type": "string",
+						"title": "Category name"
+					},
+					"id": {
+						"type": "string",
+						"title": "Category ID"
+					}
+				},
+				"additionalProperties": false
+			}
+		},
+		"maxPages": {
+			"position": 30,
+			"sectionCaption": "Scraping options",
+			"sectionDescription": "Configure how many pages to scrape and other options.",
+			"type": "integer",
+			"title": "Maximum pages",
+			"description": "Maximum number of pages to scrape",
+			"default": 10,
+			"minimum": 1,
+			"maximum": 1000
+		},
+		"proxy": {
+			"position": 40,
+			"type": "object",
+			"title": "Proxy configuration",
+			"description": "Proxy settings",
+			"default": {
+				"useApifyProxy": true
+			},
+			"properties": {
+				"useApifyProxy": {
+					"type": "boolean",
+					"default": true
+				}
+			},
+			"additionalProperties": true
+		},
+		"debugMode": {
+			"position": 60,
+			"type": "boolean",
+			"title": "Debug mode",
+			"description": "Enable debug logging",
+			"default": false
+		}
+	},
+	"required": ["startUrls", "searchTerm"],
+	"additionalProperties": false
+}

package/samples/package-json-config-merged/package.json

@@ -0,0 +1,19 @@
+{
+	"name": "test-package",
+	"version": "1.0.0",
+	"apify-schema-tools": {
+		"input": [
+			"input",
+			"dataset"
+		],
+		"output": [
+			"json-schemas",
+			"ts-types"
+		],
+		"srcInput": "custom-src-schemas/input.json",
+		"addInput": "custom-add-schemas/add-input.json",
+		"deepMerge": true,
+		"srcDataset": "custom-src-schemas/dataset-item.json",
+		"outputTSDir": "src/custom-generated"
+	}
+}

package/samples/package-json-config-merged/src/custom-generated/dataset.ts

@@ -0,0 +1,25 @@
+/**
+ * This file was automatically generated by apify-schema-tools.
+ * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
+ * and run apify-schema-tools' "sync" command to regenerate this file.
+ */
+
+export interface DatasetItem {
+	/**
+	 * Page title
+	 */
+	title: string;
+	/**
+	 * Page URL
+	 */
+	url: string;
+	/**
+	 * Extracted text
+	 */
+	text?: string;
+	/**
+	 * When the data was scraped
+	 */
+	timestamp?: string;
+	[key: string]: unknown;
+}

package/samples/package-json-config-merged/src/custom-generated/input-utils.ts

@@ -0,0 +1,73 @@
+/**
+ * This file was automatically generated by apify-schema-tools.
+ * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
+ * and run apify-schema-tools' "sync" command to regenerate this file.
+ */
+
+import { Actor } from "apify";
+import type { Input } from "./input.js";
+
+interface DefaultInputs {
+	/**
+	 * List of URLs to scrape
+	 */
+	startUrls: {
+		url?: string;
+		[key: string]: unknown;
+	}[];
+	/**
+	 * List of categories to filter results
+	 */
+	categories: {
+		name?: string;
+		id?: string;
+	}[];
+	/**
+	 * Maximum number of pages to scrape
+	 */
+	maxPages: number;
+	/**
+	 * Proxy settings
+	 */
+	proxy: {
+		useApifyProxy?: boolean;
+		proxyCountry?: string;
+		[key: string]: unknown;
+	};
+	/**
+	 * Enable debug logging
+	 */
+	debugMode: boolean;
+}
+
+export type InputWithDefaults = Input & DefaultInputs;
+
+export const DEFAULT_INPUT_VALUES: DefaultInputs = {
+	startUrls: [],
+	categories: [],
+	maxPages: 10,
+	proxy: {
+		useApifyProxy: true,
+	},
+	debugMode: false,
+};
+
+export const REQUIRED_INPUT_FIELDS_WITHOUT_DEFAULT = ["searchTerm"];
+
+export function getInputWithDefaultValues(input?: Input | null): InputWithDefaults {
+	if (Actor.isAtHome()) {
+		// The platform is supposed to fill in the default values
+		return input as InputWithDefaults;
+	}
+	if (!input) {
+		throw new Error(
+			`Input is required, because the following fields are required: ${REQUIRED_INPUT_FIELDS_WITHOUT_DEFAULT.join(
+				", ",
+			)}`,
+		);
+	}
+	return {
+		...DEFAULT_INPUT_VALUES,
+		...(input ?? ({} as Input)),
+	};
+}

package/samples/package-json-config-merged/src/custom-generated/input.ts

@@ -0,0 +1,49 @@
+/**
+ * This file was automatically generated by apify-schema-tools.
+ * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
+ * and run apify-schema-tools' "sync" command to regenerate this file.
+ */
+
+/**
+ * startUrls and searchTerm are required.
+ */
+export interface Input {
+	/**
+	 * List of URLs to scrape
+	 */
+	startUrls: {
+		url?: string;
+		[key: string]: unknown;
+	}[];
+	/**
+	 * Term to search for
+	 */
+	searchTerm: string;
+	/**
+	 * List of categories to filter results
+	 */
+	categories?: {
+		name?: string;
+		id?: string;
+	}[];
+	/**
+	 * Maximum number of pages to scrape
+	 */
+	maxPages?: number;
+	/**
+	 * Proxy settings
+	 */
+	proxy?: {
+		useApifyProxy?: boolean;
+		proxyCountry?: string;
+		[key: string]: unknown;
+	};
+	/**
+	 * Enable debug logging
+	 */
+	debugMode?: boolean;
+	/**
+	 * This field was added from the additional schema and merged into the main schema.
+	 */
+	extraField?: string;
+}

package/src/apify-schema-tools.ts

@@ -4,7 +4,7 @@
 
 import { existsSync } from "node:fs";
 import { join } from "node:path";
-import { ArgumentDefaultsHelpFormatter, type ArgumentOptions, ArgumentParser } from "argparse";
+import { ArgumentDefaultsHelpFormatter, ArgumentParser } from "argparse";
 import {
 	ACTOR_CONFIG_PATH,
 	DATASET_SCHEMA_FIELD,
@@ -14,7 +14,7 @@ import {
 	getPathRelativeToActorConfig,
 } from "./apify.js";
 import {
-	addCommonCLIArgs,
+	addCommonCliArgs,
 	type CommonCliArgs,
 	parseConfigurationFromFileOrDefault,
 	writeConfigurationToPackageJson,
@@ -50,13 +50,6 @@ const rootParser = new ArgumentParser({
 
 const subparsers = rootParser.add_subparsers();
 
-const DEEP_MERGE_PARAM = "--deep-merge" as const;
-const DEEP_MERGE_OPTIONS: ArgumentOptions = {
-	help: "whether to deep merge additional schemas into the main schema",
-	action: "store_true",
-	default: false,
-};
-
 // init command
 
 interface InitArgs extends CommonCliArgs {
@@ -68,7 +61,7 @@ const initParser = subparsers.add_parser("init", {
 	help: "Initialize the Apify Schema Tools project with default settings.",
 	formatter_class: ArgumentDefaultsHelpFormatter,
 });
-addCommonCLIArgs(initParser, configuration);
+addCommonCliArgs(initParser, configuration);
 initParser.add_argument("--no-config-file", {
 	help: "do not create a configuration file in package.json",
 	action: "store_true",
@@ -173,7 +166,6 @@ initParser.set_defaults({ func: init });
 // sync command
 
 interface SyncArgs extends CommonCliArgs {
-	deep_merge: boolean;
 	include_input_utils: string;
 	force: boolean;
 	fail_on_conflict: boolean;
@@ -184,8 +176,7 @@ const syncParser = subparsers.add_parser("sync", {
 By default, if conflicts are detected, the user will be prompted to resolve them.`,
 	formatter_class: ArgumentDefaultsHelpFormatter,
 });
-addCommonCLIArgs(syncParser, configuration);
-syncParser.add_argument(DEEP_MERGE_PARAM, DEEP_MERGE_OPTIONS);
+addCommonCliArgs(syncParser, configuration);
 syncParser.add_argument("--include-input-utils", {
 	help: "include input utilities in the generated TypeScript files: 'input' input and 'ts-types' output are required",
 	choices: ["true", "false"],
@@ -322,7 +313,6 @@ syncParser.set_defaults({ func: sync });
 // check command
 
 interface CheckArgs extends CommonCliArgs {
-	deep_merge: boolean;
 	ignore_descriptions: boolean;
 }
 
@@ -330,8 +320,7 @@ const checkParser = subparsers.add_parser("check", {
 	help: "Check the schemas for consistency and correctness. (WARNING: input utils will not be checked!)",
 	formatter_class: ArgumentDefaultsHelpFormatter,
 });
-addCommonCLIArgs(checkParser, configuration);
-checkParser.add_argument(DEEP_MERGE_PARAM, DEEP_MERGE_OPTIONS);
+addCommonCliArgs(checkParser, configuration);
 checkParser.add_argument("--ignore-descriptions", {
 	help: "ignore the 'title' and 'description' fields during the comparison",
 	action: "store_true",

package/src/apify.ts

@@ -125,9 +125,9 @@ export function getInputWithDefaultValues(input?: Input | null): InputWithDefaul
 		requiredParamsWithoutDefaults.length > 0
 			? `
 	if (!input) {
-		throw new Error(\`Input is required, because the following fields are required: $\{
-			REQUIRED_INPUT_FIELDS_WITHOUT_DEFAULT.join(", ")
-		\}\`);
+		throw new Error(\`Input is required, because the following fields are required: \${REQUIRED_INPUT_FIELDS_WITHOUT_DEFAULT.join(
+			", ",
+		)}\`);
 	}`
 			: ""
 	}

package/src/configuration.ts

@@ -2,12 +2,13 @@
 
 import { existsSync } from "node:fs";
 import type { ArgumentParser } from "argparse";
-import { array, enum as enum_, object, string, type infer as zodInfer } from "zod/v4";
+import { array, boolean, enum as enum_, object, string, type infer as zodInfer } from "zod/v4";
 import { readFile, writeFile } from "./filesystem.js";
 
 const zod = {
 	object,
 	array,
+	boolean,
 	enum: enum_,
 	string,
 };
@@ -25,6 +26,7 @@ export interface CommonCliArgs {
 	src_dataset: string;
 	add_input?: string;
 	add_dataset?: string;
+	deep_merge: boolean;
 	input_schema: string;
 	dataset_schema: string;
 	output_ts_dir: string;
@@ -37,6 +39,7 @@ export const Configuration = zod.object({
 	srcDataset: zod.string().default("src-schemas/dataset-item.json"),
 	addInput: zod.string().optional(),
 	addDataset: zod.string().optional(),
+	deepMerge: zod.boolean().default(false),
 	inputSchema: zod.string().default(".actor/input_schema.json"),
 	datasetSchema: zod.string().default(".actor/dataset_schema.json"),
 	outputTSDir: zod.string().default("src/generated"),
@@ -61,6 +64,7 @@ export function writeConfigurationToPackageJson(args: CommonCliArgs): void {
 		srcDataset: args.src_dataset,
 		addInput: args.add_input,
 		addDataset: args.add_dataset,
+		deepMerge: args.deep_merge,
 		inputSchema: args.input_schema,
 		datasetSchema: args.dataset_schema,
 		outputTSDir: args.output_ts_dir,
@@ -72,7 +76,7 @@ export function writeConfigurationToPackageJson(args: CommonCliArgs): void {
  * This function will set as default values the configuration from package.json, or the default one.
  * In this way, the CLI arguments can override the configuration.
  */
-export function addCommonCLIArgs(parser: ArgumentParser, configuration: zodInfer<typeof Configuration>): void {
+export function addCommonCliArgs(parser: ArgumentParser, configuration: zodInfer<typeof Configuration>): void {
 	parser.add_argument("-i", "--input", {
 		help: "specify which sources to use for generation",
 		choices: [...INPUTS],
@@ -103,6 +107,11 @@ export function addCommonCLIArgs(parser: ArgumentParser, configuration: zodInfer
 		help: "path to an additional schema to merge into the dataset schema",
 		default: configuration.addDataset,
 	});
+	parser.add_argument("--deep-merge", {
+		help: "whether to deep merge additional schemas into the main schema",
+		action: "store_true",
+		default: configuration.deepMerge,
+	});
 
 	parser.add_argument("--input-schema", {
 		help: "the path of the destination input schema file",

package/src/json-schema-conflicts.ts

@@ -91,7 +91,7 @@ async function checkRootFieldsConflicts(
 
 	for (const field of fieldsToCheck) {
 		if (alteredAdditionalSchema && field in alteredAdditionalSchema) {
-			// biome-ignore lint/nursery/noAwaitInLoop: user interaction for each field
+			// biome-ignore lint/performance/noAwaitInLoops: user interaction for each field
 			alteredAdditionalSchema = await manageFieldConflict(
 				[],
 				field,
@@ -148,7 +148,7 @@ async function checkPropertyFieldsConflicts(
 		const targetProp = targetSchema.properties[key];
 		for (const field of fieldsToCheck) {
 			if (alteredAdditionalSchema && additionalProp && (!deepMerge || field in additionalProp)) {
-				// biome-ignore lint/nursery/noAwaitInLoop: user interaction for each field
+				// biome-ignore lint/performance/noAwaitInLoops: user interaction for each field
 				const resultProp = await manageFieldConflict(
 					["properties", key],
 					field,
@@ -200,7 +200,7 @@ async function checkRemovedPropertyConflicts(
 	for (const [key, targetValue] of Object.entries(targetSchema.properties ?? {})) {
 		if (!(key in (alteredSourceSchema.properties ?? {}) || key in (alteredAdditionalSchema?.properties ?? {}))) {
 			const errorMessage = `Property "${key}" was removed from the source schema.`;
-			// biome-ignore lint/nursery/noAwaitInLoop: user interaction for each field
+			// biome-ignore lint/performance/noAwaitInLoops: user interaction for each field
 			await manageConflict(
 				errorMessage,
 				async () => {

package/src/json-schemas.ts

@@ -142,7 +142,7 @@ function mergeArraySchemas(baseSchema: ArraySchema, additionalSchema: ArraySchem
 export function mergeObjectSchemas(
 	baseSchema: ObjectSchema,
 	additionalSchema: ObjectSchema,
-	deep = true,
+	deep: boolean,
 ): ObjectSchema {
 	const baseProperties = baseSchema.properties || {};
 	const additionalProperties = additionalSchema.properties || {};

package/test/apify.test.ts

@@ -81,9 +81,9 @@ export function getInputWithDefaultValues(input?: Input | null): InputWithDefaul
 		return input as InputWithDefaults;
 	}
 	if (!input) {
-		throw new Error(\`Input is required, because the following fields are required: $\{
-			REQUIRED_INPUT_FIELDS_WITHOUT_DEFAULT.join(", ")
-		\}\`);
+		throw new Error(\`Input is required, because the following fields are required: \${REQUIRED_INPUT_FIELDS_WITHOUT_DEFAULT.join(
+			", ",
+		)}\`);
 	}
 	return input ?? {} as Input;
 }