apify-schema-tools 3.1.0 → 3.2.0

package/src/middle-schema/compare-schemas.ts

@@ -0,0 +1,113 @@
+import { equals } from "ramda";
+import {
+	isMiddleBasicVar,
+	isMiddleEnum,
+	isMiddleObject,
+	type MiddleBasicVar,
+	type MiddleEnum,
+	type MiddleObject,
+	type MiddleSchema,
+} from "./schema-types.js";
+
+function compareObjectProperties(key: string, propA: MiddleObject, propB: MiddleSchema, ignoreDocs?: boolean): boolean {
+	if (!isMiddleObject(propB)) {
+		console.error(`Property "${key}" is an interface in one schema but not in the other.`);
+		return false;
+	}
+	if (!compareMiddleObjects(propA, propB, ignoreDocs)) {
+		console.error(`Property "${key}" interfaces do not match.`);
+		return false;
+	}
+	return true;
+}
+
+function compareEnumProperties(key: string, propA: MiddleSchema, propB: MiddleSchema): boolean {
+	if (!isMiddleEnum(propB)) {
+		console.error(`Property "${key}" is an enum in one schema but not in the other.`);
+		return false;
+	}
+	if (JSON.stringify((propA as MiddleEnum).enum) !== JSON.stringify((propB as MiddleEnum).enum)) {
+		console.error(
+			`Property "${key}" enums do not match: ${JSON.stringify((propA as MiddleEnum).enum)} vs ${JSON.stringify((propB as MiddleEnum).enum)}`,
+		);
+		return false;
+	}
+	return true;
+}
+
+function compareBasicVarProperties(key: string, propA: MiddleSchema, propB: MiddleSchema): boolean {
+	if (!isMiddleBasicVar(propB)) {
+		console.error(`Property "${key}" is a basic var in one schema but not in the other.`);
+		return false;
+	}
+	if (!equals((propA as MiddleBasicVar).type, (propB as MiddleBasicVar).type)) {
+		console.error(
+			`Property "${key}" types do not match: ${JSON.stringify((propA as MiddleBasicVar).type)} vs ${JSON.stringify((propB as MiddleBasicVar).type)}`,
+		);
+		return false;
+	}
+	return true;
+}
+
+function compareProperties(key: string, propA: MiddleSchema, propB: MiddleSchema, ignoreDocs?: boolean): boolean {
+	if (propA.isArray !== propB.isArray) {
+		console.error(`Property "${key}" has different array status: ${propA.isArray} vs ${propB.isArray}`);
+		return false;
+	}
+
+	if (propA.isRequired !== propB.isRequired) {
+		console.error(`Property "${key}" has different required status: ${propA.isRequired} vs ${propB.isRequired}`);
+		return false;
+	}
+
+	if (!ignoreDocs && propA.doc !== propB.doc) {
+		console.error(`Property "${key}" has different documentation: "${propA.doc}" vs "${propB.doc}"`);
+		return false;
+	}
+
+	if (isMiddleObject(propA)) {
+		return compareObjectProperties(key, propA, propB, ignoreDocs);
+	}
+	if (isMiddleEnum(propA)) {
+		return compareEnumProperties(key, propA, propB);
+	}
+	return compareBasicVarProperties(key, propA, propB);
+}
+
+function compareAdditionalProperties(a: MiddleObject, b: MiddleObject, ignoreDocs?: boolean): boolean {
+	if (!(a.additionalProperties || b.additionalProperties)) {
+		return true; // Both have no additional properties
+	}
+	if (!(a.additionalProperties && b.additionalProperties)) {
+		console.error(`One interface has additionalProperties defined, but the other does not.
+If you generated an interface from a JSON schema, pay attention that "additionalProperties" is set to an empty schema by default.`);
+		return false;
+	}
+	return compareProperties("[key: string]", a.additionalProperties, b.additionalProperties, ignoreDocs);
+}
+
+export function compareMiddleObjects(a: MiddleObject, b: MiddleObject, ignoreDocs?: boolean): boolean {
+	if (Object.keys(a.properties).length !== Object.keys(b.properties).length) {
+		console.error("Interfaces have different number of properties.");
+		return false;
+	}
+
+	for (const key in a.properties) {
+		if (Object.hasOwn(a.properties, key)) {
+			if (!(key in b.properties)) {
+				return false;
+			}
+			const propA = a.properties[key];
+			const propB = b.properties[key];
+			if (!compareProperties(key, propA, propB, ignoreDocs)) {
+				return false;
+			}
+		}
+	}
+
+	if (!compareAdditionalProperties(a, b, ignoreDocs)) {
+		return false;
+	}
+
+	return true;
+}

package/src/middle-schema/generate-typescript.ts

@@ -0,0 +1,88 @@
+import { readFile, writeFile } from "../utils/filesystem.js";
+import {
+	isMiddleBasicVar,
+	isMiddleEnum,
+	isMiddleObject,
+	type MiddleBasicVar,
+	type MiddleEnum,
+	type MiddleObject,
+	type MiddleSchema,
+} from "./schema-types.js";
+
+export const TYPESCRIPT_FILE_HEADER = `\
+/**
+ * This file was automatically generated by apify-schema-tools.
+ * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
+ * and run apify-schema-tools' "sync" command to regenerate this file.
+ */
+
+`;
+
+function serializeInterfaceSchema(schema: MiddleObject, indentLevel: number): string {
+	const indent = "\t".repeat(indentLevel);
+	const properties = Object.entries(schema.properties);
+	if (properties.length === 0 && !schema.additionalProperties) {
+		return "{}";
+	}
+	const innerIndentLevel = indentLevel + 1;
+	const innerIndent = "\t".repeat(innerIndentLevel);
+	let result = "{";
+	for (const [key, value] of properties) {
+		if (value.doc) {
+			result += `\n${innerIndent}/**\n${innerIndent} * ${value.doc}\n${innerIndent} */`;
+		}
+		result += `\n${innerIndent}${key}${
+			value.isRequired ? "" : "?"
+		}: ${serializeMiddleSchemaToTypeScript(value, innerIndentLevel)};`;
+	}
+	if (schema.additionalProperties) {
+		result += `\n${innerIndent}[key: string]: ${serializeMiddleSchemaToTypeScript(
+			schema.additionalProperties,
+			innerIndentLevel,
+		)};`;
+	}
+	result += `\n${indent}}`;
+	return result;
+}
+
+function serializeEnumSchema(schema: MiddleEnum): string {
+	const serializedEnum = JSON.stringify(schema.enum).replace(/\[|\]/g, "").replace(/,/g, " | ");
+	return schema.isArray && schema.enum.length > 1 ? `(${serializedEnum})` : serializedEnum;
+}
+
+function serializeBasicVarSchema(schema: MiddleBasicVar): string {
+	if (Array.isArray(schema.type)) {
+		return schema.isArray ? `(${schema.type.join(" | ")})` : schema.type.join(" | ");
+	}
+	return schema.type;
+}
+
+export function serializeMiddleSchemaToTypeScript(schema: MiddleSchema, indentLevel = 0): string {
+	let result = "";
+	if (isMiddleObject(schema)) {
+		result = serializeInterfaceSchema(schema, indentLevel);
+	} else if (isMiddleEnum(schema)) {
+		result = serializeEnumSchema(schema);
+	} else if (isMiddleBasicVar(schema)) {
+		result = serializeBasicVarSchema(schema);
+	}
+	if (schema.isArray) {
+		result += "[]";
+	}
+	return result;
+}
+
+export function serializeMiddleObjectToTypeScript(name: string, schema: MiddleObject): string {
+	const serializedSchema = serializeMiddleSchemaToTypeScript(schema);
+	const docComment = schema.doc ? `/**\n * ${schema.doc}\n */\n` : "";
+	return `${docComment}export interface ${name} ${serializedSchema}${schema.isRequired ? "" : " | undefined"}`;
+}
+
+export function writeTypeScriptFile(filePath: string, content: string): void {
+	writeFile(filePath, TYPESCRIPT_FILE_HEADER + content);
+}
+
+export function readGeneratedTypeScriptFile(filePath: string): string {
+	const content = readFile(filePath);
+	return content.replace(TYPESCRIPT_FILE_HEADER, "");
+}

package/src/middle-schema/parse-json-schema.ts

@@ -0,0 +1,104 @@
+import type { JSONSchema4 } from "json-schema";
+import { isArraySchema, isObjectSchema, type ObjectSchema } from "../utils/json-schemas.js";
+import {
+	type AllowedType,
+	isAllowedType,
+	type MiddleBasicVar,
+	type MiddleObject,
+	type MiddleSchema,
+} from "./schema-types.js";
+
+const JSON_SCHEMA_TYPE_TO_TS_TYPE: Record<string, string> = {
+	integer: "number",
+};
+
+function generateMiddleObject(
+	schema: ObjectSchema,
+	doc: string | undefined,
+	isArray: boolean,
+	isRequired: boolean,
+): MiddleObject {
+	const requiredProperties = schema.required ?? [];
+
+	const properties = Object.fromEntries(
+		Object.entries(schema.properties ?? {}).map(([key, value]) => {
+			return [key, jsonSchemaToMiddleSchema(value, false, requiredProperties.includes(key))];
+		}),
+	);
+
+	let additionalProperties: MiddleSchema | undefined;
+	if (schema.additionalProperties === true || schema.additionalProperties === undefined) {
+		// If additionalProperties is true or undefined, we parse it as an "empty" schema
+		additionalProperties = { doc: undefined, isArray: false, isRequired: true, type: "unknown" };
+	} else if (schema.additionalProperties !== false) {
+		additionalProperties = jsonSchemaToMiddleSchema(schema.additionalProperties, false, true);
+	}
+
+	return { doc, isArray, isRequired, properties, additionalProperties };
+}
+
+function generateArrayMiddleSchema(
+	schema: JSONSchema4 & { type: "array" },
+	doc: string | undefined,
+	isRequired: boolean,
+): MiddleSchema {
+	if (Array.isArray(schema.items)) {
+		throw new Error("Array schema with multiple items is not supported");
+	}
+	if (!schema.items) {
+		return { doc, isArray: true, isRequired, type: "unknown" };
+	}
+	return jsonSchemaToMiddleSchema(schema.items, true, isRequired, doc);
+}
+
+function generateMiddleBasicVar(
+	schema: JSONSchema4,
+	doc: string | undefined,
+	isArray: boolean,
+	isRequired: boolean,
+): MiddleBasicVar {
+	const types: AllowedType[] = [];
+	for (const type of Array.isArray(schema.type) ? schema.type : [schema.type ?? "unknown"]) {
+		const tsType = JSON_SCHEMA_TYPE_TO_TS_TYPE[type] ?? type;
+		if (!isAllowedType(tsType)) {
+			throw new Error(`Unsupported JSON schema type: ${type}`);
+		}
+		types.push(tsType);
+	}
+	if (schema.nullable) {
+		types.push("null");
+	}
+	return {
+		doc,
+		isArray,
+		isRequired,
+		type: types.length === 1 ? types[0] : types,
+	};
+}
+
+function jsonSchemaToMiddleSchema(
+	schema: JSONSchema4,
+	isArray: boolean,
+	isRequired: boolean,
+	inheritedDoc?: string,
+): MiddleSchema {
+	const doc = inheritedDoc ?? schema.description;
+
+	if (isObjectSchema(schema)) {
+		return generateMiddleObject(schema, doc, isArray, isRequired);
+	}
+
+	if (isArraySchema(schema)) {
+		return generateArrayMiddleSchema(schema, doc, isRequired);
+	}
+
+	if (schema.enum) {
+		return { doc, isArray, isRequired, enum: schema.enum };
+	}
+
+	return generateMiddleBasicVar(schema, doc, isArray, isRequired);
+}
+
+export function jsonSchemaToMiddleObject(schema: ObjectSchema): MiddleObject {
+	return jsonSchemaToMiddleSchema(schema, false, true) as MiddleObject;
+}

package/src/middle-schema/parse-typescript.ts

@@ -0,0 +1,239 @@
+import { isAllowedType, type MiddleObject, type MiddleSchema } from "./schema-types.js";
+
+const TYPESCRIPT_HEADER_REGEX = /\/\*\*[\s\S]*?\*\//;
+
+export function removeTypeScriptHeader(fileContent: string): string {
+	// Match the first jsdoc and remove it
+	const headerMatch = fileContent.match(TYPESCRIPT_HEADER_REGEX);
+	if (!headerMatch || headerMatch.index === undefined) {
+		throw new Error("No header found in the TypeScript file");
+	}
+	const headerEndIndex = headerMatch.index + headerMatch[0].length;
+	return fileContent.slice(headerEndIndex).trim();
+}
+
+const WHITESPACE_REGEX = /\s/;
+
+function skipWhitespace(body: string, i: number): number {
+	let cursor = i;
+	while (cursor < body.length && WHITESPACE_REGEX.test(body[cursor])) {
+		cursor++;
+	}
+	return cursor;
+}
+
+const TYPESCRIPT_DOC_DECORATIONS_REGEX = /^\s*\*\s?/;
+const TYPESCRIPT_DOC_PREFIX = "/**";
+
+function tryParseJsDoc(body: string, i: number): { doc: string | undefined; nextIndex: number } | null {
+	if (body.slice(i, i + TYPESCRIPT_DOC_PREFIX.length) === TYPESCRIPT_DOC_PREFIX) {
+		const docEnd = body.indexOf("*/", i);
+		if (docEnd !== -1) {
+			const docContent = body.slice(i + TYPESCRIPT_DOC_PREFIX.length, docEnd);
+			const docLines = docContent.split("\n");
+			const cleanDoc = docLines
+				.map((line) => line.replace(TYPESCRIPT_DOC_DECORATIONS_REGEX, "").trim())
+				.filter((line) => line.length > 0)
+				.join(" ");
+			return { doc: cleanDoc || undefined, nextIndex: docEnd + 2 };
+		}
+	}
+	return null;
+}
+
+function extractTypeString(
+	body: string,
+	i: number,
+): { typeStr: string; isArray: boolean; isObject: boolean; typeEnd: number } {
+	let braceDepth = 0;
+	let typeEnd = i;
+
+	while (typeEnd < body.length) {
+		const char = body[typeEnd];
+		if (char === "{") {
+			braceDepth++;
+		} else if (char === "}") {
+			braceDepth--;
+			if (braceDepth < 0) {
+				break;
+			}
+		} else if (char === ";" && braceDepth === 0) {
+			break;
+		}
+		typeEnd++;
+	}
+
+	let typeStr = body.slice(i, typeEnd).trim();
+	const isArray = typeStr.endsWith("[]");
+	if (isArray) {
+		typeStr = typeStr.slice(0, -2).trim();
+	}
+	const isObject = typeStr.startsWith("{") && typeStr.endsWith("}");
+	if (isObject) {
+		typeStr = typeStr.slice(1, -1).trim();
+	}
+	return { typeStr, isArray, isObject, typeEnd };
+}
+
+function parseTypeScriptValue(text: string) {
+	try {
+		return JSON.parse(text.trim());
+	} catch {
+		throw new Error(`Invalid TypeScript value: ${text}`);
+	}
+}
+
+function extractSchemaFromText(
+	text: string,
+	currentDoc: string | undefined,
+	isRequired: boolean,
+	isArray: boolean,
+	isObject: boolean,
+): MiddleSchema {
+	let schema: MiddleSchema;
+
+	if (isObject) {
+		const { properties: nestedProperties, additionalProperties: additionalNestedProperties } =
+			parseInterfaceProperties(text);
+		schema = {
+			doc: currentDoc,
+			isArray,
+			isRequired,
+			properties: nestedProperties,
+			additionalProperties: additionalNestedProperties,
+		};
+	} else if (text.includes("|")) {
+		const unionBody = isArray ? text.slice(1, -1) : text;
+		const unionParts = unionBody.split("|").map((part) => part.trim());
+
+		// Since we currently do not support complex unions, we check if all parts are allowed types
+		if (unionParts.every((part) => isAllowedType(part))) {
+			schema = {
+				doc: currentDoc,
+				isArray,
+				isRequired,
+				type: unionParts,
+			};
+		} else {
+			schema = {
+				doc: currentDoc,
+				isArray,
+				isRequired,
+				enum: unionParts.map((value) => parseTypeScriptValue(value)),
+			};
+		}
+	} else if (isAllowedType(text)) {
+		schema = {
+			doc: currentDoc,
+			isArray,
+			isRequired,
+			type: text,
+		};
+	} else {
+		schema = {
+			doc: currentDoc,
+			isArray,
+			isRequired,
+			enum: [parseTypeScriptValue(text)],
+		};
+	}
+
+	return schema;
+}
+
+const TYPESCRIPT_ADDITIONAL_PROPERTIES_REGEX = /^\s*\[\s*key\s*:\s*string\s*\]\s*:\s*/;
+const TYPESCRIPT_PROPERTY_REGEX = /^(\w+)(\?)?:\s*/;
+
+function tryParseProperty(
+	body: string,
+	i: number,
+	currentDoc: string | undefined,
+): { propName?: string; schema: MiddleSchema; nextIndex: number } | null {
+	let cursor = i;
+
+	let propName: string | undefined;
+	let isRequired: boolean;
+
+	const bodySlice = body.slice(cursor);
+	const additionalPropertiesMatch = bodySlice.match(TYPESCRIPT_ADDITIONAL_PROPERTIES_REGEX);
+	if (additionalPropertiesMatch) {
+		cursor += additionalPropertiesMatch[0].length;
+		isRequired = true;
+	} else {
+		const propMatch = bodySlice.match(TYPESCRIPT_PROPERTY_REGEX);
+		if (!propMatch) {
+			return null;
+		}
+
+		propName = propMatch[1];
+		isRequired = !propMatch[2];
+		cursor += propMatch[0].length;
+	}
+
+	const { typeStr, isArray, isObject, typeEnd } = extractTypeString(body, cursor);
+	const schema = extractSchemaFromText(typeStr, currentDoc, isRequired, isArray, isObject);
+
+	let nextIndex = typeEnd;
+	if (body[nextIndex] === ";") {
+		nextIndex++;
+	}
+
+	return { propName, schema, nextIndex };
+}
+
+function parseInterfaceProperties(body: string): {
+	properties: Record<string, MiddleSchema>;
+	additionalProperties?: MiddleSchema;
+} {
+	const properties: Record<string, MiddleSchema> = {};
+	let additionalProperties: MiddleSchema | undefined;
+	let i = 0;
+	let currentDoc: string | undefined;
+
+	while (i < body.length) {
+		i = skipWhitespace(body, i);
+
+		const docResult = tryParseJsDoc(body, i);
+		if (docResult) {
+			currentDoc = docResult.doc;
+			i = docResult.nextIndex;
+			continue;
+		}
+
+		const propResult = tryParseProperty(body, i, currentDoc);
+		if (propResult) {
+			if (propResult.propName) {
+				properties[propResult.propName] = propResult.schema;
+			} else {
+				additionalProperties = propResult.schema;
+			}
+			currentDoc = undefined;
+			i = propResult.nextIndex;
+			continue;
+		}
+
+		i++;
+	}
+
+	return { properties, additionalProperties };
+}
+
+const TYPESCRIPT_JSDOC_REGEX = /\/\*\*\s*\n\s*\*\s*(.*?)\s*\n\s*\*\//;
+const TYPESCRIPT_INTERFACE_REGEX = /export interface (\w+) \{([\s\S]*)\}$/;
+
+export function parseTypeScriptInterface(text: string): MiddleObject {
+	// Extract JSDoc comment
+	const docMatch = text.match(TYPESCRIPT_JSDOC_REGEX);
+	const doc = docMatch ? docMatch[1] : undefined;
+
+	// Extract interface declaration
+	const interfaceMatch = text.match(TYPESCRIPT_INTERFACE_REGEX);
+	if (!interfaceMatch) {
+		throw new Error("Invalid TypeScript interface format");
+	}
+
+	const body = interfaceMatch[2];
+	const { properties, additionalProperties } = parseInterfaceProperties(body);
+
+	return { doc, isArray: false, isRequired: true, properties, additionalProperties };
+}

package/src/middle-schema/schema-types.ts

@@ -0,0 +1,40 @@
+const ALLOWED_TYPES = ["string", "number", "boolean", "object", "null", "unknown"] as const;
+
+export type AllowedType = (typeof ALLOWED_TYPES)[number];
+
+export function isAllowedType(type: string): type is AllowedType {
+	return ALLOWED_TYPES.includes(type as AllowedType);
+}
+
+interface MiddleBasicSpecs {
+	doc?: string;
+	isRequired: boolean;
+	isArray: boolean;
+}
+
+export interface MiddleBasicVar extends MiddleBasicSpecs {
+	type: AllowedType | AllowedType[];
+}
+
+export function isMiddleBasicVar(schema: MiddleSchema): schema is MiddleBasicVar {
+	return "type" in schema;
+}
+
+export interface MiddleEnum extends MiddleBasicSpecs {
+	enum: unknown[];
+}
+
+export function isMiddleEnum(schema: MiddleSchema): schema is MiddleEnum {
+	return "enum" in schema;
+}
+
+export interface MiddleObject extends MiddleBasicSpecs {
+	properties: Record<string, MiddleSchema>;
+	additionalProperties?: MiddleSchema;
+}
+
+export function isMiddleObject(schema: MiddleSchema): schema is MiddleObject {
+	return "properties" in schema;
+}
+
+export type MiddleSchema = MiddleBasicVar | MiddleEnum | MiddleObject;