@mat3ra/esse 2024.2.6-0 → 2024.2.19-0

package/src/js/esse/JSONSchemasGenerator.ts

@@ -0,0 +1,94 @@
+/* eslint-disable class-methods-use-this */
+import fs from "fs";
+import yaml from "js-yaml";
+import mergeAllOf from "json-schema-merge-allof";
+import path from "path";
+
+import { EXAMPLES_DIR, PROPERTIES_MANIFEST_PATH, SCHEMAS_DIR } from "./settings";
+import { JSONSchema, JSONSchemaWithPath, parseIncludeReferenceStatementsByDir } from "./utils";
+
+interface JSONSchemasGeneratorConfig {
+    schemasDir: string;
+    examplesDir?: string;
+    propertiesManifestDir?: string;
+}
+
+const DEFAULT_CONFIG = {
+    schemasDir: SCHEMAS_DIR,
+    examplesDir: EXAMPLES_DIR,
+    propertiesManifestDir: PROPERTIES_MANIFEST_PATH,
+};
+
+export default class JSONSchemasGenerator implements JSONSchemasGeneratorConfig {
+    readonly schemas: JSONSchema[];
+
+    readonly examples?: JSONSchema[];
+
+    readonly wrappedExamples: JSONSchemaWithPath[];
+
+    readonly propertiesManifest: object;
+
+    readonly results: object;
+
+    readonly schemasDir: string;
+
+    readonly examplesDir?: string;
+
+    readonly propertiesManifestDir?: string;
+
+    constructor(config: JSONSchemasGeneratorConfig = DEFAULT_CONFIG) {
+        this.schemasDir = config.schemasDir;
+        this.examplesDir = config.examplesDir;
+        this.propertiesManifestDir = config.propertiesManifestDir;
+
+        this.schemas = parseIncludeReferenceStatementsByDir(this.schemasDir);
+
+        this.wrappedExamples = this.examplesDir
+            ? parseIncludeReferenceStatementsByDir(this.examplesDir, true)
+            : [];
+
+        this.examples = this.wrappedExamples.map((example) => example.data);
+
+        this.propertiesManifest = this.propertiesManifestDir
+            ? (yaml.load(fs.readFileSync(this.propertiesManifestDir, "utf-8")) as object)
+            : {};
+
+        this.results = Object.entries(this.propertiesManifest)
+            .map((k) => (k[1].isResult ? k[0] : null))
+            .filter((x) => x) as object;
+    }
+
+    writeResolvedSchemas(subfolder: string, skipMergeAllOff = false) {
+        const schemasFolder = `${subfolder}/schema`;
+        fs.rmSync(schemasFolder, { recursive: true, force: true });
+
+        const mergeAllOfConfig = {
+            resolvers: { defaultResolver: mergeAllOf.options.resolvers.title },
+        };
+
+        const schemas = this.schemas.map((schema) => {
+            console.log(`Resolving schema: ${schema.$id}`);
+            const mergedSchema = skipMergeAllOff ? schema : mergeAllOf(schema, mergeAllOfConfig);
+            const idAsPath = mergedSchema.$id?.replace(/-/g, "_");
+            const fullPath = `${schemasFolder}/${idAsPath}.json`;
+
+            fs.mkdirSync(path.dirname(fullPath), { recursive: true });
+            fs.writeFileSync(fullPath, JSON.stringify(mergedSchema, null, 4), "utf8");
+
+            return mergedSchema;
+        });
+
+        fs.writeFileSync(`${subfolder}/schemas.json`, `${JSON.stringify(schemas)}`);
+    }
+
+    writeResolvedExamples(subfolder: string) {
+        const examplesFolder = `${subfolder}/example`;
+        fs.rmSync(`${examplesFolder}`, { recursive: true, force: true });
+        this.wrappedExamples.forEach((e) => {
+            const idAsPath = e.path.replace(/-/g, "_");
+            const fullPath = `${examplesFolder}/${idAsPath}.json`;
+            fs.mkdirSync(path.dirname(fullPath), { recursive: true });
+            fs.writeFileSync(fullPath, JSON.stringify(e.data, null, 4), "utf8");
+        });
+    }
+}

package/src/js/esse/JSONSchemasInterface.ts

@@ -0,0 +1,56 @@
+import { JSONSchema } from "./utils";
+
+export type JSONSchemasInterfaceQuery = { [key in keyof JSONSchema]: { $regex: string } };
+
+export default class JSONSchemasInterface {
+    static schemasCache = new Map<string, JSONSchema>();
+
+    static setSchemas(schema: JSONSchema[]) {
+        schema.forEach((schema) => this.addSchema(schema));
+    }
+
+    static addSchema(schema: JSONSchema) {
+        if (schema.$id) {
+            this.schemasCache.set(schema.$id, schema);
+        }
+    }
+
+    static getSchemaById(schemaId: string) {
+        return this.schemasCache.get(schemaId);
+    }
+
+    /**
+     * @example <caption>Search by $id regex</caption>
+     * JSONSchemasInterface.matchSchema({
+     *   $id: {
+     *     $regex: 'software-application'
+     *   }
+     * })
+     *
+     * @example <caption>Search by $id and title regex</caption>
+     * JSONSchemasInterface.matchSchema({
+     *   $id: {
+     *     $regex: 'software-application'
+     *   },
+     *   title: {
+     *     $regex: 'application'
+     *   }
+     * })
+     */
+    static matchSchema(query: JSONSchemasInterfaceQuery) {
+        const searchFields = Object.keys(query) as Array<keyof typeof query>;
+
+        return Array.from(this.schemasCache.values()).find((schema) => {
+            return searchFields.every((field) => {
+                const queryField = query[field];
+                const schemaField = schema[field];
+
+                if (!queryField || typeof schemaField !== "string") {
+                    return;
+                }
+
+                return new RegExp(queryField.$regex).test(schemaField);
+            });
+        });
+    }
+}

package/src/js/esse/JSONSchemasInterfaceServer.ts

@@ -0,0 +1,49 @@
+import { SchemaObject } from "ajv";
+import fs from "fs";
+import path from "path";
+
+import { walkDirSync } from "../utils/filesystem";
+import JSONSchemasInterface from "./JSONSchemasInterface";
+
+export function readSchemaFolderSync(folderPath: string) {
+    const schemas: SchemaObject[] = [];
+
+    walkDirSync(folderPath, (filePath: string) => {
+        if (path.extname(filePath) !== ".json") {
+            return;
+        }
+        const schema = JSON.parse(fs.readFileSync(filePath).toString());
+        schemas.push(schema);
+    });
+
+    return schemas;
+}
+
+export default class JSONSchemasInterfaceServer extends JSONSchemasInterface {
+    static schemaFolder = path.resolve(__dirname, "./../schema");
+
+    static setSchemaFolder(schemaFolder: string) {
+        if (this.schemaFolder !== schemaFolder) {
+            this.schemaFolder = schemaFolder;
+            this.readSchemaFolder();
+        }
+    }
+
+    static readSchemaFolder() {
+        const schemas = readSchemaFolderSync(this.schemaFolder);
+
+        schemas.forEach((schema) => {
+            if (schema.$id) {
+                this.schemasCache.set(schema.$id, schema);
+            }
+        });
+    }
+
+    static getSchemaById(schemaId: string) {
+        if (this.schemasCache.size === 0) {
+            this.readSchemaFolder();
+        }
+
+        return super.getSchemaById(schemaId);
+    }
+}

package/src/js/esse/schemaUtils.ts

@@ -0,0 +1,28 @@
+import { JSONSchema, JSONSchemaDefinition } from "./utils";
+
+export type MapSchema = (prop: JSONSchemaDefinition) => JSONSchemaDefinition | undefined;
+
+export function mapObjectDeep(object: JSONSchemaDefinition, mapValue: MapSchema): JSONSchema;
+
+export function mapObjectDeep(object: JSONSchemaDefinition[], mapValue: MapSchema): JSONSchema[];
+
+export function mapObjectDeep(
+    object: JSONSchemaDefinition | JSONSchemaDefinition[],
+    mapValue: MapSchema,
+): JSONSchemaDefinition | JSONSchemaDefinition[] {
+    if (typeof object !== "object" || object === null) {
+        return object;
+    }
+
+    if (Array.isArray(object)) {
+        return object.map((innerValue) => mapObjectDeep(innerValue, mapValue));
+    }
+
+    const mappedObject = mapValue(object) || object;
+
+    const entries = Object.entries(mappedObject).map(([key, value]) => {
+        return [key, mapObjectDeep(value, mapValue)];
+    });
+
+    return Object.fromEntries(entries);
+}

package/src/js/esse/types.ts

@@ -0,0 +1,3 @@
+export interface AnyObject {
+    [key: string]: unknown;
+}

package/src/js/esse/utils.ts

@@ -0,0 +1,72 @@
+import { JSONSchema7, JSONSchema7Definition } from "json-schema";
+import deref from "json-schema-deref-sync";
+import path from "path";
+
+import { JSONInclude } from "../json_include";
+import { walkDirSync } from "../utils/filesystem";
+
+export type JSONSchema = JSONSchema7;
+
+export type JSONSchemaDefinition = JSONSchema7Definition;
+
+/**
+ * Resolves `include` and `$ref` statements.
+ * @param filePath {String} file to parse.
+ */
+export function parseIncludeReferenceStatements(filePath: string): JSONSchema {
+    const jsonResolver = new JSONInclude();
+    const parsed = jsonResolver.parseIncludeStatements(filePath);
+    const dirPath = path.dirname(filePath);
+    let dereferenced = deref(parsed, { baseFolder: dirPath, removeIds: true });
+    // handle circular references and use non-dereferenced source
+    if (dereferenced instanceof Error && dereferenced.message === "Circular self reference") {
+        dereferenced = parsed as JSONSchema;
+    }
+    return dereferenced;
+}
+
+export interface JSONSchemaWithPath {
+    data: JSONSchema;
+    path: string;
+}
+
+export function parseIncludeReferenceStatementsByDir(
+    dirPath: string,
+    wrapInDataAndPath: true,
+): JSONSchemaWithPath[];
+
+export function parseIncludeReferenceStatementsByDir(
+    dirPath: string,
+    wrapInDataAndPath?: false,
+): JSONSchema[];
+
+/**
+ * Resolves `include` and `$ref` statements for all the JSON files inside a given directory.
+ * @param dirPath directory to parse.
+ */
+export function parseIncludeReferenceStatementsByDir(dirPath: string, wrapInDataAndPath = false) {
+    const schemas: JSONSchema[] = [];
+    const schemasWithPath: JSONSchemaWithPath[] = [];
+    const topDir = path.resolve(__dirname, "../../../");
+
+    walkDirSync(dirPath, (filePath) => {
+        if (filePath.endsWith(".json")) {
+            const config = parseIncludeReferenceStatements(filePath);
+            if (wrapInDataAndPath) {
+                const _path = path.join(
+                    // remove leading slashes and "example" from path
+                    path
+                        .dirname(filePath)
+                        .replace(path.join(topDir, "example"), "")
+                        .replace(/^\/+/, ""),
+                    path.basename(filePath).replace(".json", ""),
+                );
+                schemasWithPath.push({ data: config, path: _path });
+            } else {
+                schemas.push(config);
+            }
+        }
+    });
+
+    return wrapInDataAndPath ? schemasWithPath : schemas;
+}

package/src/js/json_include/{index.js → index.ts}

@@ -1,31 +1,37 @@
 import fs from "fs";
 import path from "path";
 
+import { AnyObject } from "../esse/types";
+import { isInstanceOf, safeParseJSON } from "../utils/common";
 import { INCLUDE_KEY, INCLUDE_VALUE_REGEX, OBJECT_ONLY } from "./settings";
-import { isInstanceOf, safeParseJSON } from "./utils";
 
 export class JSONInclude {
-    constructor() {
-        this.JSON_INCLUDE_CACHE = {};
-    }
+    JSON_INCLUDE_CACHE: AnyObject = {};
 
     /**
      * Extracts file name(path) from the include statement.
-     * @param value {String} string to extract the file name from.
+     * @param value string to extract the file name from.
      */
-    _getIncludeFileName = (value) => {
-        if (isInstanceOf(value, "String") && value.search(INCLUDE_VALUE_REGEX) !== -1) {
-            return value.match(INCLUDE_VALUE_REGEX)[1];
+    _getIncludeFileName = (value: unknown) => {
+        if (typeof value === "string" && value.search(INCLUDE_VALUE_REGEX) !== -1) {
+            const matched = value.match(INCLUDE_VALUE_REGEX);
+            return matched && matched[1];
         }
     };
 
     /**
      * Walks a nested object to resolve include statements.
-     * @param obj {Object} Object to traverse.
-     * @param dirpath {String} directory from which `obj` is obtained. Include statements are relative to `obj` path.
+     * @param obj Object to traverse.
+     * @param dirpath directory from which `obj` is obtained. Include statements are relative to `obj` path.
      */
-    _walkObjectToInclude(obj, dirpath) {
-        if (isInstanceOf(obj, "Object")) {
+    _walkObjectToInclude(obj: AnyObject | AnyObject[], dirpath: string) {
+        if (isInstanceOf(obj, "Array")) {
+            for (let i = 0; i < obj.length; i++) {
+                if (isInstanceOf(obj[i], "Object") || isInstanceOf(obj[i], "Array")) {
+                    this._walkObjectToInclude(obj[i], dirpath);
+                }
+            }
+        } else if (isInstanceOf(obj, "Object")) {
             if (INCLUDE_KEY in obj) {
                 const includeName = this._getIncludeFileName(obj[INCLUDE_KEY]);
                 if (includeName) {
@@ -36,6 +42,7 @@ export class JSONInclude {
                             this.parseIncludeStatements(filePath);
                     }
                     Object.keys(this.JSON_INCLUDE_CACHE[includeName]).forEach((attr) => {
+                        // @ts-ignore
                         obj[attr] = this.JSON_INCLUDE_CACHE[includeName][attr];
                     });
                 }
@@ -45,21 +52,15 @@ export class JSONInclude {
                     this._walkObjectToInclude(obj[key], dirpath);
                 }
             });
-        } else if (isInstanceOf(obj, "Array")) {
-            for (let i = 0; i < obj.length; i++) {
-                if (isInstanceOf(obj[i], "Object") || isInstanceOf(obj[i], "Array")) {
-                    this._walkObjectToInclude(obj[i], dirpath);
-                }
-            }
         }
     }
 
     /**
      * Resolves `include` statements.
-     * @param filePath {String} file to parse.
+     * @param filePath file to parse.
      */
-    parseIncludeStatements(filePath) {
-        const data = safeParseJSON(fs.readFileSync(filePath, "utf8"));
+    parseIncludeStatements(filePath: string) {
+        const data: AnyObject[] = safeParseJSON(fs.readFileSync(filePath, "utf8"));
         if (OBJECT_ONLY && !isInstanceOf(data, "Object")) {
             throw new Error(
                 "The JSON file being included should always be a dict rather than a list",

package/src/js/scripts/compileTs.ts

@@ -0,0 +1,84 @@
+import { SchemaObject } from "ajv";
+import fs from "fs";
+import { compile } from "json-schema-to-typescript";
+
+import { mapObjectDeep } from "../esse/schemaUtils";
+import { JSONSchema } from "../esse/utils";
+import { walkDir } from "../utils/filesystem";
+
+/**
+ * By definition, the "compile" function will generate 2 schemas based on the following input:
+ * {
+ *    "title": "schema1",
+ *    "type": "object",
+ *    "properties": {
+ *        "prop": {
+ *             "title": "schema2",
+ *             "type": "string"
+ *        }
+ *    }
+ * }
+ *
+ * Result:
+ * type schema2 = string;
+ *
+ * interface schema1 {
+ *      prop: schema2
+ * }
+ *
+ * To disable this behavior we need to remove "title" property from the "prop":
+ *  {
+ *    "title": "schema1",
+ *    "type": "object",
+ *    "properties": {
+ *        "prop": {
+ *             "type": "string"
+ *        }
+ *    }
+ * }
+ *
+ * New result:
+ * interface schema1 {
+ *      prop: string;
+ * }
+ * @returns Clean schema
+ */
+function cleanSchema(schema: JSONSchema): JSONSchema {
+    let firstRun = true;
+
+    return mapObjectDeep(schema, (object) => {
+        if (typeof object === "object" && object?.title && !firstRun) {
+            firstRun = false;
+
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            const { title, $schema, ...restObject } = object as SchemaObject;
+            return restObject;
+        }
+
+        firstRun = false;
+    });
+}
+
+export default async function compileTS(schemaPath: string, savePath: string) {
+    try {
+        await fs.promises.unlink(savePath);
+    } catch (err) {
+        console.log("File with types not exists");
+    }
+
+    await walkDir(schemaPath, async (filePath) => {
+        const data = await fs.promises.readFile(filePath, "utf8");
+        const schema = cleanSchema(JSON.parse(data));
+
+        console.log(`Compiling Typescript: ${filePath}`);
+
+        // @ts-ignore
+        const compiledSchema = await compile(schema, schema.title || "", {
+            unreachableDefinitions: true,
+            additionalProperties: false,
+            bannerComment: `/** Schema ${filePath} */`,
+        });
+
+        await fs.promises.appendFile(savePath, `${compiledSchema} \n`, { flag: "a+" });
+    });
+}

package/src/js/scripts/{createEnumsFromYaml.mjs → createEnumsFromYaml.ts}

@@ -1,8 +1,9 @@
-/* eslint-disable no-restricted-syntax */
 import fs from "fs";
 import yaml from "js-yaml";
-import path from "path";
 import lodash from "lodash";
+import path from "path";
+
+import { walkDir } from "../utils/filesystem";
 
 /**
  *  We use YAML files to list enum options which need to be reused in other places.
@@ -12,22 +13,7 @@ import lodash from "lodash";
 
 const SCHEMA_DIR = "../../../schema/";
 
-function walkDir(dir, callback) {
-    const subDirs = fs.readdirSync(dir);
-
-    for (const subDir of subDirs) {
-        const itemPath = path.join(dir, subDir);
-        const stat = fs.statSync(itemPath);
-
-        if (stat.isDirectory()) {
-            walkDir(itemPath, callback);
-        } else {
-            callback(itemPath);
-        }
-    }
-}
-
-walkDir(SCHEMA_DIR, (filePath) => {
+walkDir(SCHEMA_DIR, (filePath: string) => {
     if (path.extname(filePath) !== ".yml") {
         return;
     }
@@ -36,7 +22,7 @@ walkDir(SCHEMA_DIR, (filePath) => {
     const dirname = path.dirname(filePath);
 
     const fileContents = fs.readFileSync(filePath);
-    const obj = yaml.load(fileContents);
+    const obj = yaml.load(fileContents.toString()) as object;
     const enumObj = lodash.mapValues(obj, (value) => ({ enum: value }));
 
     fs.writeFileSync(`${path.join(dirname, outFilename)}`, `${JSON.stringify(enumObj, null, 4)}\n`);

package/src/js/scripts/setSchemaIds.ts

@@ -0,0 +1,28 @@
+import fs from "fs";
+import path from "path";
+
+import { walkDirSync } from "../utils/filesystem";
+
+/**
+ * Helper script.
+ * Applies correct $id to all schemas based on the schema path
+ */
+
+export default function setSchemaIds(schemaDir: string) {
+    walkDirSync(schemaDir, (filePath) => {
+        if (path.extname(filePath) !== ".json") {
+            return;
+        }
+
+        const fileContents = fs.readFileSync(filePath);
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const { $id, ...schema } = JSON.parse(fileContents.toString());
+        const schemaId = filePath.replace(schemaDir, "").replace(".json", "").replace(/_/g, "-");
+        if ($id !== schemaId) {
+            console.log(`Set correct $id: ${filePath}`);
+            const newContent = JSON.stringify({ $id: schemaId, ...schema }, null, 4);
+
+            fs.writeFileSync(filePath, `${newContent}\n`);
+        }
+    });
+}

package/src/js/utils/ajv.ts

@@ -0,0 +1,119 @@
+import type {} from "ajv"; // @see https://github.com/microsoft/TypeScript/issues/47663
+import Ajv, { SchemaObject } from "ajv";
+import { AnyValidateFunction } from "ajv/dist/core";
+
+import { mapObjectDeep } from "../esse/schemaUtils";
+import { AnyObject } from "../esse/types";
+import { JSONSchema } from "../esse/utils";
+
+function addAdditionalPropertiesToSchema(schema: JSONSchema, additionalProperties = false) {
+    return mapObjectDeep(schema, (object) => {
+        const schema = object as JSONSchema;
+
+        if (
+            typeof object === "object" &&
+            schema?.type === "object" &&
+            schema?.properties &&
+            !("additionalProperties" in schema)
+        ) {
+            return {
+                ...schema,
+                additionalProperties,
+                unevaluatedProperties: false,
+            };
+        }
+    });
+}
+
+const ajvConfig = {
+    strict: false, // TODO: adjust schemas and enable strict mode
+    useDefaults: true,
+    /**
+     * discriminator fixes default values in oneOf
+     * @see https://ajv.js.org/guide/modifying-data.html#assigning-defaults
+     */
+    discriminator: true,
+};
+
+const ajvValidator = new Ajv({ ...ajvConfig });
+const ajvValidatorAndCleaner = new Ajv({ ...ajvConfig, removeAdditional: true });
+const ajvValidatorAndCleanerWithCoercingTypes = new Ajv({
+    ...ajvConfig,
+    removeAdditional: true,
+    coerceTypes: true,
+});
+
+interface AjvInstanceOptions {
+    clean: boolean;
+    coerceTypes: boolean;
+}
+
+function getAjvInstance({ clean, coerceTypes }: AjvInstanceOptions) {
+    if (clean && coerceTypes) {
+        return ajvValidatorAndCleanerWithCoercingTypes;
+    }
+
+    if (clean) {
+        return ajvValidatorAndCleaner;
+    }
+
+    return ajvValidator;
+}
+
+export function getValidator(
+    jsonSchema: SchemaObject,
+    { clean, coerceTypes }: AjvInstanceOptions,
+): AnyValidateFunction {
+    const schemaKey = jsonSchema.$id as string;
+    const ajv = getAjvInstance({ clean, coerceTypes });
+    let validate = ajv.getSchema(schemaKey);
+
+    if (!validate) {
+        // properties that were not defined in schema will be ignored when clean = false
+        const patchedSchema = clean ? addAdditionalPropertiesToSchema(jsonSchema) : jsonSchema;
+        ajv.addSchema(patchedSchema, schemaKey);
+        validate = ajv.getSchema(schemaKey);
+    }
+
+    if (!validate) {
+        throw new Error("JSONSchemasInterface AJV validator error");
+    }
+
+    return validate;
+}
+
+/**
+ * Validates a given example against the schema.
+ * @param example example to validate.
+ * @param schema schema to validate the example with.
+ * @returns whether example is valid.
+ */
+export function validate(data: AnyObject, jsonSchema: SchemaObject) {
+    const validator = getValidator(jsonSchema, { clean: false, coerceTypes: false });
+    const isValid = validator(data);
+
+    return {
+        isValid,
+        errors: validator.errors,
+    };
+}
+
+/**
+ * Validates and clean a given example against the schema
+ * @param example example to validate.
+ * @param schema schema to validate the example with.
+ * @returns whether example is valid.
+ */
+export function validateAndClean(
+    data: AnyObject,
+    jsonSchema: SchemaObject,
+    { coerceTypes = false },
+) {
+    const validator = getValidator(jsonSchema, { clean: true, coerceTypes });
+    const isValid = validator(data);
+
+    return {
+        isValid,
+        errors: validator.errors,
+    };
+}

package/src/js/utils/common.ts

@@ -0,0 +1,15 @@
+export function isInstanceOf(object: object, type: "Array"): object is object[];
+
+export function isInstanceOf(object: object, type: "Object"): object is object;
+
+export function isInstanceOf(object: object, type: string): boolean {
+    return Object.prototype.toString.call(object).slice(8, -1) === type;
+}
+
+/**
+ * Makes sure that text representing arrays is parsed into Arrays.
+ */
+export function safeParseJSON(string: string) {
+    const obj = JSON.parse(string);
+    return string[0] === "[" ? Object.keys(obj).map((key) => obj[key]) : obj;
+}