@mat3ra/esse 2023.12.25-2

package/schema/workflow/unit/runtime/runtime_items.json

@@ -0,0 +1,36 @@
+{
+    "$id": "workflow/unit/runtime/runtime-items",
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "title": "runtime items schema (pre-/post-processors, monitors, results",
+    "type": "object",
+    "properties": {
+        "preProcessors": {
+            "description": "names of the pre-processors for this calculation",
+            "type": "array",
+            "items": {
+                "$ref": "runtime_item.json"
+            }
+        },
+        "postProcessors": {
+            "description": "names of the post-processors for this calculation",
+            "type": "array",
+            "items": {
+                "$ref": "runtime_item.json"
+            }
+        },
+        "monitors": {
+            "description": "names of the monitors for this calculation",
+            "type": "array",
+            "items": {
+                "$ref": "runtime_item.json"
+            }
+        },
+        "results": {
+            "description": "names of the results for this calculation",
+            "type": "array",
+            "items": {
+                "$ref": "runtime_item.json"
+            }
+        }
+    }
+}

package/schema/workflow/unit.json

@@ -0,0 +1,65 @@
+{
+    "$id": "workflow/unit",
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "title": "workflow unit schema",
+    "type": "object",
+    "allOf": [
+        {
+            "$ref": "../in_memory_entity/named_defaultable_runtime_items.json"
+        },
+        {
+            "$ref": "../system/tags.json"
+        }
+    ],
+    "properties": {
+        "_id": {
+            "type": "string"
+        },
+        "isDraft": {
+            "type": "boolean"
+        },
+        "type": {
+            "description": "type of the unit",
+            "type": "string"
+        },
+        "name": {
+            "description": "name of the unit. e.g. pw_scf",
+            "type": "string"
+        },
+        "status": {
+            "description": "Status of the unit.",
+            "type": "string",
+            "enum": [
+                "idle",
+                "active",
+                "warning",
+                "error",
+                "finished"
+            ]
+        },
+        "head": {
+            "description": "Whether this unit is the first one to be executed.",
+            "type": "boolean"
+        },
+        "flowchartId": {
+            "description": "Identity of the unit in the workflow. Used to trace the execution flow of the workflow.",
+            "type": "string"
+        },
+        "next": {
+            "description": "Next unit's flowchartId. If empty, the current unit is the last.",
+            "type": "string"
+        },
+        "enableRender": {
+            "description": "Whether Rupy should attempt to use Jinja templating to add context variables into the unit",
+            "type": "boolean"
+        },
+        "context": {
+            "type": "object"
+        }
+    },
+    "additionalProperties": true,
+    "required": [
+        "type",
+        "flowchartId"
+    ]
+}

package/schema/workflow.json

@@ -0,0 +1,63 @@
+{
+    "$id": "workflow",
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "title": "workflow schema",
+    "type": "object",
+    "allOf": [
+        {
+            "$ref": "workflow/base.json"
+        }
+    ],
+    "properties": {
+        "subworkflows": {
+            "description": "Array of subworkflows. Subworkflow can be an instance of workflow to allow for nesting",
+            "type": "array",
+            "items": {
+                "allOf": [
+                    {
+                        "$ref": "workflow/subworkflow.json"
+                    }
+                ]
+            }
+        },
+        "units": {
+            "description": "Contains the Units of the Workflow",
+            "type": "array",
+            "items": {
+                "oneOf": [
+                    {
+                        "$ref": "workflow/unit.json"
+                    },
+                    {
+                        "$ref": "workflow/unit/io.json"
+                    },
+                    {
+                        "$ref": "workflow/unit/map.json"
+                    },
+                    {
+                        "$ref": "workflow/unit/reduce.json"
+                    },
+                    {
+                        "$ref": "workflow/unit/condition.json"
+                    },
+                    {
+                        "$ref": "workflow/unit/assertion.json"
+                    },
+                    {
+                        "$ref": "workflow/unit/execution.json"
+                    },
+                    {
+                        "$ref": "workflow/unit/assignment.json"
+                    },
+                    {
+                        "$ref": "workflow/unit/processing.json"
+                    }
+                ]
+            }
+        }
+    },
+    "required": [
+        "units",
+        "subworkflows"
+    ]
+}

package/src/js/esse/index.js

@@ -0,0 +1,47 @@
+import Ajv from "ajv";
+
+import { buildSchemaDefinitions } from "./schemaUtils";
+import { EXAMPLES_DIR, SCHEMAS_DIR } from "./settings";
+import { parseIncludeReferenceStatementsByDir } from "./utils";
+
+const SCHEMAS = parseIncludeReferenceStatementsByDir(SCHEMAS_DIR);
+const EXAMPLES = parseIncludeReferenceStatementsByDir(EXAMPLES_DIR, true);
+
+export class ESSE {
+    constructor() {
+        this.schemas = SCHEMAS;
+        this.wrappedExamples = EXAMPLES;
+        this.examples = EXAMPLES.map((example) => example.data);
+    }
+
+    getSchemaById(schemaId) {
+        return this.schemas.find((schema) => schema.$id === schemaId);
+    }
+
+    /**
+     * Validates a given example against the schema.
+     * @param example {Object|Array} example to validate.
+     * @param schema {Object} schema to validate the example with.
+     * @returns {boolean} whether example is valid.
+     */
+    validate = (example, schema) => {
+        const ajv = new Ajv({ allErrors: true });
+        const isValid = ajv.validate(schema, example);
+
+        if (!isValid) {
+            console.error(ajv.errors);
+        }
+
+        return isValid;
+    };
+
+    buildGlobalSchema() {
+        return {
+            $id: "esse-global-schema",
+            $schema: "http://json-schema.org/draft-04/schema#",
+            title: "Global schema",
+            type: "object",
+            definitions: buildSchemaDefinitions(this.schemas),
+        };
+    }
+}

package/src/js/esse/schemaUtils.js

@@ -0,0 +1,45 @@
+import keyBy from "lodash/keyBy";
+
+export function mapObjectDeep(object, mapValue) {
+    if (typeof object !== "object") {
+        return object;
+    }
+
+    if (Array.isArray(object)) {
+        return object.map(
+            (innerValue) => mapValue(innerValue) || mapObjectDeep(innerValue, mapValue),
+        );
+    }
+
+    const entries = Object.entries(object).map(([key, value]) => {
+        const res = mapValue(value);
+
+        return [key, res === undefined ? mapObjectDeep(value, mapValue) : res];
+    });
+
+    return Object.fromEntries(entries);
+}
+
+export function makeFlatSchemaKey(schemaId) {
+    return schemaId.replace(/[^a-zA-Z-]/g, ":");
+}
+
+export function makeFlatSchemaRef(schemaId) {
+    return { $ref: `#/definitions/${makeFlatSchemaKey(schemaId)}` };
+}
+
+export function buildSchemaDefinitions(originalSchemas) {
+    const schemas = originalSchemas.map((schema) => {
+        return mapObjectDeep(schema, (value) => {
+            if (typeof value === "object" && value.$id) {
+                return makeFlatSchemaRef(value.$id);
+            }
+            if (typeof value === "object" && value.$ref) {
+                // assume value.$ref is a $id
+                return makeFlatSchemaRef(value.$ref);
+            }
+        });
+    });
+
+    return keyBy(schemas, ({ $id }) => makeFlatSchemaKey($id));
+}

package/src/js/esse/settings.js

@@ -0,0 +1,4 @@
+import path from "path";
+
+export const SCHEMAS_DIR = path.resolve(__dirname, "../../../schema");
+export const EXAMPLES_DIR = path.resolve(__dirname, "../../../example");

package/src/js/esse/utils.js

@@ -0,0 +1,52 @@
+import file from "file";
+import deref from "json-schema-deref-sync";
+import path from "path";
+
+import { JSONInclude } from "../json_include";
+
+/**
+ * Resolves `include` and `$ref` statements.
+ * @param filePath {String} file to parse.
+ */
+export function parseIncludeReferenceStatements(filePath) {
+    const jsonResolver = new JSONInclude();
+    const parsed = jsonResolver.parseIncludeStatements(filePath);
+    const dirPath = path.dirname(filePath);
+    let dereferenced = deref(parsed, { baseFolder: dirPath });
+    // handle circular references and use non-dereferenced source
+    if (dereferenced instanceof Error && dereferenced.message === "Circular self reference") {
+        dereferenced = parsed;
+    }
+    return dereferenced;
+}
+
+/**
+ * Resolves `include` and `$ref` statements for all the JSON files inside a given directory.
+ * @param dirPath {String} directory to parse.
+ */
+export function parseIncludeReferenceStatementsByDir(dirPath, wrapInDataAndPath = false) {
+    const data = [];
+    const topDir = path.resolve(__dirname, "../../../");
+    file.walkSync(dirPath, (dirPath_, dirs_, files_) => {
+        files_.forEach((file_) => {
+            const filePath = path.join(dirPath_, file_);
+            if (filePath.endsWith(".json")) {
+                const config = parseIncludeReferenceStatements(filePath);
+                if (wrapInDataAndPath) {
+                    const _path = path.join(
+                        // remove leading slashes and "example" from path
+                        path
+                            .dirname(filePath)
+                            .replace(path.join(topDir, "example"), "")
+                            .replace(/^\/+/, ""),
+                        path.basename(filePath).replace(".json", ""),
+                    );
+                    data.push({ data: config, path: _path });
+                } else {
+                    data.push(config);
+                }
+            }
+        });
+    });
+    return data;
+}

package/src/js/json_include/index.js

@@ -0,0 +1,71 @@
+import fs from "fs";
+import path from "path";
+
+import { INCLUDE_KEY, INCLUDE_VALUE_REGEX, OBJECT_ONLY } from "./settings";
+import { isInstanceOf, safeParseJSON } from "./utils";
+
+export class JSONInclude {
+    constructor() {
+        this.JSON_INCLUDE_CACHE = {};
+    }
+
+    /**
+     * Extracts file name(path) from the include statement.
+     * @param value {String} string to extract the file name from.
+     */
+    _getIncludeFileName = (value) => {
+        if (isInstanceOf(value, "String") && value.search(INCLUDE_VALUE_REGEX) !== -1) {
+            return value.match(INCLUDE_VALUE_REGEX)[1];
+        }
+    };
+
+    /**
+     * Walks a nested object to resolve include statements.
+     * @param obj {Object} Object to traverse.
+     * @param dirpath {String} directory from which `obj` is obtained. Include statements are relative to `obj` path.
+     */
+    _walkObjectToInclude(obj, dirpath) {
+        if (isInstanceOf(obj, "Object")) {
+            if (INCLUDE_KEY in obj) {
+                const includeName = this._getIncludeFileName(obj[INCLUDE_KEY]);
+                if (includeName) {
+                    delete obj[INCLUDE_KEY];
+                    if (!(includeName in this.JSON_INCLUDE_CACHE)) {
+                        const filePath = path.join(dirpath, includeName);
+                        this.JSON_INCLUDE_CACHE[includeName] =
+                            this.parseIncludeStatements(filePath);
+                    }
+                    Object.keys(this.JSON_INCLUDE_CACHE[includeName]).forEach((attr) => {
+                        obj[attr] = this.JSON_INCLUDE_CACHE[includeName][attr];
+                    });
+                }
+            }
+            Object.keys(obj).forEach((key) => {
+                if (isInstanceOf(obj[key], "Object") || isInstanceOf(obj[key], "Array")) {
+                    this._walkObjectToInclude(obj[key], dirpath);
+                }
+            });
+        } else if (isInstanceOf(obj, "Array")) {
+            for (let i = 0; i < obj.length; i++) {
+                if (isInstanceOf(obj[i], "Object") || isInstanceOf(obj[i], "Array")) {
+                    this._walkObjectToInclude(obj[i], dirpath);
+                }
+            }
+        }
+    }
+
+    /**
+     * Resolves `include` statements.
+     * @param filePath {String} file to parse.
+     */
+    parseIncludeStatements(filePath) {
+        const data = safeParseJSON(fs.readFileSync(filePath, "utf8"));
+        if (OBJECT_ONLY && !isInstanceOf(data, "Object")) {
+            throw new Error(
+                "The JSON file being included should always be a dict rather than a list",
+            );
+        }
+        this._walkObjectToInclude(data, path.dirname(filePath));
+        return data;
+    }
+}

package/src/js/json_include/settings.js

@@ -0,0 +1,3 @@
+export const INCLUDE_KEY = "...";
+export const INCLUDE_VALUE_REGEX = /^include\((.+)\)$/;
+export const OBJECT_ONLY = false; // whether to only include Object, and not Array.

package/src/js/json_include/utils.js

@@ -0,0 +1,11 @@
+export function isInstanceOf(object, type) {
+    return Object.prototype.toString.call(object).slice(8, -1) === type;
+}
+
+/**
+ * Makes sure that text representing arrays is parsed into Arrays.
+ */
+export function safeParseJSON(string) {
+    const obj = JSON.parse(string);
+    return string[0] === "[" ? Object.keys(obj).map((key) => obj[key]) : obj;
+}

package/src/js/scripts/createEnumsFromYaml.mjs

@@ -0,0 +1,43 @@
+/* eslint-disable no-restricted-syntax */
+import fs from "fs";
+import yaml from "js-yaml";
+import path from "path";
+import lodash from "lodash";
+
+/**
+ *  We use YAML files to list enum options which need to be reused in other places.
+ *  The format also allows for additional context to the often shortened slugs, such as DOIs.
+ *  This script converts the enum lists to JSON files for $ref blocks inside schemas.
+ */
+
+const SCHEMA_DIR = "../../../schema/";
+
+function walkDir(dir, callback) {
+    const subDirs = fs.readdirSync(dir);
+
+    for (const subDir of subDirs) {
+        const itemPath = path.join(dir, subDir);
+        const stat = fs.statSync(itemPath);
+
+        if (stat.isDirectory()) {
+            walkDir(itemPath, callback);
+        } else {
+            callback(itemPath);
+        }
+    }
+}
+
+walkDir(SCHEMA_DIR, (filePath) => {
+    if (path.extname(filePath) !== ".yml") {
+        return;
+    }
+    console.log(filePath);
+    const outFilename = `${path.basename(filePath, ".yml")}.json`;
+    const dirname = path.dirname(filePath);
+
+    const fileContents = fs.readFileSync(filePath);
+    const obj = yaml.load(fileContents);
+    const enumObj = lodash.mapValues(obj, (value) => ({ enum: value }));
+
+    fs.writeFileSync(`${path.join(dirname, outFilename)}`, `${JSON.stringify(enumObj, null, 4)}\n`);
+});

package/src/js/scripts/setSchemaIds.mjs

@@ -0,0 +1,34 @@
+/* eslint-disable no-restricted-syntax */
+/* eslint-disable no-await-in-loop */
+import fs from "fs/promises";
+import path from "path";
+
+const SCHEMA_DIR = "../../../schema/";
+
+async function walkDir(dir, callback) {
+    const subDirs = await fs.readdir(dir);
+
+    for (const subDir of subDirs) {
+        const itemPath = path.join(dir, subDir);
+        const stat = await fs.stat(itemPath);
+
+        if (stat.isDirectory()) {
+            await walkDir(itemPath, callback);
+        } else {
+            await callback(itemPath);
+        }
+    }
+}
+
+await walkDir(SCHEMA_DIR, async (filePath) => {
+    if (path.extname(filePath) !== ".json") {
+        return;
+    }
+
+    const fileContents = await fs.readFile(filePath);
+    const { $id: oldSchemaId, ...schema } = JSON.parse(fileContents);
+    const schemaId = filePath.replace(SCHEMA_DIR, "").replace(".json", "").replace(/_/g, "-");
+    const newContent = JSON.stringify({ $id: schemaId, ...schema }, null, 4);
+
+    await fs.writeFile(filePath, `${newContent}\n`);
+});