@mat3ra/esse 2026.5.18-3 → 2026.5.28-0

package/schema/workflow/unit/mixins/io.json

@@ -0,0 +1,46 @@
+{
+    "$id": "workflow/unit/mixins/io",
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "data IO unit mixin schema",
+    "type": "object",
+    "properties": {
+        "type": {
+            "const": "io"
+        },
+        "subtype": {
+            "enum": [
+                "input",
+                "output",
+                "dataFrame"
+            ]
+        },
+        "source": {
+            "enum": [
+                "api",
+                "object_storage"
+            ]
+        },
+        "input": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "oneOf": [
+                    {
+                        "$ref": "../io/api.json"
+                    },
+                    {
+                        "$ref": "../io/object_storage.json"
+                    }
+                ],
+                "discriminator": {
+                    "propertyName": "type"
+                }
+            }
+        }
+    },
+    "required": [
+        "subtype",
+        "source",
+        "input"
+    ]
+}

package/schema/workflow/unit/mixins/map.json

@@ -0,0 +1,61 @@
+{
+    "$id": "workflow/unit/mixins/map",
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "map unit mixin schema",
+    "type": "object",
+    "properties": {
+        "type": {
+            "const": "map"
+        },
+        "workflowId": {
+            "description": "Id of workflow to run inside map",
+            "type": "string"
+        },
+        "input": {
+            "description": "Input information for map.",
+            "type": "object",
+            "properties": {
+                "target": {
+                    "description": "Name of the target variable to substitute using the values below. e.g. K_POINTS",
+                    "type": "string"
+                },
+                "scope": {
+                    "description": "Scope to retrieve `values` from, global or flowchartId. Optional if `values` is given.",
+                    "type": "string"
+                },
+                "name": {
+                    "description": "Name of the variable inside the scope to retrieve `values` from. Optional if `values` is given.",
+                    "type": "string"
+                },
+                "values": {
+                    "description": "Sequence of values for the target Jinja variable. Optional if `scope` and `name` are given. This can be used for map-reduce type parallel execution",
+                    "type": "array",
+                    "items": {
+                        "oneOf": [
+                            {
+                                "type": "string"
+                            },
+                            {
+                                "type": "number"
+                            },
+                            {
+                                "type": "object"
+                            }
+                        ]
+                    }
+                },
+                "useValues": {
+                    "type": "boolean"
+                }
+            },
+            "required": [
+                "target"
+            ]
+        }
+    },
+    "required": [
+        "input",
+        "workflowId"
+    ]
+}
+

package/schema/workflow/unit/{processing.json → mixins/processing.json}

@@ -1,13 +1,8 @@
 {
-    "$id": "workflow/unit/processing",
+    "$id": "workflow/unit/mixins/processing",
     "$schema": "http://json-schema.org/draft-07/schema#",
-    "title": "processing unit schema",
+    "title": "processing unit mixin schema",
     "type": "object",
-    "allOf": [
-        {
-            "$ref": "./base.json"
-        }
-    ],
     "properties": {
         "type": {
             "const": "processing"
@@ -30,3 +25,4 @@
         "inputData"
     ]
 }
+

package/schema/workflow/unit/mixins/reduce.json

@@ -0,0 +1,44 @@
+{
+    "$id": "workflow/unit/mixins/reduce",
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "reduce unit mixin schema",
+    "type": "object",
+    "properties": {
+        "type": {
+            "const": "reduce"
+        },
+        "mapFlowchartId": {
+            "description": "corresponding map unit flowchart ID",
+            "type": "string"
+        },
+        "input": {
+            "description": "input information for reduce unit",
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "operation": {
+                        "description": "reduce operation, e.g. aggregate",
+                        "type": "string"
+                    },
+                    "arguments": {
+                        "description": "arguments which are passed to reduce operation function",
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        }
+                    }
+                },
+                "required": [
+                    "operation",
+                    "arguments"
+                ]
+            }
+        }
+    },
+    "required": [
+        "mapFlowchartId",
+        "input"
+    ]
+}
+

package/schema/workflow/unit/mixins/subworkflow.json

@@ -0,0 +1,12 @@
+{
+    "$id": "workflow/unit/mixins/subworkflow",
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "subworkflow unit mixin schema",
+    "type": "object",
+    "properties": {
+        "type": {
+            "const": "subworkflow"
+        }
+    }
+}
+

package/schema/workflow/unit/reduce.json

@@ -6,43 +6,9 @@
     "allOf": [
         {
             "$ref": "./base.json"
-        }
-    ],
-    "properties": {
-        "type": {
-            "const": "reduce"
-        },
-        "mapFlowchartId": {
-            "description": "corresponding map unit flowchart ID",
-            "type": "string"
         },
-        "input": {
-            "description": "input information for reduce unit",
-            "type": "array",
-            "items": {
-                "type": "object",
-                "properties": {
-                    "operation": {
-                        "description": "reduce operation, e.g. aggregate",
-                        "type": "string"
-                    },
-                    "arguments": {
-                        "description": "arguments which are passed to reduce operation function",
-                        "type": "array",
-                        "items": {
-                            "type": "string"
-                        }
-                    }
-                },
-                "required": [
-                    "operation",
-                    "arguments"
-                ]
-            }
+        {
+            "$ref": "./mixins/reduce.json"
         }
-    },
-    "required": [
-        "mapFlowchartId",
-        "input"
     ]
 }

package/schema/workflow/unit/subworkflow.json

@@ -6,11 +6,9 @@
     "allOf": [
         {
             "$ref": "./base.json"
+        },
+        {
+            "$ref": "./mixins/subworkflow.json"
         }
-    ],
-    "properties": {
-        "type": {
-            "const": "subworkflow"
-        }
-    }
+    ]
 }

package/schema/workflow/unit.json

@@ -5,29 +5,11 @@
     "type": "object",
     "oneOf": [
         {
-            "$ref": "./unit/io.json"
+            "$ref": "./unit/map.json"
         },
         {
             "$ref": "./unit/reduce.json"
         },
-        {
-            "$ref": "./unit/condition.json"
-        },
-        {
-            "$ref": "./unit/assertion.json"
-        },
-        {
-            "$ref": "./unit/execution.json"
-        },
-        {
-            "$ref": "./unit/assignment.json"
-        },
-        {
-            "$ref": "./unit/processing.json"
-        },
-        {
-            "$ref": "./unit/map.json"
-        },
         {
             "$ref": "./unit/subworkflow.json"
         }

package/schema/workflow.json

@@ -4,32 +4,36 @@
     "title": "workflow schema",
     "type": "object",
     "allOf": [
+        {
+            "$ref": "in_memory_entity/named.json"
+        },
+        {
+            "$ref": "system/metadata.json"
+        },
         {
             "$ref": "workflow/base.json"
+        },
+        {
+            "$ref": "system/description.json"
+        },
+        {
+            "$ref": "system/tags.json"
         }
     ],
     "properties": {
-        "subworkflows": {
-            "description": "Array of subworkflows. Subworkflow can be an instance of workflow to allow for nesting",
+        "workflows": {
+            "description": "Array of workflows with the same schema as the current one.",
             "type": "array",
             "items": {
-                "allOf": [
-                    {
-                        "$ref": "workflow/subworkflow.json"
-                    }
-                ]
+                "type": "object"
             }
         },
-        "units": {
-            "description": "Contains the Units of the Workflow",
-            "type": "array",
-            "items": {
-                "$ref": "workflow/unit.json"
-            }
+        "compute": {
+            "description": "compute parameters",
+            "$ref": "job/compute.json"
         }
     },
     "required": [
-        "units",
-        "subworkflows"
+        "workflows"
     ]
 }

package/src/js/esse/JSONSchemasGenerator.ts

@@ -4,8 +4,9 @@ import yaml from "js-yaml";
 import mergeAllOf from "json-schema-merge-allof";
 import path from "path";
 
+import { JSONSchemaWithPath, parseIncludeReferenceStatementsByDir } from "./server/utils";
 import { EXAMPLES_DIR, PROPERTIES_MANIFEST_PATH, SCHEMAS_DIR } from "./settings";
-import { JSONSchema, JSONSchemaWithPath, parseIncludeReferenceStatementsByDir } from "./utils";
+import type { JSONSchema } from "./utils";
 
 interface JSONSchemasGeneratorConfig {
     schemasDir: string;

package/src/js/esse/JSONSchemasInterface.ts

@@ -1,4 +1,4 @@
-import { applyPatchTree, JSONSchema } from "./utils";
+import { type JSONSchema, applyPatchTree } from "./utils";
 
 export type JSONSchemasInterfaceQuery = { [key in keyof JSONSchema]: { $regex: string } };
 

package/src/js/esse/schemaUtils.ts

@@ -1,4 +1,4 @@
-import { JSONSchema, JSONSchemaDefinition } from "./utils";
+import type { JSONSchema, JSONSchemaDefinition } from "./utils";
 
 export type MapSchema = (prop: JSONSchemaDefinition) => JSONSchemaDefinition | undefined;
 

package/src/js/esse/server/utils.ts

@@ -0,0 +1,82 @@
+/**
+ * Server-side schema parsing utilities.
+ *
+ * Use this module only in Node/server contexts where the filesystem is available.
+ * It walks directories and reads JSON schema files from disk; do not import it in
+ * browser bundles.
+ */
+// @ts-ignore
+import deref from "json-schema-deref-sync";
+import path from "path";
+
+import { JSONInclude } from "../../json_include";
+import { walkDirSync } from "../../utils/filesystem";
+import type { JSONSchema } from "../utils";
+
+export interface JSONSchemaWithPath {
+    data: JSONSchema;
+    path: string;
+}
+
+/**
+ * Resolves `include` and `$ref` statements.
+ * @param filePath {String} file to parse.
+ */
+export function parseIncludeReferenceStatements(filePath: string): JSONSchema {
+    const jsonResolver = new JSONInclude();
+    const parsed = jsonResolver.parseIncludeStatements(filePath) as JSONSchema;
+    const dirPath = path.dirname(filePath);
+    // Store the original $id before dereferencing
+    const originalId = parsed.$id;
+    let dereferenced = deref(parsed, { baseFolder: dirPath, removeIds: true });
+    // handle circular references and use non-dereferenced source
+    if (dereferenced instanceof Error && dereferenced.message === "Circular self reference") {
+        dereferenced = parsed;
+    }
+    // Restore the original $id after dereferencing
+    if (originalId) {
+        dereferenced.$id = originalId;
+    }
+    return dereferenced;
+}
+
+export function parseIncludeReferenceStatementsByDir(
+    dirPath: string,
+    wrapInDataAndPath: true,
+): JSONSchemaWithPath[];
+
+export function parseIncludeReferenceStatementsByDir(
+    dirPath: string,
+    wrapInDataAndPath?: false,
+): JSONSchema[];
+
+/**
+ * Resolves `include` and `$ref` statements for all the JSON files inside a given directory.
+ * @param dirPath directory to parse.
+ */
+export function parseIncludeReferenceStatementsByDir(dirPath: string, wrapInDataAndPath = false) {
+    const schemas: JSONSchema[] = [];
+    const schemasWithPath: JSONSchemaWithPath[] = [];
+    const topDir = path.resolve(__dirname, "../../../../");
+
+    walkDirSync(dirPath, (filePath) => {
+        if (filePath.endsWith(".json")) {
+            const config = parseIncludeReferenceStatements(filePath);
+            if (wrapInDataAndPath) {
+                const _path = path.join(
+                    // remove leading slashes and "example" from path
+                    path
+                        .dirname(filePath)
+                        .replace(path.join(topDir, "example"), "")
+                        .replace(/^\/+/, ""),
+                    path.basename(filePath).replace(".json", ""),
+                );
+                schemasWithPath.push({ data: config, path: _path });
+            } else {
+                schemas.push(config);
+            }
+        }
+    });
+
+    return wrapInDataAndPath ? schemasWithPath : schemas;
+}

package/src/js/esse/utils.ts

@@ -1,83 +1,9 @@
 import { JSONSchema7, JSONSchema7Definition } from "json-schema";
-// @ts-ignore
-import deref from "json-schema-deref-sync";
-import path from "path";
-
-import { JSONInclude } from "../json_include";
-import { walkDirSync } from "../utils/filesystem";
 
 export type JSONSchema = JSONSchema7;
 
 export type JSONSchemaDefinition = JSONSchema7Definition;
 
-/**
- * Resolves `include` and `$ref` statements.
- * @param filePath {String} file to parse.
- */
-export function parseIncludeReferenceStatements(filePath: string): JSONSchema {
-    const jsonResolver = new JSONInclude();
-    const parsed = jsonResolver.parseIncludeStatements(filePath) as JSONSchema;
-    const dirPath = path.dirname(filePath);
-    // Store the original $id before dereferencing
-    const originalId = parsed.$id;
-    let dereferenced = deref(parsed, { baseFolder: dirPath, removeIds: true });
-    // handle circular references and use non-dereferenced source
-    if (dereferenced instanceof Error && dereferenced.message === "Circular self reference") {
-        dereferenced = parsed;
-    }
-    // Restore the original $id after dereferencing
-    if (originalId) {
-        dereferenced.$id = originalId;
-    }
-    return dereferenced;
-}
-
-export interface JSONSchemaWithPath {
-    data: JSONSchema;
-    path: string;
-}
-
-export function parseIncludeReferenceStatementsByDir(
-    dirPath: string,
-    wrapInDataAndPath: true,
-): JSONSchemaWithPath[];
-
-export function parseIncludeReferenceStatementsByDir(
-    dirPath: string,
-    wrapInDataAndPath?: false,
-): JSONSchema[];
-
-/**
- * Resolves `include` and `$ref` statements for all the JSON files inside a given directory.
- * @param dirPath directory to parse.
- */
-export function parseIncludeReferenceStatementsByDir(dirPath: string, wrapInDataAndPath = false) {
-    const schemas: JSONSchema[] = [];
-    const schemasWithPath: JSONSchemaWithPath[] = [];
-    const topDir = path.resolve(__dirname, "../../../");
-
-    walkDirSync(dirPath, (filePath) => {
-        if (filePath.endsWith(".json")) {
-            const config = parseIncludeReferenceStatements(filePath);
-            if (wrapInDataAndPath) {
-                const _path = path.join(
-                    // remove leading slashes and "example" from path
-                    path
-                        .dirname(filePath)
-                        .replace(path.join(topDir, "example"), "")
-                        .replace(/^\/+/, ""),
-                    path.basename(filePath).replace(".json", ""),
-                );
-                schemasWithPath.push({ data: config, path: _path });
-            } else {
-                schemas.push(config);
-            }
-        }
-    });
-
-    return wrapInDataAndPath ? schemasWithPath : schemas;
-}
-
 function isPlainObject(value: unknown): value is Record<string, unknown> {
     return value !== null && typeof value === "object" && !Array.isArray(value);
 }

package/src/js/scripts/compileTs.ts

@@ -3,7 +3,7 @@ import fs from "fs";
 import { compile } from "json-schema-to-typescript";
 
 import { mapObjectDeep } from "../esse/schemaUtils";
-import { JSONSchema } from "../esse/utils";
+import type { JSONSchema } from "../esse/utils";
 import { walkDir } from "../utils/filesystem";
 
 /**

package/src/js/scripts/setSchemaIds.ts

@@ -15,6 +15,7 @@ export default function setSchemaIds(schemaDir: string) {
         }
 
         const fileContents = fs.readFileSync(filePath);
+        console.log(filePath);
         // eslint-disable-next-line @typescript-eslint/no-unused-vars
         const { $id, ...schema } = JSON.parse(fileContents.toString());
         const schemaId = filePath.replace(schemaDir, "").replace(".json", "").replace(/_/g, "-");

package/src/js/utils/ajv.ts

@@ -4,7 +4,7 @@ import addFormats from "ajv-formats";
 
 import { mapObjectDeep } from "../esse/schemaUtils";
 import { AnyObject } from "../esse/types";
-import { JSONSchema } from "../esse/utils";
+import type { JSONSchema } from "../esse/utils";
 
 function addAdditionalPropertiesToSchema(schema: JSONSchema, additionalProperties = false) {
     return mapObjectDeep(schema, (object) => {

package/dist/js/example/workflow/unit/io/db.json

@@ -1,15 +0,0 @@
-{
-    "features": [
-        "elemental_ratio:Si",
-        "elemental_ratio:Ge",
-        "ionization_potential:Ge",
-        "ionization_potential:Si"
-    ],
-    "ids": [
-        "KuAsBRwofzGfHPWiT"
-    ],
-    "targets": [
-        "band_gaps:indirect",
-        "band_gaps:direct"
-    ]
-}

package/dist/js/example/workflow/unit/processing.json

@@ -1,13 +0,0 @@
-{
-    "flowchartId": "data_transformation_manipulation",
-    "inputData": {
-        "cleanMissingData": true,
-        "removeDuplicateRows": true,
-        "replaceNoneValuesWith": 0
-    },
-    "name": "data_transformation_manipulation",
-    "operation": "data_transformation",
-    "operationType": "manipulation",
-    "status": "idle",
-    "type": "processing"
-}

package/dist/js/schema/workflow/unit/io/db.json

@@ -1,40 +0,0 @@
-{
-    "$id": "workflow/unit/io/db",
-    "$schema": "http://json-schema.org/draft-07/schema#",
-    "title": "data IO database input/output schema",
-    "type": "object",
-    "oneOf": [
-        {
-            "properties": {
-                "ids": {
-                    "description": "IDs of item to retrieve from db",
-                    "type": "array",
-                    "items": {
-                        "type": "string"
-                    }
-                }
-            },
-            "required": [
-                "ids"
-            ],
-            "additionalProperties": true
-        },
-        {
-            "properties": {
-                "collection": {
-                    "description": "db collection name",
-                    "type": "string"
-                },
-                "draft": {
-                    "description": "whether the result should be saved as draft",
-                    "type": "boolean"
-                }
-            },
-            "required": [
-                "collection",
-                "draft"
-            ],
-            "additionalProperties": true
-        }
-    ]
-}