@finos/legend-graph 1.2.0 → 2.0.0

package/lib/models/protocols/pure/v1/V1_PureGraphManager.js

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 import { flow, flowResult, makeObservable, runInAction } from 'mobx';
-import { GRAPH_MANAGER_LOG_EVENT } from '../../../../graphManager/GraphManagerLogEvent';
+import { GRAPH_MANAGER_EVENT } from '../../../../graphManager/GraphManagerEvent';
 import { CORE_PURE_PATH, ELEMENT_PATH_DELIMITER, ROOT_PACKAGE_NAME, SOURCE_INFORMATION_KEY, } from '../../../../MetaModelConst';
-import { TracerService, LogEvent, getClass, guaranteeNonNullable, UnsupportedOperationError, recursiveOmit, assertTrue, assertErrorThrown, promisify, } from '@finos/legend-shared';
+import { TracerService, LogEvent, getClass, guaranteeNonNullable, UnsupportedOperationError, recursiveOmit, assertTrue, assertErrorThrown, promisify, StopWatch, } from '@finos/legend-shared';
 import { AbstractPureGraphManager, } from '../../../../graphManager/AbstractPureGraphManager';
 import { PureModel, } from '../../../../graph/PureModel';
 import { RawLambda } from '../../../metamodels/pure/rawValueSpecification/RawLambda';
@@ -78,6 +78,8 @@ import { V1_buildExecutionResult } from './engine/V1_ExecutionHelper';
 import { ENTITY_PATH_DELIMITER, } from '@finos/legend-model-storage';
 import { DependencyGraphBuilderError, GraphBuilderError, SystemGraphBuilderError, } from '../../../../graphManager/GraphManagerUtils';
 import { PackageableElementReference } from '../../../metamodels/pure/packageableElements/PackageableElementReference';
+import { V1_ExternalFormatModelGenerationInput } from './engine/externalFormat/V1_ExternalFormatModelGeneration';
+import { GraphBuilderReport } from '../../../../graphManager/GraphBuilderReport';
 const V1_FUNCTION_SUFFIX_MULTIPLICITY_INFINITE = 'MANY';
 const getMultiplicitySuffix = (multiplicity) => {
     if (multiplicity.lowerBound === multiplicity.upperBound) {
@@ -114,7 +116,16 @@ class V1_PureModelContextDataIndex {
     generationSpecifications = [];
     otherElementsByBuilder = new Map();
 }
-const indexPureModelContextData = (data, extensions) => {
+const mergePureModelContextData = (...data) => {
+    const mergedData = new V1_PureModelContextData();
+    for (const _data of data) {
+        mergedData.elements = mergedData.elements.concat(_data.elements);
+        mergedData.serializer = _data.serializer ?? mergedData.serializer;
+        mergedData.origin = _data.origin ?? mergedData.origin;
+    }
+    return mergedData;
+};
+const indexPureModelContextData = (report, data, extensions) => {
     const index = new V1_PureModelContextDataIndex();
     index.elements = data.elements;
     const otherElementsByClass = new Map();
@@ -180,6 +191,37 @@ const indexPureModelContextData = (data, extensions) => {
         const builder = extensions.getExtraBuilderForProtocolClassOrThrow(_class);
         index.otherElementsByBuilder.set(builder, (index.otherElementsByBuilder.get(builder) ?? []).concat(elements));
     });
+    // report
+    report.elementCount.total = report.elementCount.total + index.elements.length;
+    report.elementCount.other =
+        (report.elementCount.other ?? 0) +
+            otherElementsByClass.size +
+            index.fileGenerations.length +
+            index.generationSpecifications.length;
+    report.elementCount.sectionIndex =
+        (report.elementCount.sectionIndex ?? 0) + index.sectionIndices.length;
+    report.elementCount.association =
+        (report.elementCount.association ?? 0) + index.associations.length;
+    report.elementCount.class =
+        (report.elementCount.class ?? 0) + index.classes.length;
+    report.elementCount.enumeration =
+        (report.elementCount.enumeration ?? 0) + index.enumerations.length;
+    report.elementCount.function =
+        (report.elementCount.function ?? 0) + index.functions.length;
+    report.elementCount.profile =
+        (report.elementCount.profile ?? 0) + index.profiles.length;
+    report.elementCount.measure =
+        (report.elementCount.measure ?? 0) + index.measures.length;
+    report.elementCount.store =
+        (report.elementCount.store ?? 0) + index.stores.length;
+    report.elementCount.mapping =
+        (report.elementCount.mapping ?? 0) + index.mappings.length;
+    report.elementCount.connection =
+        (report.elementCount.connection ?? 0) + index.connections.length;
+    report.elementCount.runtime =
+        (report.elementCount.runtime ?? 0) + index.runtimes.length;
+    report.elementCount.service =
+        (report.elementCount.service ?? 0) + index.services.length;
     return index;
 };
 export class V1_PureGraphManager extends AbstractPureGraphManager {
@@ -192,6 +234,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
             buildSystem: flow,
             buildDependencies: flow,
             buildGraph: flow,
+            buildGraphFromInputs: flow,
             buildGenerations: flow,
             initializeAndIndexElements: flow,
             postProcess: flow,
@@ -199,11 +242,11 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
             buildStores: flow,
             buildMappings: flow,
             buildConnectionsAndRuntimes: flow,
-            buildSectionIndex: flow,
+            buildSectionIndices: flow,
             buildOtherElements: flow,
             buildServices: flow,
             buildFileGenerations: flow,
-            buildGenerationSpecificationss: flow,
+            buildGenerationSpecifications: flow,
         });
         // setup plugins
         this.extensions = new V1_GraphBuilderExtensions(this.pluginManager.getPureProtocolProcessorPlugins());
@@ -226,204 +269,121 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     }
     // --------------------------------------------- Graph Builder ---------------------------------------------
     *buildSystem(coreModel, systemModel, options) {
-        const startTime = Date.now();
+        const stopWatch = new StopWatch();
+        const report = new GraphBuilderReport();
+        const graphBuilderState = systemModel.buildState;
+        graphBuilderState.reset();
         // Create a dummy graph for system processing. This is to ensure system model does not depend on the main graph
         const graph = new PureModel(coreModel, systemModel, this.pluginManager.getPureGraphPlugins());
         try {
-            const systemData = new V1_PureModelContextData();
-            yield V1_entitiesToPureModelContextData(
-            /**
-             * Get all system entities.
-             *
-             * NOTE: right now, we are doing extra work here: JSON -> protocol -> entities -> protocol, since we are
-             * expecting to get these models from some a remote SDLC project in the future.
-             */
-            V1_deserializePureModelContextData(V1_CORE_SYSTEM_MODELS)
-                .elements.concat(this.pluginManager
+            // deserialize
+            graphBuilderState.setMessage(`Collecting and deserializing elements...`);
+            const systemData = mergePureModelContextData(V1_deserializePureModelContextData(V1_CORE_SYSTEM_MODELS), ...this.pluginManager
                 .getPureProtocolProcessorPlugins()
                 .flatMap((plugin) => plugin.V1_getExtraSystemModels?.() ?? [])
-                .flatMap((modelContextData) => V1_deserializePureModelContextData(modelContextData).elements))
-                .map((element) => this.elementProtocolToEntity(element)), systemData, this.pluginManager.getPureProtocolProcessorPlugins());
-            const systemGraphBuilderInput = [
+                .map((modelContextData) => V1_deserializePureModelContextData(modelContextData)));
+            stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_ELEMENTS_DESERIALIZED);
+            // prepare build inputs
+            const buildInputs = [
                 {
                     model: systemModel,
-                    data: indexPureModelContextData(systemData, this.extensions),
+                    data: indexPureModelContextData(report, systemData, this.extensions),
                 },
             ];
-            yield flowResult(this.initializeAndIndexElements(graph, systemGraphBuilderInput));
-            // NOTE: right now we only have profile and enumeration for system, we might need to generalize this step in the future
-            yield flowResult(this.buildTypes(graph, systemGraphBuilderInput));
-            yield flowResult(this.buildOtherElements(graph, systemGraphBuilderInput));
-            yield flowResult(this.postProcess(graph, systemGraphBuilderInput));
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_SYSTEM_BUILT), Date.now() - startTime, 'ms', `[profile: ${systemModel.ownProfiles.length}, enumeration: ${systemModel.ownEnumerations.length}]`);
-            }
-            systemModel.buildState.pass();
+            // build
+            yield flowResult(this.buildGraphFromInputs(graph, buildInputs, report, stopWatch, graphBuilderState, options));
+            graphBuilderState.pass();
+            report.timings = {
+                ...Object.fromEntries(stopWatch.records),
+                [GRAPH_MANAGER_EVENT.GRAPH_BUILDER_COMPLETED]: stopWatch.elapsed,
+            };
+            return report;
         }
         catch (error) {
             assertErrorThrown(error);
-            systemModel.buildState.fail();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_FAILURE), '[ERROR]', Date.now() - startTime, 'ms');
-            }
+            graphBuilderState.fail();
+            this.log.error(LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE));
             throw new SystemGraphBuilderError(error);
         }
+        finally {
+            graphBuilderState.setMessage(undefined);
+        }
     }
     *buildDependencies(coreModel, systemModel, dependencyManager, dependencyEntitiesMap, options) {
-        const startTime = Date.now();
-        dependencyManager.buildState.reset();
+        const stopWatch = new StopWatch();
+        const report = new GraphBuilderReport();
+        const graphBuilderState = dependencyManager.buildState;
+        graphBuilderState.reset();
         // Create a dummy graph for system processing. This is to ensure dependency models do not depend on the main graph
         const graph = new PureModel(coreModel, systemModel, this.pluginManager.getPureGraphPlugins());
         graph.setDependencyManager(dependencyManager);
         try {
             dependencyManager.initialize(dependencyEntitiesMap);
-            // Parse/Build Data
+            // deserialize
+            graphBuilderState.setMessage(`Partitioning and deserializing elements...`);
             const dependencyDataMap = new Map();
             yield Promise.all(Array.from(dependencyEntitiesMap.entries()).map(([dependencyKey, entities]) => {
                 const projectModelData = new V1_PureModelContextData();
                 dependencyDataMap.set(dependencyKey, projectModelData);
                 return V1_entitiesToPureModelContextData(entities, projectModelData, this.pluginManager.getPureProtocolProcessorPlugins());
             }));
-            const preprocessingFinishedTime = Date.now();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_DEPENDENCIES_PREPROCESSED), preprocessingFinishedTime - startTime, 'ms');
-            }
-            const graphBuilderInput = Array.from(dependencyDataMap.entries()).map(([dependencyKey, dependencyData]) => ({
-                data: indexPureModelContextData(dependencyData, this.extensions),
+            stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_ELEMENTS_DESERIALIZED);
+            // prepare build inputs
+            const buildInputs = Array.from(dependencyDataMap.entries()).map(([dependencyKey, dependencyData]) => ({
+                data: indexPureModelContextData(report, dependencyData, this.extensions),
                 model: graph.dependencyManager.getModel(dependencyKey),
             }));
-            yield flowResult(this.initializeAndIndexElements(graph, graphBuilderInput, options));
-            // NOTE: we might need to process sectionIndex if we support unresolved element paths in dependencies
-            yield flowResult(this.buildTypes(graph, graphBuilderInput, options));
-            yield flowResult(this.buildStores(graph, graphBuilderInput, options));
-            yield flowResult(this.buildMappings(graph, graphBuilderInput, options));
-            yield flowResult(this.buildConnectionsAndRuntimes(graph, graphBuilderInput, options));
-            yield flowResult(this.buildServices(graph, graphBuilderInput, options));
-            yield flowResult(this.buildFileGenerations(graph, graphBuilderInput, options));
-            yield flowResult(this.buildGenerationSpecificationss(graph, graphBuilderInput, options));
-            yield flowResult(this.buildOtherElements(graph, graphBuilderInput, options));
-            yield flowResult(this.postProcess(graph, graphBuilderInput));
-            const processingFinishedTime = Date.now();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_DEPENDENCIES_PROCESSED), processingFinishedTime - preprocessingFinishedTime, 'ms');
-            }
-            dependencyManager.buildState.pass();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_DEPENDENCIES_BUILT), '[TOTAL]', Date.now() - startTime, 'ms');
-            }
+            // build
+            yield flowResult(this.buildGraphFromInputs(graph, buildInputs, report, stopWatch, graphBuilderState, options));
+            graphBuilderState.pass();
+            report.otherStats.projectCount = dependencyEntitiesMap.size;
+            report.timings = {
+                ...Object.fromEntries(stopWatch.records),
+                [GRAPH_MANAGER_EVENT.GRAPH_BUILDER_COMPLETED]: stopWatch.elapsed,
+            };
+            return report;
         }
         catch (error) {
             assertErrorThrown(error);
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_FAILURE), '[ERROR]', Date.now() - startTime, 'ms');
-            }
-            dependencyManager.buildState.fail();
+            this.log.error(LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE));
+            graphBuilderState.fail();
             throw new DependencyGraphBuilderError(error);
         }
+        finally {
+            graphBuilderState.setMessage(undefined);
+        }
     }
     *buildGraph(graph, entities, options) {
-        let stepStartTime = Date.now();
-        let stepFinishedTime;
-        const startTime = stepStartTime;
+        const stopWatch = new StopWatch();
+        const report = new GraphBuilderReport();
+        const graphBuilderState = graph.buildState;
+        graphBuilderState.reset();
         try {
-            // Parse/Build Data
+            // deserialize
+            graphBuilderState.setMessage(`Deserializing elements...`);
             const data = new V1_PureModelContextData();
             yield V1_entitiesToPureModelContextData(entities, data, this.pluginManager.getPureProtocolProcessorPlugins());
-            const graphBuilderInput = [
+            stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_ELEMENTS_DESERIALIZED);
+            // prepare build inputs
+            const buildInputs = [
                 {
                     model: graph,
-                    data: indexPureModelContextData(data, this.extensions),
+                    data: indexPureModelContextData(report, data, this.extensions),
                 },
             ];
-            stepFinishedTime = Date.now();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_DATA_MODEL_PARSED), stepFinishedTime - stepStartTime, 'ms');
-            }
-            stepStartTime = stepFinishedTime;
-            yield flowResult(this.initializeAndIndexElements(graph, graphBuilderInput, options));
-            stepFinishedTime = Date.now();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_ELEMENTS_INITIALIZED_AND_INDEXED), stepFinishedTime - stepStartTime, 'ms', `[element: ${data.elements.length}]`);
-            }
-            stepStartTime = stepFinishedTime;
-            // Section index
-            yield flowResult(this.buildSectionIndex(graph, graphBuilderInput, options));
-            stepFinishedTime = Date.now();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_SECTION_INDICES_BUILT), stepFinishedTime - stepStartTime, 'ms', `[sectionIndex: ${graph.ownSectionIndices.length}]`);
-            }
-            stepStartTime = stepFinishedTime;
-            // Types
-            yield flowResult(this.buildTypes(graph, graphBuilderInput, options));
-            stepFinishedTime = Date.now();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_DOMAIN_MODELS_BUILT), stepFinishedTime - stepStartTime, 'ms', `[class: ${graph.ownClasses.length}, enumeration: ${graph.ownEnumerations.length}, association: ${graph.ownAssociations.length}, profile: ${graph.ownProfiles.length}, functions: ${graph.ownFunctions.length}]`);
-            }
-            stepStartTime = stepFinishedTime;
-            // Stores
-            yield flowResult(this.buildStores(graph, graphBuilderInput, options));
-            stepFinishedTime = Date.now();
-            // TODO: we might want to detail out the number of stores by type
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_STORES_BUILT), stepFinishedTime - stepStartTime, 'ms', `[store: ${graph.ownStores.length}]`);
-            }
-            stepStartTime = stepFinishedTime;
-            // Mappings
-            yield flowResult(this.buildMappings(graph, graphBuilderInput, options));
-            stepFinishedTime = Date.now();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_MAPPINGS_BUILT), stepFinishedTime - stepStartTime, 'ms', `[mapping: ${graph.ownMappings.length}]`);
-            }
-            stepStartTime = stepFinishedTime;
-            // Connections and runtimes
-            yield flowResult(this.buildConnectionsAndRuntimes(graph, graphBuilderInput, options));
-            stepFinishedTime = Date.now();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_CONNECTIONS_BUILT), stepFinishedTime - stepStartTime, 'ms', `[connection: ${graph.ownConnections.length}]`);
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_RUNTIMES_BUILT), stepFinishedTime - stepStartTime, 'ms', `[runtime: ${graph.ownRuntimes.length}]`);
-            }
-            stepStartTime = stepFinishedTime;
-            // Services
-            yield flowResult(this.buildServices(graph, graphBuilderInput, options));
-            stepFinishedTime = Date.now();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_SERVICES_BUILT), stepFinishedTime - stepStartTime, 'ms', `[service: ${graph.ownServices.length}]`);
-            }
-            stepStartTime = stepFinishedTime;
-            // File Generation
-            yield flowResult(this.buildFileGenerations(graph, graphBuilderInput, options));
-            stepFinishedTime = Date.now();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_FILE_GENERATIONS_BUILT), stepFinishedTime - stepStartTime, 'ms', `[file-generation: ${graph.ownFileGenerations.length}]`);
-            }
-            stepStartTime = stepFinishedTime;
-            // Generation Specifications (tree)
-            yield flowResult(this.buildGenerationSpecificationss(graph, graphBuilderInput, options));
-            stepFinishedTime = Date.now();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_GENERATION_TREE_BUILT), stepFinishedTime - stepStartTime, 'ms', `[generation-specification: ${graph.ownGenerationSpecifications.length}]`);
-            }
-            stepStartTime = stepFinishedTime;
-            // Other elements
-            yield flowResult(this.buildOtherElements(graph, graphBuilderInput, options));
-            stepFinishedTime = Date.now();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_OTHER_ELEMENTS_BUILT), stepFinishedTime - stepStartTime, 'ms');
-            }
-            yield flowResult(this.postProcess(graph, graphBuilderInput, {
-                TEMPORARY__keepSectionIndex: options?.TEMPORARY__keepSectionIndex,
-            }));
-            graph.buildState.pass();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_COMPLETED), '[TOTAL]', Date.now() - startTime, 'ms');
-            }
+            // build
+            yield flowResult(this.buildGraphFromInputs(graph, buildInputs, report, stopWatch, graphBuilderState, options));
+            graphBuilderState.pass();
+            report.timings = {
+                ...Object.fromEntries(stopWatch.records),
+                [GRAPH_MANAGER_EVENT.GRAPH_BUILDER_COMPLETED]: stopWatch.elapsed,
+            };
+            return report;
         }
         catch (error) {
             assertErrorThrown(error);
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_FAILURE), '[ERROR]', Date.now() - startTime, 'ms');
-            }
-            graph.buildState.fail();
+            this.log.error(LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE));
+            graphBuilderState.fail();
             /**
              * Wrap all error with `GraphBuilderError`, as we throw a lot of assertion error in the graph builder
              * But we might want to rethink this decision in the future and throw appropriate type of error
@@ -432,47 +392,46 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
                 ? error
                 : new GraphBuilderError(error);
         }
+        finally {
+            graphBuilderState.setMessage(undefined);
+        }
     }
     *buildGenerations(graph, generatedEntities, options) {
-        const stepStartTime = Date.now();
+        const stopWatch = new StopWatch();
+        const report = new GraphBuilderReport();
         const generatedModel = graph.generationModel;
-        generatedModel.buildState.reset();
+        const graphBuilderState = generatedModel.buildState;
+        graphBuilderState.reset();
         try {
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_DATA_MODEL_PARSED));
-            }
+            // deserialize
+            graphBuilderState.setMessage(`Deserializing elements...`);
             const generatedDataMap = new Map();
             yield Promise.all(Array.from(generatedEntities.entries()).map(([generationParentPath, entities]) => {
                 const generatedData = new V1_PureModelContextData();
                 generatedDataMap.set(generationParentPath, generatedData);
                 return V1_entitiesToPureModelContextData(entities, generatedData, this.pluginManager.getPureProtocolProcessorPlugins());
             }));
-            const generationGraphBuilderInput = Array.from(generatedDataMap.entries()).map(([generationParentPath, generatedData]) => ({
+            stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_ELEMENTS_DESERIALIZED);
+            // prepare build inputs
+            const buildInputs = Array.from(generatedDataMap.entries()).map(([generationParentPath, generatedData]) => ({
                 parentElementPath: generationParentPath,
-                data: indexPureModelContextData(generatedData, this.extensions),
+                data: indexPureModelContextData(report, generatedData, this.extensions),
                 model: generatedModel,
             }));
-            yield flowResult(this.initializeAndIndexElements(graph, generationGraphBuilderInput));
-            yield flowResult(this.buildTypes(graph, generationGraphBuilderInput));
-            yield flowResult(this.buildStores(graph, generationGraphBuilderInput));
-            yield flowResult(this.buildMappings(graph, generationGraphBuilderInput));
-            yield flowResult(this.buildConnectionsAndRuntimes(graph, generationGraphBuilderInput));
-            yield flowResult(this.buildServices(graph, generationGraphBuilderInput));
-            yield flowResult(this.buildFileGenerations(graph, generationGraphBuilderInput));
-            yield flowResult(this.buildGenerationSpecificationss(graph, generationGraphBuilderInput));
-            yield flowResult(this.buildOtherElements(graph, generationGraphBuilderInput));
-            yield flowResult(this.postProcess(graph, generationGraphBuilderInput));
-            generatedModel.buildState.pass();
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_GENERATIONS_BUILT), Date.now() - stepStartTime, `${graph.generationModel.allOwnElements.length} generated elements processed`, 'ms');
-            }
+            // build
+            yield flowResult(this.buildGraphFromInputs(graph, buildInputs, report, stopWatch, graphBuilderState, options));
+            graphBuilderState.pass();
+            report.otherStats.generationCount = generatedDataMap.size;
+            report.timings = {
+                ...Object.fromEntries(stopWatch.records),
+                [GRAPH_MANAGER_EVENT.GRAPH_BUILDER_COMPLETED]: stopWatch.elapsed,
+            };
+            return report;
         }
         catch (error) {
             assertErrorThrown(error);
-            if (!options?.quiet) {
-                this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_BUILDER_FAILURE), Date.now() - stepStartTime, 'ms');
-            }
-            generatedModel.buildState.fail();
+            this.log.error(LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE));
+            graphBuilderState.fail();
             /**
              * Wrap all error with `GraphBuilderError`, as we throw a lot of assertion error in the graph builder
              * But we might want to rethink this decision in the future and throw appropriate type of error
@@ -481,6 +440,51 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
                 ? error
                 : new GraphBuilderError(error);
         }
+        finally {
+            graphBuilderState.setMessage(undefined);
+        }
+    }
+    *buildGraphFromInputs(graph, inputs, report, stopWatch, graphBuilderState, options) {
+        // index
+        graphBuilderState.setMessage(`Indexing ${report.elementCount.total} elements...`);
+        yield flowResult(this.initializeAndIndexElements(graph, inputs, options));
+        stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_ELEMENTS_INDEXED);
+        // build section index
+        graphBuilderState.setMessage(`Building section indices...`);
+        yield flowResult(this.buildSectionIndices(graph, inputs, options));
+        stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_SECTION_INDICES_BUILT);
+        // build types
+        graphBuilderState.setMessage(`Building domain models...`);
+        yield flowResult(this.buildTypes(graph, inputs, options));
+        stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_DOMAIN_MODELS_BUILT);
+        // build stores
+        graphBuilderState.setMessage(`Building stores...`);
+        yield flowResult(this.buildStores(graph, inputs, options));
+        stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_STORES_BUILT);
+        // build mappings
+        graphBuilderState.setMessage(`Building mappings...`);
+        yield flowResult(this.buildMappings(graph, inputs, options));
+        stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_MAPPINGS_BUILT);
+        // build connections and runtimes
+        graphBuilderState.setMessage(`Building connections and runtimes...`);
+        yield flowResult(this.buildConnectionsAndRuntimes(graph, inputs, options));
+        stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_CONNECTIONS_AND_RUNTIMES_BUILT);
+        // build services
+        graphBuilderState.setMessage(`Building services...`);
+        yield flowResult(this.buildServices(graph, inputs, options));
+        stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_SERVICES_BUILT);
+        // build other elements
+        graphBuilderState.setMessage(`Building other elements...`);
+        yield flowResult(this.buildFileGenerations(graph, inputs, options));
+        yield flowResult(this.buildGenerationSpecifications(graph, inputs, options));
+        yield flowResult(this.buildOtherElements(graph, inputs, options));
+        stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_OTHER_ELEMENTS_BUILT);
+        // post-process
+        graphBuilderState.setMessage(`Post-processing graph...`);
+        yield flowResult(this.postProcess(graph, inputs, {
+            TEMPORARY__keepSectionIndex: options?.TEMPORARY__keepSectionIndex,
+        }));
+        stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_POST_PROCESSED);
     }
     getBuilderContext(graph, currentSubGraph, element, options) {
         return new V1_GraphBuilderContextBuilder(graph, currentSubGraph, this.extensions, this.log, options)
@@ -559,10 +563,10 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     *buildFileGenerations(graph, inputs, options) {
         yield Promise.all(inputs.flatMap((input) => input.data.fileGenerations.map((element) => this.visitWithErrorHandling(element, new V1_ProtocolToMetaModelGraphSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
     }
-    *buildGenerationSpecificationss(graph, inputs, options) {
+    *buildGenerationSpecifications(graph, inputs, options) {
         yield Promise.all(inputs.flatMap((input) => input.data.generationSpecifications.map((element) => this.visitWithErrorHandling(element, new V1_ProtocolToMetaModelGraphSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
     }
-    *buildSectionIndex(graph, inputs, options) {
+    *buildSectionIndices(graph, inputs, options) {
         yield Promise.all(inputs.flatMap((input) => input.data.sectionIndices.map((element) => this.visitWithErrorHandling(element, new V1_ProtocolToMetaModelGraphSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
     }
     *buildOtherElements(graph, inputs, options) {
@@ -589,13 +593,13 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
         const startTime = Date.now();
         const graphData = this.graphToPureModelContextData(graph);
         const grammarToJson = await this.engine.pureModelContextDataToPureCode(graphData);
-        this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_MODEL_TO_GRAMMAR_TRANSFORMED), Date.now() - startTime, 'ms');
+        this.log.info(LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_MODEL_TO_GRAMMAR_TRANSFORMED), Date.now() - startTime, 'ms');
         return grammarToJson;
     }
     async entitiesToPureCode(entities) {
         const startTime = Date.now();
         const grammarToJson = await this.engine.pureModelContextDataToPureCode(await this.entitiesToPureModelContextData(entities));
-        this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_MODEL_TO_GRAMMAR_TRANSFORMED), Date.now() - startTime, 'ms');
+        this.log.info(LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_MODEL_TO_GRAMMAR_TRANSFORMED), Date.now() - startTime, 'ms');
         return grammarToJson;
     }
     async pureCodeToEntities(code, options) {
@@ -687,6 +691,19 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     serializeRawValueSpecification(metamodel) {
         return V1_serializeRawValueSpecification(metamodel.accept_RawValueSpecificationVisitor(new V1_RawValueSpecificationTransformer(new V1_GraphTransformerContextBuilder(this.pluginManager.getPureProtocolProcessorPlugins()).build())));
     }
+    // ------------------------------------------- External Format --------------------------------
+    getAvailableExternalFormatsDescriptions() {
+        return this.engine.getAvailableExternalFormatsDescriptions();
+    }
+    generateModelFromExternalFormat(configurationProperties, graph) {
+        const config = {};
+        configurationProperties.forEach((property) => {
+            config[property.name] = property.value;
+        });
+        const model = this.getFullGraphModelData(graph);
+        const input = new V1_ExternalFormatModelGenerationInput(model, config);
+        return this.engine.generateModel(input);
+    }
     // ------------------------------------------- Import -------------------------------------------
     getAvailableImportConfigurationDescriptions() {
         return this.engine.getAvailableImportConfigurationDescriptions();
@@ -711,7 +728,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     async entitiesToPureProtocolText(entities) {
         return JSON.stringify(V1_serializePureModelContext(await this.entitiesToPureModelContextData(entities)), undefined, this.engine.config.tabSize);
     }
-    pureProtocolToEntities = (protocol) => {
+    pureProtocolTextToEntities = (protocol) => {
         const graphData = V1_deserializePureModelContextData(JSON.parse(protocol));
         return this.pureModelContextDataToEntities(graphData);
     };
@@ -781,8 +798,8 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     // --------------------------------------------- Service ---------------------------------------------
     async runServiceTests(service, graph) {
         const protocolGraph = this.getFullGraphModelData(graph);
-        const targetService = guaranteeNonNullable(protocolGraph
-            .getElementsOfType(V1_Service)
+        const targetService = guaranteeNonNullable(protocolGraph.elements
+            .filter((element) => element instanceof V1_Service)
             .find((element) => element.path === service.path), `Can't run service test: service '${service.path}' not found`);
         protocolGraph.elements = protocolGraph.elements.filter((element) => !(element instanceof V1_Service));
         protocolGraph.elements.push(targetService);
@@ -1019,7 +1036,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
         graphData.elements = graph.allOwnElements.map((element) => this.elementToProtocol(element, {
             keepSourceInformation: options?.keepSourceInformation,
         }));
-        this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_META_MODEL_TO_PROTOCOL_TRANSFORMED), Date.now() - startTime, 'ms');
+        this.log.info(LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_META_MODEL_TO_PROTOCOL_TRANSFORMED), Date.now() - startTime, 'ms');
         return graphData;
     };
     getGraphCompileContext = (graph) => {
@@ -1031,7 +1048,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
             ...dependencyManager.allElements,
             ...generatedModel.allOwnElements,
         ].map((element) => this.elementToProtocol(element));
-        this.log.info(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_COMPILE_CONTEXT_COLLECTED), Date.now() - startTime, 'ms');
+        this.log.info(LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_COMPILE_CONTEXT_COLLECTED), Date.now() - startTime, 'ms');
         return graphData;
     };
     // --------------------------------------------- Utilities ---------------------------------------------
@@ -1129,7 +1146,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
         }
         catch (error) {
             assertErrorThrown(error);
-            this.log.warn(LogEvent.create(GRAPH_MANAGER_LOG_EVENT.GRAPH_MANAGER_FAILURE), `Can't extract assertion result`);
+            this.log.warn(LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_MANAGER_FAILURE), `Can't extract assertion result`);
             json = undefined;
         }
         if (!json) {