@finos/legend-graph 28.0.3 → 28.1.0

package/lib/graph-manager/protocol/pure/v1/V1_PureGraphManager.js

@@ -100,9 +100,10 @@ import { V1_LambdaReturnTypeInput } from './engine/compilation/V1_LambdaReturnTy
 import { MultiExecutionServiceTestResult } from '../../../../graph/metamodel/pure/packageableElements/service/MultiExecutionServiceTestResult.js';
 import { V1_ExecutionEnvironmentInstance } from './model/packageableElements/service/V1_ExecutionEnvironmentInstance.js';
 import { V1_EntitlementReportAnalyticsInput, V1_StoreEntitlementAnalysisInput, V1_buildDatasetEntitlementReport, V1_buildDatasetSpecification, V1_transformDatasetSpecification, } from './engine/analytics/V1_StoreEntitlementAnalysis.js';
-import { LegendSDLC, } from '../../../../graph/GraphDataOrigin.js';
-import { LiveGraphData, GraphDataWithOrigin, } from '../../../GraphData.js';
+import { LegendSDLC, GraphEntities, } from '../../../../graph/GraphDataOrigin.js';
+import { InMemoryGraphData, GraphDataWithOrigin, } from '../../../GraphData.js';
 import { DEPRECATED__validate_MappingTest } from '../../../action/validation/DSL_Mapping_ValidationHelper.js';
+import { V1_SERVICE_ELEMENT_PROTOCOL_TYPE } from './transformation/pureProtocol/serializationHelpers/V1_ServiceSerializationHelper.js';
 class V1_PureModelContextDataIndex {
     elements = [];
     nativeElements = [];
@@ -128,6 +129,8 @@ const mergePureModelContextData = (...data) => {
     const mergedData = new V1_PureModelContextData();
     for (const _data of data) {
         mergedData.elements = mergedData.elements.concat(_data.elements);
+        mergedData.INTERNAL__rawDependencyEntities =
+            mergedData.INTERNAL__rawDependencyEntities.concat(_data.INTERNAL__rawDependencyEntities);
         mergedData.serializer = _data.serializer ?? mergedData.serializer;
         mergedData.origin = _data.origin ?? mergedData.origin;
     }
@@ -344,6 +347,12 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
             }));
             // build
             await this.buildGraphFromInputs(graph, buildInputs, report, stopWatch, buildState, options);
+            // set dependency manager graph origin to entities
+            if (dependencyManager.origin === undefined) {
+                dependencyManager.setOrigin(new GraphEntities(Array.from(dependencyEntitiesIndex.values())
+                    .map((e) => e.entities)
+                    .flat()));
+            }
             buildState.pass();
             const totalTime = stopWatch.elapsed;
             report.timings = {
@@ -389,10 +398,6 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
             if (!options?.TEMPORARY__preserveSectionIndex) {
                 graph.TEMPORARY__deleteOwnSectionIndex();
             }
-            /**
-             * This sets the SDLC of the graph if there exists a versioned `SDLC` of the graph. This means the graph is immutable and we can easily `refetch`
-             * the graph with the provided sdlc pointer
-             */
             if (options?.origin) {
                 graph.setOrigin(options.origin);
             }
@@ -958,14 +963,10 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
         return this.pureModelContextDataToEntities(graphData);
     };
     // --------------------------------------------- Execution ---------------------------------------------
-    createExecutionInput = (graph, mapping, lambda, runtime, clientVersion, parameterValues) => this.buildExecutionInput(graph, mapping, lambda, runtime, clientVersion, new V1_ExecuteInput(), parameterValues);
-    buildExecutionInput = (graph, mapping, lambda, runtime, clientVersion, executeInput, parameterValues) => {
-        const pureModelContext = graph.origin
-            ? this.buildPureModelSDLCPointer(graph.origin, undefined)
-            : this.buildExecutionInputGraphData(graph, mapping, runtime);
-        return this.buildExecutionInputWithPureModelContext(pureModelContext, mapping, lambda, runtime, clientVersion, executeInput, parameterValues);
-    };
-    buildExecutionInputWithPureModelContext = (pureModelContext, mapping, lambda, runtime, clientVersion, executeInput, parameterValues) => {
+    createExecutionInput = (graph, mapping, lambda, runtime, clientVersion, parameterValues) => this.createExecutionInputWithPureModelContext(graph.origin
+        ? this.buildPureModelSDLCPointer(graph.origin, undefined)
+        : this.buildExecutionInputGraphData(graph), mapping, lambda, runtime, clientVersion, new V1_ExecuteInput(), parameterValues);
+    createExecutionInputWithPureModelContext = (data, mapping, lambda, runtime, clientVersion, executeInput, parameterValues) => {
         // NOTE: for execution, we usually will just assume that we send the connections embedded in the runtime value, since we don't want the user to have to create
         // packageable runtime and connection just to play with execution.
         executeInput.clientVersion = clientVersion;
@@ -974,14 +975,14 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
         executeInput.runtime = runtime
             ? V1_transformRuntime(runtime, new V1_GraphTransformerContextBuilder(this.pluginManager.getPureProtocolProcessorPlugins()).build())
             : undefined;
-        executeInput.model = pureModelContext;
+        executeInput.model = data;
         executeInput.context = new V1_RawBaseExecutionContext(); // TODO: potentially need to support more types
         if (parameterValues) {
-            executeInput.parameterValues = parameterValues.map((p) => V1_transformParameterValue(p));
+            executeInput.parameterValues = parameterValues.map((parameterValue) => V1_transformParameterValue(parameterValue));
         }
         return executeInput;
     };
-    buildExecutionInputGraphData(graph, mapping, runtime) {
+    buildExecutionInputGraphData(graph) {
         /**
          * NOTE: to lessen network load, we might need to think of a way to only include relevant part of the pure model context data here
          *
@@ -996,13 +997,7 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
          * for mapping used for execution, but this can generalized if we introduce hierarchy/ranking for model type
          */
         const graphData = this.getFullGraphModelData(graph);
-        const prunedGraphData = new V1_PureModelContextData();
-        const extraExecutionElements = this.pluginManager
-            .getPureProtocolProcessorPlugins()
-            .flatMap((element) => element.V1_getExtraExecutionInputCollectors?.() ?? [])
-            .flatMap((getter) => getter(graph, mapping, runtime, graphData));
-        prunedGraphData.elements = uniq(graphData.elements
-            .filter((element) => element instanceof V1_Class ||
+        const prunedGraphData = this.prunePureModelContextData(graphData, (element) => element instanceof V1_Class ||
             element instanceof V1_Enumeration ||
             element instanceof V1_Profile ||
             element instanceof V1_Association ||
@@ -1011,8 +1006,15 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
             element instanceof V1_Store ||
             element instanceof V1_PackageableConnection ||
             element instanceof V1_PackageableRuntime ||
-            element instanceof V1_Mapping)
-            .concat(extraExecutionElements));
+            element instanceof V1_Mapping, undefined);
+        const extraExecutionElements = this.pluginManager
+            .getPureProtocolProcessorPlugins()
+            .flatMap((element) => element.V1_getExtraExecutionInputCollectors?.() ?? [])
+            .flatMap((getter) => getter(graph, graphData));
+        prunedGraphData.elements = uniq([
+            ...prunedGraphData.elements,
+            ...extraExecutionElements,
+        ]);
         return prunedGraphData;
     }
     async runQuery(lambda, mapping, runtime, graph, options, _report) {
@@ -1033,17 +1035,17 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
         const stopWatch = new StopWatch();
         const pureModelContext = graph.origin
             ? this.buildPureModelSDLCPointer(graph.origin, undefined)
-            : this.buildExecutionInputGraphData(graph, mapping, undefined);
+            : this.buildExecutionInputGraphData(graph);
         stopWatch.record(GRAPH_MANAGER_EVENT.V1_ENGINE_OPERATION_INPUT__SUCCESS);
         await Promise.all(tests.map((t) => this.DEPRECATED__runLegacyMappingTest(t, mapping, pureModelContext, options, report)));
     }
-    async DEPRECATED__runLegacyMappingTest(testInfo, mapping, pureModelContext, options, _report) {
+    async DEPRECATED__runLegacyMappingTest(testInfo, mapping, data, options, _report) {
         const report = _report ?? createGraphManagerOperationReport();
         try {
             const stopWatch = new StopWatch();
             stopWatch.record(GRAPH_MANAGER_EVENT.V1_ENGINE_OPERATION_INPUT__SUCCESS);
             DEPRECATED__validate_MappingTest(testInfo.test);
-            const input = this.buildExecutionInputWithPureModelContext(pureModelContext, mapping, testInfo.test.query, testInfo.runtime, V1_PureGraphManager.PROD_PROTOCOL_VERSION, new V1_ExecuteInput(), options?.parameterValues);
+            const input = this.createExecutionInputWithPureModelContext(data, mapping, testInfo.test.query, testInfo.runtime, V1_PureGraphManager.PROD_PROTOCOL_VERSION, new V1_ExecuteInput(), options?.parameterValues);
             const result = V1_buildExecutionResult(await this.engine.executeMapping(input, options));
             stopWatch.record(GRAPH_MANAGER_EVENT.V1_ENGINE_OPERATION_SERVER_CALL__SUCCESS);
             report.timings = {
@@ -1090,7 +1092,9 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
         const report = _report ?? createGraphManagerOperationReport();
         const stopWatch = new StopWatch();
         const testDataGenerationExecuteInput = new V1_TestDataGenerationExecutionInput();
-        this.buildExecutionInput(graph, mapping, lambda, runtime, V1_PureGraphManager.DEV_PROTOCOL_VERSION, testDataGenerationExecuteInput);
+        this.createExecutionInputWithPureModelContext(graph.origin
+            ? this.buildPureModelSDLCPointer(graph.origin, undefined)
+            : this.buildExecutionInputGraphData(graph), mapping, lambda, runtime, V1_PureGraphManager.DEV_PROTOCOL_VERSION, testDataGenerationExecuteInput);
         testDataGenerationExecuteInput.parameters = parameters;
         testDataGenerationExecuteInput.hashStrings = Boolean(options?.anonymizeGeneratedData);
         stopWatch.record(GRAPH_MANAGER_EVENT.V1_ENGINE_OPERATION_INPUT__SUCCESS);
@@ -1119,7 +1123,8 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
         const protocol = new V1_Protocol(V1_PureGraphManager.PURE_PROTOCOL_NAME, serverServiceInfo.services.dependencies.pure);
         switch (executionMode) {
             case ServiceExecutionMode.FULL_INTERACTIVE: {
-                const data = this.createServiceRegistrationInput(graph, service);
+                const data = this.createServiceRegistrationInputGraphData(graph);
+                data.elements.push(this.elementToProtocol(service));
                 data.origin = new V1_PureModelContextPointer(protocol);
                 input = data;
                 break;
@@ -1180,8 +1185,9 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
         const protocol = new V1_Protocol(V1_PureGraphManager.PURE_PROTOCOL_NAME, serverServiceInfo.services.dependencies.pure);
         switch (executionMode) {
             case ServiceExecutionMode.FULL_INTERACTIVE: {
-                const pmcp = this.createBulkServiceRegistrationInput(graph, services);
-                pmcp.forEach((data) => {
+                services.forEach((service) => {
+                    const data = this.prunePureModelContextData(this.createServiceRegistrationInputGraphData(graph));
+                    data.elements.push(this.elementToProtocol(service));
                     data.origin = new V1_PureModelContextPointer(protocol);
                     input.push(data);
                 });
@@ -1264,23 +1270,16 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
         const serviceStorage = await this.engine.getServiceVersionInfo(serviceUrl, serviceId);
         await this.engine.activateServiceGeneration(serviceUrl, serviceStorage.getGenerationId());
     }
-    createServiceRegistrationInput = (graph, service) => {
-        const graphData = this.getFullGraphModelData(graph);
-        const prunedGraphData = new V1_PureModelContextData();
-        prunedGraphData.elements = graphData.elements.filter((element) => !(element instanceof V1_Service));
-        prunedGraphData.elements.push(this.elementToProtocol(service));
-        return prunedGraphData;
-    };
-    createBulkServiceRegistrationInput = (graph, services) => {
+    // NOTE: We almost should never be poking into dependency entities. However for service registration the input
+    // expects only one service in the graph data. Perhaps, the service registration API should be modified to accept
+    // service as a parameter outside the model
+    createServiceRegistrationInputGraphData = (graph) => {
         const graphData = this.getFullGraphModelData(graph);
-        const results = [];
-        services.forEach((service) => {
-            const prunedGraphData = new V1_PureModelContextData();
-            prunedGraphData.elements = graphData.elements.filter((element) => !(element instanceof V1_Service));
-            prunedGraphData.elements.push(this.elementToProtocol(service));
-            results.push(prunedGraphData);
+        const prunedGraphData = this.prunePureModelContextData(graphData, (element) => !(element instanceof V1_Service), (entity) => {
+            const content = entity.content;
+            return content._type !== V1_SERVICE_ELEMENT_PROTOCOL_TYPE;
         });
-        return results;
+        return prunedGraphData;
     };
     // --------------------------------------------- Query ---------------------------------------------
     async searchQueries(searchSpecification) {
@@ -1341,44 +1340,13 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
             : element.path, element.hashCode))));
         return hashMap;
     }
-    // ---------------------------------------- Analysis ----------------------------------------
-    buildMappingModelCoverageAnalysisInputContextData = (graph) => {
-        /**
-         * To lessen network load, we only include relevant part of the pure model context data here
-         *
-         * Graph data models can be classified based on dependency hieararchy:
-         * 1. Building blocks: models that all other models depend on: e.g. domain models, connections, etc.
-         * 2. Consumers: models that depends on other models: e.g. mapping, service, etc.
-         * 3. Unrelated: models that depends on nothing and vice versa: e.g. text
-         *
-         * For analyzing mapping, we choose to include some building blocks and mappings.
-         */
-        const graphData = this.getFullGraphModelData(graph);
-        const prunedGraphData = new V1_PureModelContextData();
-        const extraElements = this.pluginManager
-            .getPureProtocolProcessorPlugins()
-            .flatMap((element) => element.V1_getExtraMappingModelCoverageAnalysisInputCollectors?.() ??
-            [])
-            .flatMap((getter) => getter(graph, graphData));
-        prunedGraphData.elements = uniq(graphData.elements
-            .filter((element) => element instanceof V1_Class ||
-            element instanceof V1_Enumeration ||
-            element instanceof V1_Profile ||
-            element instanceof V1_Association ||
-            element instanceof V1_ConcreteFunctionDefinition ||
-            element instanceof V1_Measure ||
-            element instanceof V1_Store ||
-            element instanceof V1_Mapping)
-            .concat(extraElements));
-        return prunedGraphData;
-    };
     async analyzeMappingModelCoverage(mapping, graph) {
         const input = new V1_MappingModelCoverageAnalysisInput();
         input.clientVersion = V1_PureGraphManager.DEV_PROTOCOL_VERSION;
         input.mapping = mapping.path;
         input.model = graph.origin
             ? this.buildPureModelSDLCPointer(graph.origin, undefined)
-            : this.buildMappingModelCoverageAnalysisInputContextData(graph);
+            : this.buildExecutionInputGraphData(graph);
         return V1_buildModelCoverageAnalysisResult(await this.engine.analyzeMappingModelCoverage(input), mapping);
     }
     buildMappingModelCoverageAnalysisResult(input, mapping) {
@@ -1392,8 +1360,8 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
         input.query = query
             ? V1_transformRawLambda(query, new V1_GraphTransformerContextBuilder(this.pluginManager.getPureProtocolProcessorPlugins()).build())
             : undefined;
-        if (graphData instanceof LiveGraphData) {
-            input.model = this.buildExecutionInputGraphData(graphData.graph, mapping, undefined);
+        if (graphData instanceof InMemoryGraphData) {
+            input.model = this.buildExecutionInputGraphData(graphData.graph);
         }
         else if (graphData instanceof GraphDataWithOrigin) {
             input.model = this.buildPureModelSDLCPointer(graphData.origin, V1_PureGraphManager.PROD_PROTOCOL_VERSION);
@@ -1446,6 +1414,13 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
         return entity;
     };
     // --------------------------------------------- Shared ---------------------------------------------
+    prunePureModelContextData = (data, elementFilter, entityFilter) => {
+        const prunedGraphData = new V1_PureModelContextData();
+        prunedGraphData.elements = data.elements.filter((element) => elementFilter ? elementFilter(element) : true);
+        prunedGraphData.INTERNAL__rawDependencyEntities =
+            data.INTERNAL__rawDependencyEntities.filter((entity) => entityFilter ? entityFilter(entity) : true);
+        return prunedGraphData;
+    };
     buildPureModelSDLCPointer(origin, clientVersion) {
         if (origin instanceof LegendSDLC) {
             return new V1_PureModelContextPointer(clientVersion
@@ -1534,6 +1509,10 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
             ...contextData1.elements,
             ...contextData2.elements,
         ];
+        contextData1.INTERNAL__rawDependencyEntities = [
+            ...contextData1.INTERNAL__rawDependencyEntities,
+            ...contextData2.INTERNAL__rawDependencyEntities,
+        ];
         return contextData1;
     }
     elementProtocolToEntity = (elementProtocol) => ({
@@ -1628,10 +1607,18 @@ class V1_PureGraphManager extends AbstractPureGraphManager {
         const graphData = new V1_PureModelContextData();
         const dependencyManager = graph.dependencyManager;
         const generatedModel = graph.generationModel;
-        graphData.elements = [
-            ...dependencyManager.allOwnElements,
-            ...generatedModel.allOwnElements,
-        ].map((element) => this.elementToProtocol(element));
+        let elements = [...generatedModel.allOwnElements];
+        if (dependencyManager.origin instanceof GraphEntities) {
+            // If dependency manager holds the original entities we will just use those to save on transforming/serialization
+            // of dependency elements. This can further be improved by adding support for PureModelContext composite so engine understands
+            // list of pure model context sdlc pointers for dependencies.
+            graphData.INTERNAL__rawDependencyEntities =
+                dependencyManager.origin.entities;
+        }
+        else {
+            elements = [...dependencyManager.allOwnElements, ...elements];
+        }
+        graphData.elements = elements.map((element) => this.elementToProtocol(element));
         this.logService.info(LogEvent.create(GRAPH_MANAGER_EVENT.COLLECT_GRAPH_COMPILE_CONTEXT__SUCCESS), Date.now() - startTime, 'ms');
         return graphData;
     };