@finos/legend-graph 8.0.3 → 10.0.0

package/src/models/protocols/pure/v1/V1_PureGraphManager.ts

@@ -18,7 +18,6 @@ import { GRAPH_MANAGER_EVENT } from '../../../../graphManager/GraphManagerEvent.
 import {
   CORE_PURE_PATH,
   ELEMENT_PATH_DELIMITER,
-  SOURCE_INFORMATION_KEY,
   PackageableElementPointerType,
 } from '../../../../MetaModelConst.js';
 import {
@@ -32,7 +31,6 @@ import {
   getClass,
   guaranteeNonNullable,
   UnsupportedOperationError,
-  recursiveOmit,
   assertTrue,
   assertErrorThrown,
   promisify,
@@ -40,6 +38,10 @@ import {
   filterByType,
   isNonNullable,
   guaranteeNonEmptyString,
+  addUniqueEntry,
+  uuid,
+  deleteEntry,
+  assertType,
 } from '@finos/legend-shared';
 import type { TEMPORARY__AbstractEngineConfig } from '../../../../graphManager/action/TEMPORARY__AbstractEngineConfig.js';
 import {
@@ -48,7 +50,7 @@ import {
   type GraphBuilderOptions,
   type ExecutionOptions,
 } from '../../../../graphManager/AbstractPureGraphManager.js';
-import { Mapping } from '../../../metamodels/pure/packageableElements/mapping/Mapping.js';
+import type { Mapping } from '../../../metamodels/pure/packageableElements/mapping/Mapping.js';
 import {
   type Runtime,
   EngineRuntime,
@@ -208,7 +210,7 @@ import {
   V1_buildLightQuery,
   V1_transformQuerySearchSpecification,
 } from './engine/V1_EngineHelper.js';
-import { V1_buildExecutionResult } from './engine/V1_ExecutionHelper.js';
+import { V1_buildExecutionResult } from './engine/execution/V1_ExecutionHelper.js';
 import {
   type Entity,
   ENTITY_PATH_DELIMITER,
@@ -216,6 +218,7 @@ import {
 import {
   DependencyGraphBuilderError,
   GraphBuilderError,
+  PureClientVersion,
   SystemGraphBuilderError,
 } from '../../../../graphManager/GraphManagerUtils.js';
 import {
@@ -246,7 +249,10 @@ import {
 import { V1_AtomicTestId } from './model/test/V1_AtomicTestId.js';
 import type { RunTestsTestableInput } from '../../../metamodels/pure/test/result/RunTestsTestableInput.js';
 import { V1_buildTestsResult } from './engine/test/V1_RunTestsResult.js';
-import type { TestResult } from '../../../metamodels/pure/test/result/TestResult.js';
+import {
+  type TestResult,
+  TestFailed,
+} from '../../../metamodels/pure/test/result/TestResult.js';
 import type { Service } from '../../../../DSLService_Exports.js';
 import type { Testable } from '../../../metamodels/pure/test/Testable.js';
 import { stub_RawLambda } from '../../../../graphManager/action/creation/RawValueSpecificationCreatorHelper.js';
@@ -254,7 +260,25 @@ import {
   getNullableIDFromTestable,
   getNullableTestable,
 } from '../../../../helpers/Testable_Helper.js';
+import type { TestAssertion } from '../../../metamodels/pure/test/assertion/TestAssertion.js';
+import type { AssertFail } from '../../../metamodels/pure/test/assertion/status/AssertFail.js';
+import {
+  type AtomicTest,
+  TestSuite,
+} from '../../../metamodels/pure/test/Test.js';
 import { V1_getIncludedMappingPath } from './helper/V1_DSLMapping_Helper.js';
+import { pruneSourceInformation } from '../../../../MetaModelUtils.js';
+import { stub_Mapping } from '../../../../graphManager/action/creation/DSLMapping_ModelCreatorHelper.js';
+import {
+  V1_buildModelCoverageAnalysisResult,
+  V1_MappingModelCoverageAnalysisInput,
+  V1_MappingModelCoverageAnalysisResult,
+} from './engine/analytics/V1_MappingAnalytics.js';
+import type {
+  MappingModelCoverageAnalysisResult,
+  RawMappingModelCoverageAnalysisResult,
+} from '../../../../graphManager/action/analytics/MappingAnalytics.js';
+import { deserialize } from 'serializr';
 
 const V1_FUNCTION_SUFFIX_MULTIPLICITY_INFINITE = 'MANY';
 
@@ -1666,6 +1690,56 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     return result;
   }
 
+  async generateExpectedResult(
+    testable: Testable,
+    test: AtomicTest,
+    baseAssertion: TestAssertion,
+    graph: PureModel,
+  ): Promise<AssertFail> {
+    const id = uuid();
+    try {
+      baseAssertion.id = id;
+      addUniqueEntry(test.assertions, baseAssertion);
+      const runTestsInput = new V1_RunTestsInput();
+      runTestsInput.model = this.getFullGraphModelData(graph);
+      const runTestableInput = new V1_RunTestsTestableInput();
+      const unitAtomicTest = new V1_AtomicTestId();
+      runTestableInput.testable = guaranteeNonNullable(
+        getNullableIDFromTestable(
+          testable,
+          graph,
+          this.pluginManager.getPureGraphManagerPlugins(),
+        ),
+      );
+      runTestsInput.testables = [runTestableInput];
+      const parent = test.__parent;
+      unitAtomicTest.testSuiteId =
+        parent instanceof TestSuite ? parent.id : undefined;
+      unitAtomicTest.atomicTestId = test.id;
+      const runTestsResult = await this.engine.runTests(runTestsInput);
+      const results = V1_buildTestsResult(
+        runTestsResult,
+        (_id: string): Testable | undefined =>
+          getNullableTestable(
+            _id,
+            graph,
+            this.pluginManager.getPureGraphManagerPlugins(),
+          ),
+      );
+      const result = results[0];
+      assertType(result, TestFailed);
+      const status = result.assertStatuses.find(
+        (e) => e.assertion === baseAssertion,
+      );
+      return guaranteeNonNullable(status);
+    } catch (error) {
+      assertErrorThrown(error);
+      throw error;
+    } finally {
+      deleteEntry(test.assertions, baseAssertion);
+    }
+  }
+
   // ------------------------------------------- ValueSpecification -------------------------------------------
 
   buildValueSpecification(
@@ -1840,13 +1914,14 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
      * object to identify relevant classes yet. so the more economical way to to base on the classification above and the knowledge about hierarchy between
      * models (e.g. service can use mapping, runtime, connection, store, etc.) we can roughly prune the graph model data by group. Following is an example
      * for mapping used for execution, but this can generalized if we introduce hierarchy/ranking for model type
-     *
      */
     const graphData = this.getFullGraphModelData(graph);
     const prunedGraphData = new V1_PureModelContextData();
     const extraExecutionElements = this.pluginManager
       .getPureProtocolProcessorPlugins()
-      .flatMap((element) => element.V1_getExtraExecutionInputGetters?.() ?? [])
+      .flatMap(
+        (element) => element.V1_getExtraExecutionInputCollectors?.() ?? [],
+      )
       .flatMap((getter) => getter(graph, mapping, runtime, graphData));
     prunedGraphData.elements = graphData.elements
       .filter(
@@ -1889,7 +1964,6 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     mapping: Mapping,
     lambda: RawLambda,
     runtime: Runtime,
-    clientVersion: string,
     options?: ExecutionOptions,
   ): Promise<ExecutionResult> {
     return V1_buildExecutionResult(
@@ -1899,19 +1973,18 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
           mapping,
           lambda,
           runtime,
-          clientVersion,
+          PureClientVersion.VX_X_X,
         ),
         options,
       ),
     );
   }
 
-  generateMappingTestData(
+  generateExecuteTestData(
     graph: PureModel,
     mapping: Mapping,
     lambda: RawLambda,
     runtime: Runtime,
-    clientVersion: string,
     parameters: (string | number | boolean)[],
     options?: {
       anonymizeGeneratedData?: boolean;
@@ -1924,14 +1997,14 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
       mapping,
       lambda,
       runtime,
-      clientVersion,
+      PureClientVersion.VX_X_X,
       testDataGenerationExecuteInput,
     );
     testDataGenerationExecuteInput.parameters = parameters;
     testDataGenerationExecuteInput.hashStrings = Boolean(
       options?.anonymizeGeneratedData,
     );
-    return this.engine.generateMappingTestData(testDataGenerationExecuteInput);
+    return this.engine.generateExecuteTestData(testDataGenerationExecuteInput);
   }
 
   generateExecutionPlan(
@@ -1939,10 +2012,15 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     mapping: Mapping,
     lambda: RawLambda,
     runtime: Runtime,
-    clientVersion: string,
   ): Promise<RawExecutionPlan> {
     return this.engine.generateExecutionPlan(
-      this.createExecutionInput(graph, mapping, lambda, runtime, clientVersion),
+      this.createExecutionInput(
+        graph,
+        mapping,
+        lambda,
+        runtime,
+        PureClientVersion.VX_X_X,
+      ),
     );
   }
 
@@ -1951,10 +2029,15 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     mapping: Mapping,
     lambda: RawLambda,
     runtime: Runtime,
-    clientVersion: string,
   ): Promise<{ plan: RawExecutionPlan; debug: string }> {
     const result = await this.engine.debugExecutionPlanGeneration(
-      this.createExecutionInput(graph, mapping, lambda, runtime, clientVersion),
+      this.createExecutionInput(
+        graph,
+        mapping,
+        lambda,
+        runtime,
+        PureClientVersion.VX_X_X,
+      ),
     );
     return {
       plan: result.plan,
@@ -2339,7 +2422,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
                 (keyedExecutionParameter) =>
                   new KeyedExecutionParameter(
                     keyedExecutionParameter.key,
-                    PackageableElementExplicitReference.create(new Mapping('')),
+                    PackageableElementExplicitReference.create(stub_Mapping()),
                     new EngineRuntime(),
                   ),
               );
@@ -2348,7 +2431,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
             service.execution = new PureSingleExecution(
               stub_RawLambda(),
               service,
-              PackageableElementExplicitReference.create(new Mapping('')),
+              PackageableElementExplicitReference.create(stub_Mapping()),
               new EngineRuntime(),
             );
           }
@@ -2416,64 +2499,224 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     return hashMap;
   }
 
-  // ------------------------------------------- Raw Protocol Handling -------------------------------------------
+  // ---------------------------------------- Analysis ----------------------------------------
 
-  /**
-   * As mentioned, this method is needed because sometimes we want to parse and store protocol from grammar
-   * However, we want to prune away the source information. There is no realy good way to do this, other than
-   * hard-coding the source informations fields like this. We should cleanup these in the backend and use
-   * pointer instead so source information is coupled with the value instead of having custom-name source information
-   * fields like these, polluting the protocol models.
-   */
-  pruneSourceInformation = (
-    object: Record<PropertyKey, unknown>,
-  ): Record<PropertyKey, unknown> =>
-    recursiveOmit(
-      object,
-      [
-        SOURCE_INFORMATION_KEY,
-        // domain
-        'profileSourceInformation',
-        'propertyTypeSourceInformation',
-        // connection
-        'elementSourceInformation',
-        'classSourceInformation',
-        'mappingsSourceInformation',
-        // mapping
-        'classSourceInformation',
-        'sourceClassSourceInformation',
-        'enumMappingIdSourceInformation',
-        'flatDataSourceInformation',
-        // service
-        'mappingSourceInformation',
-        // flat-data store
-        'nameSourceInformation',
-        'driverIdSourceInformation',
-        // file generation
-        'typeSourceInformation',
-      ].concat(
-        this.pluginManager
-          .getPureProtocolProcessorPlugins()
-          .flatMap(
-            (plugin) => plugin.V1_getExtraSourceInformationKeys?.() ?? [],
-          ),
+  private buildMappingModelCoverageAnalysisInputContextData = (
+    graph: PureModel,
+  ): V1_PureModelContextData => {
+    const graphData = this.getFullGraphModelData(graph);
+    const prunedGraphData = new V1_PureModelContextData();
+    const extraElements = this.pluginManager
+      .getPureProtocolProcessorPlugins()
+      .flatMap(
+        (element) =>
+          element.V1_getExtraMappingModelCoverageAnalysisInputCollectors?.() ??
+          [],
+      )
+      .flatMap((getter) => getter(graph, graphData));
+    prunedGraphData.elements = this.getFullGraphModelData(graph)
+      .elements.filter(
+        (element) =>
+          element instanceof V1_Class ||
+          element instanceof V1_Enumeration ||
+          element instanceof V1_Profile ||
+          element instanceof V1_Association ||
+          element instanceof V1_ConcreteFunctionDefinition ||
+          element instanceof V1_Measure ||
+          element instanceof V1_Store ||
+          element instanceof V1_Mapping,
+      )
+      .concat(extraElements);
+    return prunedGraphData;
+  };
+
+  async analyzeMappingModelCoverage(
+    graph: PureModel,
+    mapping: Mapping,
+  ): Promise<MappingModelCoverageAnalysisResult> {
+    const modelCoverageAnalysisInput =
+      new V1_MappingModelCoverageAnalysisInput();
+    modelCoverageAnalysisInput.clientVersion = PureClientVersion.VX_X_X;
+    modelCoverageAnalysisInput.mapping = mapping.path;
+    modelCoverageAnalysisInput.model =
+      this.buildMappingModelCoverageAnalysisInputContextData(graph);
+    return V1_buildModelCoverageAnalysisResult(
+      await this.engine.analyzeMappingModelCoverage(modelCoverageAnalysisInput),
+    );
+  }
+
+  buildMappingModelCoverageAnalysisResult(
+    input: RawMappingModelCoverageAnalysisResult,
+  ): MappingModelCoverageAnalysisResult {
+    return V1_buildModelCoverageAnalysisResult(
+      deserialize(
+        V1_MappingModelCoverageAnalysisResult,
+        input as PlainObject<V1_MappingModelCoverageAnalysisResult>,
       ),
     );
+  }
+
+  // --------------------------------------------- Utilities ---------------------------------------------
 
   elementToEntity = (
     element: PackageableElement,
-    pruneSourceInformation?: boolean,
+    options?: {
+      pruneSourceInformation?: boolean;
+    },
   ): Entity => {
     const entity = this.elementProtocolToEntity(
       this.elementToProtocol<V1_PackageableElement>(element),
     );
-    if (pruneSourceInformation) {
-      entity.content = this.pruneSourceInformation(entity.content);
+    if (options?.pruneSourceInformation) {
+      entity.content = pruneSourceInformation(entity.content);
     }
     return entity;
   };
 
-  // --------------------------------------------- SHARED ---------------------------------------------
+  async buildDatabase(input: DatabaseBuilderInput): Promise<Entity[]> {
+    const dbBuilderInput = new V1_DatabaseBuilderInput();
+    dbBuilderInput.connection = V1_transformRelationalDatabaseConnection(
+      input.connection,
+      new V1_GraphTransformerContextBuilder(
+        this.pluginManager.getPureProtocolProcessorPlugins(),
+      ).build(),
+    );
+    const targetDatabase = new V1_TargetDatabase();
+    targetDatabase.package = input.targetDatabase.package;
+    targetDatabase.name = input.targetDatabase.name;
+    dbBuilderInput.targetDatabase = targetDatabase;
+    const config = new V1_DatabaseBuilderConfig();
+    config.maxTables = input.config.maxTables;
+    config.enrichTables = input.config.enrichTables;
+    config.enrichPrimaryKeys = input.config.enrichPrimaryKeys;
+    config.enrichColumns = input.config.enrichColumns;
+    config.patterns = input.config.patterns.map(
+      (storePattern: DatabasePattern): V1_DatabasePattern => {
+        const pattern = new V1_DatabasePattern();
+        pattern.schemaPattern = storePattern.schemaPattern;
+        pattern.tablePattern = storePattern.tablePattern;
+        pattern.escapeSchemaPattern = storePattern.escapeSchemaPattern;
+        pattern.escapeTablePattern = storePattern.escapeTablePattern;
+        return pattern;
+      },
+    );
+    dbBuilderInput.config = config;
+    return this.pureModelContextDataToEntities(
+      await this.engine.buildDatabase(dbBuilderInput),
+    );
+  }
+
+  // --------------------------------------------- HACKY ---------------------------------------------
+
+  HACKY__createGetAllLambda(_class: Class): RawLambda {
+    return new RawLambda(
+      [],
+      [
+        {
+          _type: 'func',
+          function: 'getAll',
+          parameters: [
+            {
+              _type: 'packageableElementPtr',
+              fullPath: _class.path,
+            },
+          ],
+        },
+      ],
+    );
+  }
+
+  HACKY__createServiceTestAssertLambda(assertData: string): RawLambda {
+    return new RawLambda(
+      [
+        {
+          _type: 'var',
+          class: 'meta::pure::mapping::Result',
+          multiplicity: { lowerBound: 1, upperBound: 1 },
+          name: 'res',
+        },
+      ],
+      [
+        {
+          _type: 'func',
+          function: 'equalJsonStrings',
+          parameters: [
+            {
+              _type: 'func',
+              function: 'toString',
+              parameters: [
+                {
+                  _type: 'func',
+                  function: 'toOne',
+                  parameters: [
+                    {
+                      _type: 'property',
+                      parameters: [
+                        {
+                          _type: 'var',
+                          name: 'res',
+                        },
+                      ],
+                      property: 'values',
+                    },
+                  ],
+                },
+              ],
+            },
+            {
+              _type: 'string',
+              multiplicity: {
+                lowerBound: 1,
+                upperBound: 1,
+              },
+              values: [assertData],
+            },
+          ],
+        },
+      ],
+    );
+  }
+
+  HACKY__extractServiceTestAssertionData(query: RawLambda): string | undefined {
+    let json: string | undefined;
+    try {
+      json = (
+        (
+          ((query.body as unknown[])[0] as Record<PropertyKey, unknown>) // FunctionValue
+            .parameters as unknown[]
+        )[1] as {
+          values: (string | undefined)[];
+        }
+      ).values[0];
+      assertTrue(typeof json === 'string', `Expected value of type 'string'`);
+    } catch (error) {
+      assertErrorThrown(error);
+      this.log.warn(
+        LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_MANAGER_FAILURE),
+        `Can't extract assertion result`,
+      );
+      json = undefined;
+    }
+    if (!json) {
+      /* Add other assertion cases if we read others */
+    }
+    return json;
+  }
+
+  HACKY__createDefaultBlankLambda(): RawLambda {
+    return new RawLambda(
+      [{ _type: 'var', name: 'x' }],
+      [
+        {
+          _type: 'string',
+          multiplicity: { lowerBound: 1, upperBound: 1 },
+          values: [''],
+        },
+      ],
+    );
+  }
+
+  // --------------------------------------------- Shared ---------------------------------------------
 
   private async entitiesToPureModelContextData(
     entities: Entity[],
@@ -2643,149 +2886,4 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     );
     return graphData;
   };
-
-  // --------------------------------------------- Utilities ---------------------------------------------
-
-  async buildDatabase(input: DatabaseBuilderInput): Promise<Entity[]> {
-    const dbBuilderInput = new V1_DatabaseBuilderInput();
-    dbBuilderInput.connection = V1_transformRelationalDatabaseConnection(
-      input.connection,
-      new V1_GraphTransformerContextBuilder(
-        this.pluginManager.getPureProtocolProcessorPlugins(),
-      ).build(),
-    );
-    const targetDatabase = new V1_TargetDatabase();
-    targetDatabase.package = input.targetDatabase.package;
-    targetDatabase.name = input.targetDatabase.name;
-    dbBuilderInput.targetDatabase = targetDatabase;
-    const config = new V1_DatabaseBuilderConfig();
-    config.maxTables = input.config.maxTables;
-    config.enrichTables = input.config.enrichTables;
-    config.enrichPrimaryKeys = input.config.enrichPrimaryKeys;
-    config.enrichColumns = input.config.enrichColumns;
-    config.patterns = input.config.patterns.map(
-      (storePattern: DatabasePattern): V1_DatabasePattern => {
-        const pattern = new V1_DatabasePattern();
-        pattern.schemaPattern = storePattern.schemaPattern;
-        pattern.tablePattern = storePattern.tablePattern;
-        pattern.escapeSchemaPattern = storePattern.escapeSchemaPattern;
-        pattern.escapeTablePattern = storePattern.escapeTablePattern;
-        return pattern;
-      },
-    );
-    dbBuilderInput.config = config;
-    return this.pureModelContextDataToEntities(
-      await this.engine.buildDatabase(dbBuilderInput),
-    );
-  }
-
-  // --------------------------------------------- HACKY ---------------------------------------------
-
-  HACKY__createGetAllLambda(_class: Class): RawLambda {
-    return new RawLambda(
-      [],
-      [
-        {
-          _type: 'func',
-          function: 'getAll',
-          parameters: [
-            {
-              _type: 'packageableElementPtr',
-              fullPath: _class.path,
-            },
-          ],
-        },
-      ],
-    );
-  }
-
-  HACKY__createServiceTestAssertLambda(assertData: string): RawLambda {
-    return new RawLambda(
-      [
-        {
-          _type: 'var',
-          class: 'meta::pure::mapping::Result',
-          multiplicity: { lowerBound: 1, upperBound: 1 },
-          name: 'res',
-        },
-      ],
-      [
-        {
-          _type: 'func',
-          function: 'equalJsonStrings',
-          parameters: [
-            {
-              _type: 'func',
-              function: 'toString',
-              parameters: [
-                {
-                  _type: 'func',
-                  function: 'toOne',
-                  parameters: [
-                    {
-                      _type: 'property',
-                      parameters: [
-                        {
-                          _type: 'var',
-                          name: 'res',
-                        },
-                      ],
-                      property: 'values',
-                    },
-                  ],
-                },
-              ],
-            },
-            {
-              _type: 'string',
-              multiplicity: {
-                lowerBound: 1,
-                upperBound: 1,
-              },
-              values: [assertData],
-            },
-          ],
-        },
-      ],
-    );
-  }
-
-  HACKY__extractServiceTestAssertionData(query: RawLambda): string | undefined {
-    let json: string | undefined;
-    try {
-      json = (
-        (
-          ((query.body as unknown[])[0] as Record<PropertyKey, unknown>) // FunctionValue
-            .parameters as unknown[]
-        )[1] as {
-          values: (string | undefined)[];
-        }
-      ).values[0];
-      assertTrue(typeof json === 'string', `Expected value of type 'string'`);
-    } catch (error) {
-      assertErrorThrown(error);
-      this.log.warn(
-        LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_MANAGER_FAILURE),
-        `Can't extract assertion result`,
-      );
-      json = undefined;
-    }
-    if (!json) {
-      /* Add other assertion cases if we read others */
-    }
-    return json;
-  }
-
-  HACKY__createDefaultBlankLambda(): RawLambda {
-    return new RawLambda(
-      [{ _type: 'var', name: 'x' }],
-      [
-        {
-          _type: 'string',
-          multiplicity: { lowerBound: 1, upperBound: 1 },
-          values: [''],
-        },
-      ],
-    );
-  }
 }