@finos/legend-graph 32.3.28 → 32.3.31

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@finos/legend-graph",
-  "version": "32.3.28",
+  "version": "32.3.31",
   "description": "Legend graph and graph manager",
   "keywords": [
     "legend",
@@ -43,12 +43,12 @@
   },
   "dependencies": {
     "@finos/legend-shared": "11.0.22",
-    "@finos/legend-storage": "3.0.141",
+    "@finos/legend-storage": "3.0.143",
     "mobx": "6.13.6",
     "serializr": "3.0.3"
   },
   "devDependencies": {
-    "@finos/legend-dev-utils": "2.2.5",
+    "@finos/legend-dev-utils": "2.2.6",
     "@jest/globals": "29.7.0",
     "axios": "1.8.2",
     "cross-env": "7.0.3",

package/src/graph/BasicModel.ts

@@ -247,7 +247,7 @@ export abstract class BasicModel {
   get ownFileGenerations(): FileGenerationSpecification[] {
     return Array.from(this.fileGenerationsIndex.values());
   }
-  get ownProducts(): DataProduct[] {
+  get ownDataProducts(): DataProduct[] {
     return Array.from(this.productsIndex.values());
   }
   get ownDataElements(): DataElement[] {
@@ -592,7 +592,7 @@ export abstract class BasicModel {
       ...this.ownConnections,
       ...this.ownGenerationSpecifications,
       ...this.ownFileGenerations,
-      ...this.ownProducts,
+      ...this.ownDataProducts,
       ...this.ownDataElements,
       ...this.ownExecutionEnvironments,
       ...Array.from(this.INTERNAL__unknownElementsIndex.values()),

package/src/graph/DependencyManager.ts

@@ -47,6 +47,7 @@ import type { FunctionActivator } from './metamodel/pure/packageableElements/fun
 import type { PureGraphPlugin } from './PureGraphPlugin.js';
 import type { Testable } from './metamodel/pure/test/Testable.js';
 import type { IngestDefinition } from './metamodel/pure/packageableElements/ingest/IngestDefinition.js';
+import type { DataProduct } from './metamodel/pure/dataProduct/DataProduct.js';
 
 export const DEPENDENCY_ROOT_PACKAGE_PREFIX = '@dependency__';
 export const generateDependencyRootPackageName = (
@@ -319,6 +320,10 @@ export class DependencyManager {
     return this.dependencyGraphs.flatMap((dep) => dep.ownIngests);
   }
 
+  get dataProducts(): DataProduct[] {
+    return this.dependencyGraphs.flatMap((dep) => dep.ownDataProducts);
+  }
+
   getExtensionElements<T extends PackageableElement>(
     extensionElementClass: Clazz<T>,
   ): T[] {

package/src/graph/PureModel.ts

@@ -302,6 +302,15 @@ export class PureModel extends BasicModel {
       ...this.generationModel.ownFunctions,
     ];
   }
+  get dataProducts(): DataProduct[] {
+    return [
+      ...this.coreModel.ownDataProducts,
+      ...this.systemModel.ownDataProducts,
+      ...this.dependencyManager.dataProducts,
+      ...this.ownDataProducts,
+      ...this.generationModel.ownDataProducts,
+    ];
+  }
   get functionActivators(): FunctionActivator[] {
     return [
       ...this.coreModel.ownFunctionActivators,

package/src/graph/metamodel/pure/dataProduct/DataProduct.ts

@@ -139,6 +139,11 @@ export class DataProductRuntimeInfo {
 
 export type DataProductElement = Package | Class | Enumeration | Association;
 
+export interface ElementScope {
+  element: PackageableElementReference<DataProductElement>;
+  exclude?: boolean | undefined;
+}
+
 export class DataProductElementScope implements Hashable {
   exclude: boolean | undefined;
   element!: PackageableElementReference<DataProductElement>;
@@ -222,8 +227,8 @@ export class NativeModelExecutionContext implements Hashable {
 
 export class NativeModelAccess implements Hashable {
   featuredElements: DataProductElementScope[] = [];
-  nativeModelExecutionContexts!: NativeModelExecutionContext[];
-  defaultExecutionContext!: string;
+  nativeModelExecutionContexts: NativeModelExecutionContext[] = [];
+  defaultExecutionContext!: NativeModelExecutionContext;
   diagrams: DataProductDiagram[] = [];
   sampleQueries: SampleQuery[] = [];
 
@@ -234,7 +239,7 @@ export class NativeModelAccess implements Hashable {
       hashArray(this.featuredElements),
       hashArray(this.diagrams),
       hashArray(this.sampleQueries),
-      this.defaultExecutionContext,
+      this.defaultExecutionContext.key,
     ]);
   }
 }

package/src/graph/metamodel/pure/packageableElements/function/test/FunctionTest.ts

@@ -47,6 +47,7 @@ export class FunctionTest extends AtomicTest implements Hashable {
       this.id,
       this.doc ?? '',
       this.parameters ? hashArray(this.parameters) : '',
+      hashArray(this.assertions),
     ]);
   }
 }

package/src/graph-manager/action/changeDetection/GraphObserverHelper.ts

@@ -86,7 +86,7 @@ const observe_Abstract_BasicModel = (metamodel: BasicModel): void => {
     ownFunctions: computed,
     ownFunctionActivators: computed,
     ownStores: computed,
-    ownProducts: computed,
+    ownDataProducts: computed,
     ownIngests: computed,
     ownFlatDatas: computed,
     ownDatabases: computed,

package/src/graph-manager/action/query/Query.ts

@@ -46,6 +46,11 @@ export class QueryDataSpaceExecutionContext extends QueryExecutionContext {
   executionKey: string | undefined;
 }
 
+export class QueryDataProductExecutionContext extends QueryExecutionContext {
+  dataProductPath!: string;
+  executionKey: string | undefined;
+}
+
 export interface QueryGridConfig {
   columns: object[];
   isPivotModeEnabled: boolean | undefined;
@@ -154,6 +159,12 @@ export class QueryDataSpaceExecutionContextInfo extends QueryExecutionContextInf
   dataSpacePath!: string;
   executionKey: string | undefined;
 }
+
+export class QueryDataProductNativeExecutionContextInfo extends QueryExecutionContextInfo {
+  dataProductPath!: string;
+  executionKey: string | undefined;
+}
+
 export interface QueryInfo {
   name: string;
   id: string;

package/src/graph-manager/helpers/DataProductHelper.ts

@@ -0,0 +1,113 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { assertTrue, uniq, guaranteeNonNullable } from '@finos/legend-shared';
+import {
+  ModelAccessPointGroup,
+  type NativeModelAccess,
+  type DataProduct,
+  type DataProductElement,
+  type ElementScope,
+} from '../../graph/metamodel/pure/dataProduct/DataProduct.js';
+import type { Mapping } from '../../graph/metamodel/pure/packageableElements/mapping/Mapping.js';
+import type { GraphManagerState } from '../GraphManagerState.js';
+import type { MappingModelCoverageAnalysisResult } from '../action/analytics/MappingModelCoverageAnalysis.js';
+import { getMappingCompatibleClasses } from '../../graph/helpers/DSL_Mapping_Helper.js';
+import type { Class } from '../../graph/metamodel/pure/packageableElements/domain/Class.js';
+import { elementBelongsToPackage } from '../../graph/helpers/DomainHelper.js';
+import { Package } from '../../graph/metamodel/pure/packageableElements/domain/Package.js';
+
+export const isDataProductNative = (dataProduct: DataProduct): boolean => {
+  return dataProduct.nativeModelAccess !== undefined;
+};
+
+export const isDataProductModeled = (dataProduct: DataProduct): boolean => {
+  const groups = dataProduct.accessPointGroups;
+  if (groups.length === 1) {
+    const group = groups[0];
+    if (group instanceof ModelAccessPointGroup) {
+      return true;
+    }
+  }
+  return false;
+};
+
+export const getModelGroupFromNativeDataProduct = (
+  dataProduct: DataProduct,
+): NativeModelAccess => {
+  assertTrue(isDataProductNative(dataProduct), 'Data product is not native');
+  return guaranteeNonNullable(
+    dataProduct.nativeModelAccess,
+    'Native access point group is expected',
+  );
+};
+
+const matchesDataElement = (
+  _class: Class,
+  element: DataProductElement,
+): boolean => {
+  if (_class === element) {
+    return true;
+  }
+  if (element instanceof Package) {
+    return elementBelongsToPackage(_class, element);
+  }
+  return false;
+};
+
+export const resolveUsableDataProductClasses = (
+  elements: ElementScope[] | undefined,
+  mapping: Mapping,
+  graphManagerState: GraphManagerState,
+  mappingModelCoverageAnalysisResult?: MappingModelCoverageAnalysisResult,
+): Class[] => {
+  let compatibleClasses = getMappingCompatibleClasses(
+    mapping,
+    graphManagerState.usableClasses,
+  );
+  if (
+    // This check is to make sure that we have `info` field present in `MappedEntity` which
+    // contains information about the mapped class path
+    mappingModelCoverageAnalysisResult?.mappedEntities.some(
+      (m) => m.info !== undefined,
+    )
+  ) {
+    const uniqueCompatibleClasses = uniq(
+      mappingModelCoverageAnalysisResult.mappedEntities
+        // is root entity filters out mapped classes
+        .filter((e) => e.info?.isRootEntity)
+        .map((e) => e.info?.classPath),
+    );
+    compatibleClasses = graphManagerState.graph.classes.filter((c) =>
+      uniqueCompatibleClasses.includes(c.path),
+    );
+  }
+  if (elements?.length) {
+    return compatibleClasses.filter((_class) => {
+      const _classElements = elements
+        .filter((e) => matchesDataElement(_class, e.element.value))
+        // we sort because we respect the closest definition to the element.
+        .sort(
+          (a, b) => b.element.value.path.length - a.element.value.path.length,
+        );
+      if (!_classElements.length) {
+        return false;
+      }
+      return !_classElements[0]?.exclude;
+    });
+  }
+  return compatibleClasses;
+};

package/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts

@@ -32,6 +32,7 @@ import {
   type QueryExecutionContextInfo,
   QueryExplicitExecutionContextInfo,
   QueryDataSpaceExecutionContextInfo,
+  QueryDataProductExecutionContext,
 } from '../../../../../graph-manager/action/query/Query.js';
 import {
   type V1_LightQuery,
@@ -40,6 +41,7 @@ import {
   V1_QueryExplicitExecutionContext,
   V1_QueryDataSpaceExecutionContext,
   type V1_QueryExecutionContext,
+  V1_QueryDataProductExecutionContext,
 } from './query/V1_Query.js';
 import type { PureModel } from '../../../../../graph/PureModel.js';
 import { DEPRECATED__ServiceTestResult } from '../../../../../graph-manager/action/service/DEPRECATED__ServiceTestResult.js';
@@ -124,6 +126,7 @@ export const V1_buildLightQuery = (
     `Query 'artifactId' field is missing`,
   );
   metamodel.owner = protocol.owner;
+  metamodel.description = protocol.description;
   metamodel.lastUpdatedAt = protocol.lastUpdatedAt;
   metamodel.createdAt = protocol.createdAt;
   metamodel.lastOpenAt = protocol.lastOpenAt;
@@ -337,6 +340,11 @@ export const V1_transformQueryExecutionContext = (
     protocol.dataSpacePath = execContext.dataSpacePath;
     protocol.executionKey = execContext.executionKey;
     return protocol;
+  } else if (execContext instanceof QueryDataProductExecutionContext) {
+    const protocol = new V1_QueryDataProductExecutionContext();
+    protocol.dataProductPath = execContext.dataProductPath;
+    protocol.executionKey = execContext.executionKey;
+    return protocol;
   }
   throw new UnsupportedOperationError('Unsupported query execution context');
 };

package/src/graph-manager/protocol/pure/v1/engine/query/V1_Query.ts

@@ -65,6 +65,7 @@ export class V1_QueryExecutionContext {}
 export enum V1_QueryExecutionContextType {
   QUERY_EXPLICIT_EXECUTION_CONTEXT = 'explicitExecutionContext',
   QUERY_DATASAPCE_EXECUTION_CONTEXT = 'dataSpaceExecutionContext',
+  QUERY_DATAPRODUCT_EXECUTION_CONTEXT = 'dataProductExecutionContext',
 }
 
 export class V1_QueryExplicitExecutionContext extends V1_QueryExecutionContext {
@@ -103,6 +104,24 @@ export class V1_QueryDataSpaceExecutionContext extends V1_QueryExecutionContext
   );
 }
 
+export class V1_QueryDataProductExecutionContext extends V1_QueryExecutionContext {
+  dataProductPath!: string;
+  executionKey: string | undefined;
+
+  static readonly serialization = new SerializationFactory(
+    createModelSchema(V1_QueryDataProductExecutionContext, {
+      _type: usingConstantValueSchema(
+        V1_QueryExecutionContextType.QUERY_DATAPRODUCT_EXECUTION_CONTEXT,
+      ),
+      dataProductPath: primitive(),
+      executionKey: optional(primitive()),
+    }),
+    {
+      deserializeNullAsUndefined: true,
+    },
+  );
+}
+
 export const V1_deserializeQueryExecutionContext = (
   json: PlainObject<V1_QueryExecutionContext>,
 ): V1_QueryExecutionContext => {
@@ -117,6 +136,11 @@ export const V1_deserializeQueryExecutionContext = (
         V1_QueryDataSpaceExecutionContext.serialization.schema,
         json,
       );
+    case V1_QueryExecutionContextType.QUERY_DATAPRODUCT_EXECUTION_CONTEXT:
+      return deserialize(
+        V1_QueryDataProductExecutionContext.serialization.schema,
+        json,
+      );
     default: {
       throw new UnsupportedOperationError(
         `Can't deserialize authentication strategy of type '${json._type}': no compatible deserializer available from plugins`,
@@ -138,6 +162,11 @@ export const V1_serializeQueryExecutionContext = (
       V1_QueryDataSpaceExecutionContext.serialization.schema,
       protocol,
     );
+  } else if (protocol instanceof V1_QueryDataProductExecutionContext) {
+    return serialize(
+      V1_QueryDataProductExecutionContext.serialization.schema,
+      protocol,
+    );
   }
   throw new UnsupportedOperationError(
     `Can't serialize authentication strategy: no compatible serializer available from plugins`,
@@ -217,6 +246,7 @@ export class V1_LightQuery {
   artifactId!: string;
   versionId!: string;
   originalVersionId?: string | undefined;
+  description?: string | undefined;
   lastUpdatedAt?: number | undefined;
   createdAt?: number | undefined;
   lastOpenAt?: number | undefined;
@@ -231,6 +261,7 @@ export class V1_LightQuery {
       lastOpenAt: optional(primitive()),
       name: primitive(),
       owner: optional(primitive()),
+      description: optional(primitive()),
       versionId: primitive(),
       originalVersionId: optional(primitive()),
     }),

package/src/graph-manager/protocol/pure/v1/model/packageableElements/function/test/V1_FunctionTest.ts

@@ -48,6 +48,7 @@ export class V1_FunctionTest extends V1_AtomicTest {
       this.id,
       this.doc ?? '',
       this.parameters ? hashArray(this.parameters) : '',
+      hashArray(this.assertions),
     ]);
   }
 }

package/src/graph-manager/protocol/pure/v1/model/valueSpecification/raw/classInstance/relation/V1_RelationStoreAccessor.ts

@@ -23,3 +23,7 @@ export class V1_DataProductAccessor {
   path: string[] = [];
   parameters: string[] = [];
 }
+
+export class V1_SQLAccessor {
+  sql!: string;
+}

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/from/V1_DataProductTransformer.ts

@@ -167,7 +167,7 @@ export const V1_transformDataProduct = (
       );
     }
     nativeModelAccess.defaultExecutionContext =
-      metamodelNativeModelAccess.defaultExecutionContext;
+      metamodelNativeModelAccess.defaultExecutionContext.key;
     nativeModelAccess.nativeModelExecutionContexts =
       metamodelNativeModelAccess.nativeModelExecutionContexts.map(
         (executionContext) => {

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/to/helpers/V1_DataProductBuilder.ts

@@ -310,12 +310,16 @@ export const V1_buildNativeModelAccess = (
   context: V1_GraphBuilderContext,
 ): NativeModelAccess => {
   const metamodelNativeModelAccess = new NativeModelAccess();
-  metamodelNativeModelAccess.defaultExecutionContext =
-    nativeModelAccess.defaultExecutionContext;
   metamodelNativeModelAccess.nativeModelExecutionContexts =
     nativeModelAccess.nativeModelExecutionContexts.map((executionContext) =>
       V1_buildNativeModelExecutionContext(executionContext, context),
     );
+  metamodelNativeModelAccess.defaultExecutionContext = guaranteeNonNullable(
+    metamodelNativeModelAccess.nativeModelExecutionContexts.find(
+      (e) => e.key === nativeModelAccess.defaultExecutionContext,
+    ),
+    `Default execution context ${nativeModelAccess.defaultExecutionContext} not found among native model execution contexts`,
+  );
   if (nativeModelAccess.sampleQueries) {
     metamodelNativeModelAccess.sampleQueries =
       nativeModelAccess.sampleQueries.map((sampleQuery) =>

package/src/graph-manager/protocol/pure/v1/transformation/pureProtocol/serializationHelpers/V1_ValueSpecificationSerializer.ts

@@ -70,6 +70,7 @@ import { V1_ColSpec } from '../../../model/valueSpecification/raw/classInstance/
 import {
   V1_DataProductAccessor,
   V1_RelationStoreAccessor,
+  V1_SQLAccessor,
 } from '../../../model/valueSpecification/raw/classInstance/relation/V1_RelationStoreAccessor.js';
 import { V1_multiplicityModelSchema } from './V1_CoreSerializationHelper.js';
 import type {
@@ -129,6 +130,7 @@ export enum V1_ClassInstanceType {
   RELATION_STORE_ACCESSOR = '>',
   INGEST_ACCESSOR = 'I',
   DATA_PRODUCT_ACCESSOR = 'P',
+  SQL_ACCESSOR = 'SQL',
 
   TDS_AGGREGATE_VALUE = 'tdsAggregateValue',
   TDS_COLUMN_INFORMATION = 'tdsColumnInformation',
@@ -779,6 +781,10 @@ const dataProductAccessorModelSchema = createModelSchema(
   },
 );
 
+const sqlAccessorModelSchema = createModelSchema(V1_SQLAccessor, {
+  sql: primitive(),
+});
+
 const colSpecModelSchema = (
   plugins: PureProtocolProcessorPlugin[],
 ): ModelSchema<V1_ColSpec> =>
@@ -854,6 +860,8 @@ export function V1_deserializeClassInstanceValue(
       return deserialize(relationStoreAccessorModelSchema, json);
     case V1_ClassInstanceType.DATA_PRODUCT_ACCESSOR:
       return deserialize(dataProductAccessorModelSchema, json);
+    case V1_ClassInstanceType.SQL_ACCESSOR:
+      return deserialize(sqlAccessorModelSchema, json);
     default: {
       const deserializers = plugins.flatMap(
         (plugin) =>
@@ -910,6 +918,8 @@ export function V1_serializeClassInstanceValue(
     return serialize(relationStoreAccessorModelSchema, protocol);
   } else if (protocol instanceof V1_DataProductAccessor) {
     return serialize(dataProductAccessorModelSchema, protocol);
+  } else if (protocol instanceof V1_SQLAccessor) {
+    return serialize(sqlAccessorModelSchema, protocol);
   }
   const serializers = plugins.flatMap(
     (plugin) =>

package/src/index.ts

@@ -287,6 +287,7 @@ export { V1_TestAuthenticationStrategy } from './graph-manager/protocol/pure/v1/
 export {
   V1_RelationStoreAccessor,
   V1_DataProductAccessor,
+  V1_SQLAccessor,
 } from './graph-manager/protocol/pure/v1/model/valueSpecification/raw/classInstance/relation/V1_RelationStoreAccessor.js';
 export { V1_PackageableConnection } from './graph-manager/protocol/pure/v1/model/packageableElements/connection/V1_PackageableConnection.js';
 export {
@@ -398,6 +399,7 @@ export {
 } from './graph-manager/protocol/pure/v1/lakehouse/subscriptions/V1_ConsumerSubscriptions.js';
 export {
   V1_DataProductArtifact,
+  V1_NativeModelExecutionContextInfo,
   V1_AccessPointImplementation,
   V1_AccessPointGroupInfo,
   V1_SampleQueryInfo,
@@ -780,8 +782,6 @@ export {
   SupportInfo,
   ModelAccessPointGroup,
   InLineSampleQuery,
-  NativeModelAccess,
-  NativeModelExecutionContext,
   PackageableElementSampleQuery,
   SampleQuery,
   DataProductRuntimeInfo,
@@ -793,6 +793,8 @@ export {
   Expertise,
   ExternalDataProductType,
   type DataProductElement,
+  NativeModelAccess,
+  NativeModelExecutionContext,
   DataProductOperationalMetadata,
 } from './graph/metamodel/pure/dataProduct/DataProduct.js';
 export * from './graph-manager/action/changeDetection/DataProductObserveHelper.js';
@@ -971,6 +973,7 @@ export {
   DatasetEntitlementAccessApprovedReport,
   DatasetEntitlementUnsupportedReport,
 } from './graph-manager/action/analytics/StoreEntitlementAnalysis.js';
+export * from './graph-manager/helpers/DataProductHelper.js';
 export {
   FunctionAnalysisInfo,
   FunctionAnalysisParameterInfo,

package/tsconfig.json

@@ -435,6 +435,7 @@
     "./src/graph-manager/extensions/Testable_PureGraphManagerPlugin_Extension.ts",
     "./src/graph-manager/helpers/DSL_Data_GraphManagerHelper.ts",
     "./src/graph-manager/helpers/DSL_ExternalFormat_GraphManagerHelper.ts",
+    "./src/graph-manager/helpers/DataProductHelper.ts",
     "./src/graph-manager/helpers/ValueSpecificationGraphManagerHelper.ts",
     "./src/graph-manager/protocol/pure/PureGraphManagerBuilder.ts",
     "./src/graph-manager/protocol/pure/PureProtocolProcessorPlugin.ts",