@finos/legend-graph 32.6.1 → 32.6.3

package/src/graph/MetaModelConst.ts

@@ -85,6 +85,7 @@ export enum ATOMIC_TEST_TYPE {
   Service_Test = 'serviceTest',
   Mapping_Test = 'mappingTest',
   Function_Test = 'functionTest',
+  DataProduct_AccessPoint_Test = 'accessPointTest',
 }
 
 // NOTE: the list of auto-import are kept in `m3.pure` file in `finos/legend-pure`,
@@ -210,6 +211,15 @@ export const DATE_FORMAT = 'yyyy-MM-dd';
 
 // Generation Const
 export const DEFAULT_GENERATION_PACKAGE = 'generated';
+
+export enum MILESTONE_INGEST_COLUMNS {
+  INGEST_LAKE_IN_ID = 'LAKE_IN_ID',
+  INGEST_LAKE_OUT_ID = 'LAKE_OUT_ID',
+  INGEST_LAKE_DIGEST = 'LAKE_DIGEST',
+  INGEST_LAKE_FROM = 'LAKE_FROM',
+  INGEST_LAKE_THRU = 'LAKE_THRU',
+}
+
 // Query
 export const QUERY_PROFILE_TAG_CLASS = 'class';
 export const QUERY_PROFILE_PATH = 'meta::pure::profiles::query';

package/src/graph/metamodel/pure/data/DataResolver.ts

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { hashArray, type Hashable } from '@finos/legend-shared';
+import type { PackageableElementReference } from '../packageableElements/PackageableElementReference.js';
+import type { EmbeddedData } from './EmbeddedData.js';
+import { CORE_HASH_STRUCTURE } from '../../../Core_HashUtils.js';
+import type { PackageableElement } from '../packageableElements/PackageableElement.js';
+
+export abstract class DataResolver implements Hashable {
+  abstract get hashCode(): string;
+}
+
+export class BaseDataResolver extends DataResolver implements Hashable {
+  data!: EmbeddedData;
+  element!: PackageableElementReference<PackageableElement>;
+
+  get hashCode(): string {
+    return hashArray([
+      CORE_HASH_STRUCTURE.BASE_DATA_RESOLVER,
+      this.data,
+      this.element.valueForSerialization ?? '',
+    ]);
+  }
+}
+
+export class ReferenceDataResolver extends DataResolver implements Hashable {
+  element!: PackageableElementReference<PackageableElement>;
+
+  get hashCode(): string {
+    return hashArray([
+      CORE_HASH_STRUCTURE.REFERENCE_DATA_RESOLVER,
+      this.element.valueForSerialization ?? '',
+    ]);
+  }
+}

package/src/graph/metamodel/pure/data/EmbeddedData.ts

@@ -135,15 +135,15 @@ export class RelationRowTestData implements Hashable {
 }
 
 export class RelationElement implements Hashable {
-  paths!: string[];
   columns!: string[];
+  paths!: string[];
   rows: RelationRowTestData[] = [];
 
   get hashCode(): string {
     return hashArray([
       CORE_HASH_STRUCTURE.RELATION_ELEMENT,
-      hashArray(this.paths),
       hashArray(this.columns),
+      hashArray(this.paths),
       hashArray(this.rows),
     ]);
   }

package/src/graph/metamodel/pure/dataProduct/DataProduct.ts

@@ -38,10 +38,13 @@ import type { Class } from '../packageableElements/domain/Class.js';
 import type { Enumeration } from '../packageableElements/domain/Enumeration.js';
 import type { Association } from '../packageableElements/domain/Association.js';
 import type { EmbeddedData } from '../data/EmbeddedData.js';
+import type { DataProductTestSuite } from './test/DataProductTestSuite.js';
+import type { Testable } from '../test/Testable.js';
 import { ConcreteFunctionDefinition } from '../packageableElements/function/ConcreteFunctionDefinition.js';
 import { generateFunctionPrettyName } from '../../../helpers/PureLanguageHelper.js';
 import type { StereotypeReference } from '../packageableElements/domain/StereotypeReference.js';
 import type { AppDirNode } from '../packageableElements/ingest/IngestDefinition.js';
+import type { RelationType } from '../packageableElements/relation/RelationType.js';
 
 export abstract class AccessPoint implements Hashable {
   id: string;
@@ -101,6 +104,7 @@ export class LakehouseAccessPoint extends AccessPoint {
   classification: string | undefined;
   func: RawLambda;
   reproducible: boolean | undefined;
+  __internal__RelationType: RelationType | undefined;
 
   constructor(
     id: string,
@@ -507,7 +511,7 @@ export class AppDirOwner extends DataProductOwner implements Hashable {
   }
 }
 
-export class DataProduct extends PackageableElement {
+export class DataProduct extends PackageableElement implements Testable {
   title: string | undefined;
   description: string | undefined;
   icon: DataProductIcon | undefined;
@@ -518,6 +522,7 @@ export class DataProduct extends PackageableElement {
   sampleValues: EmbeddedData[] | undefined;
   operationalMetadata: DataProductOperationalMetadata | undefined;
   owner: DataProductOwner | undefined;
+  tests: DataProductTestSuite[] = [];
 
   override accept_PackageableElementVisitor<T>(
     visitor: PackageableElementVisitor<T>,
@@ -540,6 +545,7 @@ export class DataProduct extends PackageableElement {
       hashArray(this.sampleValues ?? []),
       this.operationalMetadata ?? '',
       this.owner ?? '',
+      hashArray(this.tests),
     ]);
   }
 }

package/src/graph/metamodel/pure/dataProduct/test/DataProductAccessPointTest.ts

@@ -0,0 +1,33 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { hashArray, type Hashable } from '@finos/legend-shared';
+import { AtomicTest } from '../../test/Test.js';
+import { CORE_HASH_STRUCTURE } from '../../../../Core_HashUtils.js';
+
+export class DataProductAccessPointTest extends AtomicTest implements Hashable {
+  accessPointId!: string;
+
+  get hashCode(): string {
+    return hashArray([
+      CORE_HASH_STRUCTURE.DATA_PRODUCT_ACCESS_POINT_TEST,
+      this.id,
+      this.doc ?? '',
+      this.accessPointId,
+      hashArray(this.assertions),
+    ]);
+  }
+}

package/src/graph/metamodel/pure/dataProduct/test/DataProductTestSuite.ts

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { hashArray, type Hashable } from '@finos/legend-shared';
+import { TestSuite } from '../../test/Test.js';
+import type { DataResolver } from '../../data/DataResolver.js';
+import { CORE_HASH_STRUCTURE } from '../../../../Core_HashUtils.js';
+
+export class DataProductTestSuite extends TestSuite implements Hashable {
+  testData: DataResolver[] | undefined;
+
+  override get hashCode(): string {
+    return hashArray([
+      CORE_HASH_STRUCTURE.DATA_PRODUCT_TEST_SUITE,
+      this.id,
+      this.doc ?? '',
+      this.testData?.length ? hashArray(this.testData) : '',
+      hashArray(this.tests),
+    ]);
+  }
+}

package/src/graph/metamodel/pure/packageableElements/relation/Accessor.ts

@@ -16,8 +16,8 @@
 
 import { hashArray, isNonNullable, type Hashable } from '@finos/legend-shared';
 import type { RelationType } from './RelationType.js';
-import type { DataProduct } from '../../dataProduct/DataProduct.js';
-import type { IngestDefinition } from '../ingest/IngestDefinition.js';
+import { DataProduct } from '../../dataProduct/DataProduct.js';
+import { IngestDefinition } from '../ingest/IngestDefinition.js';
 import type { Database } from '../store/relational/model/Database.js';
 import { InstanceValue } from '../../valueSpecification/InstanceValue.js';
 import { Multiplicity } from '../domain/Multiplicity.js';
@@ -93,7 +93,7 @@ export class IngestionAccessor extends Accessor {
     return 'Ingestion Source';
   }
   override get accessorLabel(): string {
-    return 'Data Set';
+    return 'Dataset';
   }
 
   override get schemaLabel(): string | undefined {
@@ -116,6 +116,24 @@ export class RelationalStoreAccessor extends Accessor implements Hashable {
   }
 }
 
+/**
+ * Returns the accessor item label for the given element owner.
+ * Labels mirror the corresponding Accessor subclass `accessorLabel` getters,
+ * which are the authoritative source but require a full Accessor instance to
+ * call. Use this helper when only the owner element is available.
+ */
+export const getAccessorItemLabelForElement = (
+  element: AccessorOwner,
+): string => {
+  if (element instanceof DataProduct) {
+    return 'Access Point'; // DataProductAccessor.accessorLabel
+  }
+  if (element instanceof IngestDefinition) {
+    return 'Data Set'; // IngestionAccessor.accessorLabel
+  }
+  return 'Table'; // RelationalStoreAccessor.accessorLabel
+};
+
 export class AccessorInstanceValue extends InstanceValue implements Hashable {
   override values: Accessor[] = [];
 

package/src/graph/metamodel/pure/test/assertion/EqualToRelation.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { hashArray } from '@finos/legend-shared';
+import { CORE_HASH_STRUCTURE } from '../../../../../graph/Core_HashUtils.js';
+import type { RelationElement } from '../../data/EmbeddedData.js';
+import { type TestAssertionVisitor, TestAssertion } from './TestAssertion.js';
+
+export class EqualToRelation extends TestAssertion {
+  expected!: RelationElement;
+
+  get hashCode(): string {
+    return hashArray([
+      CORE_HASH_STRUCTURE.EQUAL_TO_RELATION,
+      this.id,
+      this.expected,
+    ]);
+  }
+
+  accept_TestAssertionVisitor<T>(visitor: TestAssertionVisitor<T>): T {
+    return visitor.visit_EqualToRelation(this);
+  }
+}

package/src/graph/metamodel/pure/test/assertion/TestAssertion.ts

@@ -18,6 +18,7 @@ import type { Hashable } from '@finos/legend-shared';
 import type { AtomicTest } from '../Test.js';
 import type { EqualTo } from './EqualTo.js';
 import type { EqualToJson } from './EqualToJson.js';
+import type { EqualToRelation } from './EqualToRelation.js';
 import type { EqualToTDS } from './EqualToTDS.js';
 
 export abstract class TestAssertion implements Hashable {
@@ -33,4 +34,5 @@ export interface TestAssertionVisitor<T> {
   visit_EqualTo(equal: EqualTo): T;
   visit_EqualToJSON(equal: EqualToJson): T;
   visit_EqualToTDS(equal: EqualToTDS): T;
+  visit_EqualToRelation(equal: EqualToRelation): T;
 }

package/src/graph/metamodel/pure/test/assertion/status/EqualToRelationAssertFail.ts

@@ -0,0 +1,22 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { AssertFail } from './AssertFail.js';
+
+export class EqualToRelationAssertFail extends AssertFail {
+  expected!: string;
+  actual!: string;
+}

package/src/graph-manager/AbstractPureGraphManager.ts

@@ -130,8 +130,15 @@ import type {
 import type { DeployProjectResponse } from './action/dev-metadata/DeployProjectResponse.js';
 import type { DataProductAnalysisQueryResult } from './action/analytics/data-product/DataProductAnalysis.js';
 import type { MetadataRequestOptions } from './action/dev-metadata/MetadataRequestOptions.js';
-import type { DataProductAccessType } from '../graph/metamodel/pure/dataProduct/DataProduct.js';
-import type { Accessor } from '../graph/metamodel/pure/packageableElements/relation/Accessor.js';
+import type {
+  DataProduct,
+  DataProductAccessType,
+} from '../graph/metamodel/pure/dataProduct/DataProduct.js';
+import type {
+  Accessor,
+  DataProductAccessor,
+} from '../graph/metamodel/pure/packageableElements/relation/Accessor.js';
+import type { IngestDefinition } from '../graph/metamodel/pure/packageableElements/ingest/IngestDefinition.js';
 
 export interface TEMPORARY__EngineSetupConfig {
   env: string;
@@ -449,6 +456,19 @@ export abstract class AbstractPureGraphManager {
     },
   ): Accessor | undefined;
 
+  abstract buildDataProductAccessor(
+    element: DataProduct,
+    graph: PureModel,
+    options?: {
+      tableName: string | undefined;
+    },
+  ): Promise<DataProductAccessor | undefined>;
+
+  abstract collectAccessorsInRawLambda(
+    rawLambda: RawLambda,
+    graph: PureModel,
+  ): Promise<Accessor[]>;
+
   // ------------------------------------------- SDLC -------------------------------------------
 
   abstract createSandboxProject(): Promise<{
@@ -797,7 +817,7 @@ export abstract class AbstractPureGraphManager {
     sql: string,
   ): Promise<string>;
 
-  // ------------------------------------------- Function -------------------------------------------
+  // ------------------------------------------- Function ------------------------------------createDataProductAccessor-------
 
   abstract getAvailableFunctionActivatorConfigurations(
     coreModel: CoreModel,
@@ -819,6 +839,12 @@ export abstract class AbstractPureGraphManager {
     graphData: GraphData,
   ): Promise<DeploymentResult>;
 
+  // --------------------------------------------- Ingeset Defintion --------------------------------------
+
+  abstract getIngestDefinitionDatasetNames(
+    ingestDefinition: IngestDefinition,
+  ): string[];
+
   // --------------------------------------------- Relational ---------------------------------------------
 
   abstract generateModelsFromDatabaseSpecification(

package/src/graph-manager/action/changeDetection/DataProductObserveHelper.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { makeObservable, observable, override } from 'mobx';
+import { computed, makeObservable, observable, override } from 'mobx';
 import {
   type AccessPoint,
   type AccessPointGroup,
@@ -35,16 +35,26 @@ import {
   type DataProductOperationalMetadata,
   AppDirOwner,
 } from '../../../graph/metamodel/pure/dataProduct/DataProduct.js';
-import { type AppDirNode } from '../../../graph/metamodel/pure/packageableElements/ingest/IngestDefinition.js';
+import type { AppDirNode } from '../../../graph/metamodel/pure/packageableElements/ingest/IngestDefinition.js';
+import {
+  type DataResolver,
+  BaseDataResolver,
+  ReferenceDataResolver,
+} from '../../../graph/metamodel/pure/data/DataResolver.js';
+import type { DataProductTestSuite } from '../../../graph/metamodel/pure/dataProduct/test/DataProductTestSuite.js';
 import {
   observe_Abstract_PackageableElement,
+  type ObserverContext,
   observe_PackageableElementReference,
   skipObserved,
+  skipObservedWithContext,
 } from './CoreObserverHelper.js';
 import {
   observe_StereotypeReference,
   observe_TaggedValue,
 } from './DomainObserverHelper.js';
+import { observe_EmbeddedData } from './DSL_Data_ObserverHelper.js';
+import { observe_AtomicTest } from './Testable_ObserverHelper.js';
 
 export const observe_AccessPoint = skipObserved(
   (metamodel: AccessPoint): AccessPoint => {
@@ -252,8 +262,48 @@ export const observe_AppDirOwner = skipObserved(
   },
 );
 
-export const observe_DataProduct = skipObserved(
-  (metamodel: DataProduct): DataProduct => {
+export const observe_DataResolver = skipObservedWithContext(
+  (metamodel: DataResolver, context: ObserverContext) => {
+    if (metamodel instanceof BaseDataResolver) {
+      makeObservable(metamodel, {
+        element: observable,
+        data: observable,
+        hashCode: computed,
+      });
+      observe_PackageableElementReference(metamodel.element);
+      observe_EmbeddedData(metamodel.data, context);
+    } else if (metamodel instanceof ReferenceDataResolver) {
+      makeObservable(metamodel, {
+        element: observable,
+        hashCode: computed,
+      });
+      observe_PackageableElementReference(metamodel.element);
+    }
+    return metamodel;
+  },
+);
+
+export const observe_DataProductTestSuite = skipObservedWithContext(
+  (
+    metamodel: DataProductTestSuite,
+    context: ObserverContext,
+  ): DataProductTestSuite => {
+    makeObservable(metamodel, {
+      id: observable,
+      tests: observable,
+      testData: observable,
+      hashCode: computed,
+    });
+    metamodel.tests.forEach((test) => observe_AtomicTest(test, context));
+    metamodel.testData?.forEach((testData) =>
+      observe_DataResolver(testData, context),
+    );
+    return metamodel;
+  },
+);
+
+export const observe_DataProduct = skipObservedWithContext(
+  (metamodel: DataProduct, context: ObserverContext): DataProduct => {
     observe_Abstract_PackageableElement(metamodel);
 
     makeObservable<DataProduct, '_elementHashCode'>(metamodel, {
@@ -266,6 +316,7 @@ export const observe_DataProduct = skipObserved(
       type: observable,
       operationalMetadata: observable,
       owner: observable,
+      tests: observable,
     });
 
     if (metamodel.supportInfo) {
@@ -281,6 +332,9 @@ export const observe_DataProduct = skipObserved(
       observe_AppDirOwner(metamodel.owner);
     }
     metamodel.accessPointGroups.forEach(observe_APG);
+    metamodel.tests.forEach((testSuite) =>
+      observe_DataProductTestSuite(testSuite, context),
+    );
     return metamodel;
   },
 );

package/src/graph-manager/action/changeDetection/PackageableElementObserver.ts

@@ -185,7 +185,7 @@ class PackageableElementObserver implements PackageableElementVisitor<void> {
   }
 
   visit_DataProduct(element: DataProduct): void {
-    observe_DataProduct(element);
+    observe_DataProduct(element, this.observerContext);
   }
 
   visit_Mapping(element: Mapping): void {

package/src/graph-manager/action/changeDetection/Testable_ObserverHelper.ts

@@ -15,6 +15,7 @@
  */
 
 import { computed, makeObservable, observable } from 'mobx';
+import { DataProductAccessPointTest } from '../../../graph/metamodel/pure/dataProduct/test/DataProductAccessPointTest.js';
 import { MappingTest } from '../../../graph/metamodel/pure/packageableElements/mapping/MappingTest.js';
 import { MappingTestSuite } from '../../../graph/metamodel/pure/packageableElements/mapping/MappingTestSuite.js';
 import { ServiceTest } from '../../../graph/metamodel/pure/packageableElements/service/ServiceTest.js';
@@ -41,6 +42,20 @@ import {
 import { FunctionTest } from '../../../graph/metamodel/pure/packageableElements/function/test/FunctionTest.js';
 import { observe_FunctionTest } from './DomainObserverHelper.js';
 
+export const observe_DataProductAccessPointTest = skipObserved(
+  (metamodel: DataProductAccessPointTest): DataProductAccessPointTest => {
+    makeObservable(metamodel, {
+      id: observable,
+      doc: observable,
+      accessPointId: observable,
+      assertions: observable,
+      hashCode: computed,
+    });
+    metamodel.assertions.forEach(observe_TestAssertion);
+    return metamodel;
+  },
+);
+
 export const observe_EqualTo = skipObserved((metamodel: EqualTo): EqualTo => {
   makeObservable(metamodel, {
     id: observable,
@@ -87,6 +102,8 @@ export function observe_AtomicTest(
     return observe_MappingTest(metamodel, context);
   } else if (metamodel instanceof FunctionTest) {
     return observe_FunctionTest(metamodel);
+  } else if (metamodel instanceof DataProductAccessPointTest) {
+    return observe_DataProductAccessPointTest(metamodel);
   }
   const extraAtomicTestBuilder = context.plugins.flatMap(
     (plugin) =>