@finos/legend-graph 32.1.67 → 32.2.1

package/src/graph/MetaModelConst.ts

@@ -119,6 +119,8 @@ export const AUTO_IMPORTS = [
   // 'meta::pure::tds',
   // 'meta::pure::tools',
   'meta::pure::profiles',
+  'meta::pure::metamodel::relation',
+  'meta::pure::metamodel::variant',
 ];
 
 export enum CORE_PURE_PATH {

package/src/graph/PureModel.ts

@@ -180,6 +180,8 @@ export class SystemModel extends BasicModel {
    *
    * `meta::pure::metamodel::type::Any` covers `meta::pure::metamodel::type`
    * `meta::pure::profiles::doc` covers `meta::pure::profiles`
+   * `meta::pure::metamodel::relation::Relation` covers `meta::pure::metamodel::relation`
+   * `meta::pure::metamodel::variant::Variant` covers `meta::pure::metamodel::variant`
    */
   initializeAutoImports(): void {
     this.autoImports = AUTO_IMPORTS.map((_package) =>

package/src/graph/metamodel/pure/dataProduct/DataProduct.ts

@@ -253,8 +253,33 @@ export abstract class DataProductIcon implements Hashable {
   abstract get hashCode(): string;
 }
 
-export class EmbeddedImageIcon extends DataProductIcon implements Hashable {
-  imageUrl: string;
+export class DataProductLibraryIcon
+  extends DataProductIcon
+  implements Hashable
+{
+  libraryId!: string;
+  iconId!: string;
+
+  constructor(libraryId: string, iconId: string) {
+    super();
+    this.libraryId = libraryId;
+    this.iconId = iconId;
+  }
+
+  get hashCode(): string {
+    return hashArray([
+      CORE_HASH_STRUCTURE.DATA_PRODUCT_ICON_LIBRARY,
+      this.libraryId,
+      this.iconId,
+    ]);
+  }
+}
+
+export class DataProductEmbeddedImageIcon
+  extends DataProductIcon
+  implements Hashable
+{
+  imageUrl!: string;
 
   constructor(imageUrl: string) {
     super();
@@ -269,25 +294,42 @@ export class EmbeddedImageIcon extends DataProductIcon implements Hashable {
   }
 }
 
-export class LibraryIcon extends DataProductIcon implements Hashable {
-  libraryId: string;
-  iconId: string;
+// handle incoming icons not yet modeled
+export class UnknownDataProductIcon
+  extends DataProductIcon
+  implements Hashable
+{
+  content!: PlainObject;
 
-  constructor(libraryId: string, iconId: string) {
+  constructor(content: PlainObject) {
     super();
-    this.libraryId = libraryId;
-    this.iconId = iconId;
+    this.content = content;
   }
 
-  get hashCode(): string {
+  override get hashCode(): string {
     return hashArray([
-      CORE_HASH_STRUCTURE.DATA_PRODUCT_ICON_LIBRARY,
-      this.libraryId,
-      this.iconId,
+      CORE_HASH_STRUCTURE.INTERNAL__UNKNOWN_DATA_PRODUCT_ICON,
+      hashObjectWithoutSourceInformation(this.content),
     ]);
   }
 }
 
+export abstract class DataProductType implements Hashable {
+  get hashCode(): string {
+    return hashArray([CORE_HASH_STRUCTURE.DATA_PRODUCT_TYPE]);
+  }
+}
+
+export class InternalDataProductType extends DataProductType {}
+
+export class ExternalDataProductType extends DataProductType {
+  link!: DataProductLink;
+
+  override get hashCode(): string {
+    return hashArray([CORE_HASH_STRUCTURE.DATA_PRODUCT_TYPE, this.link]);
+  }
+}
+
 export class DataProduct extends PackageableElement {
   title: string | undefined;
   description: string | undefined;
@@ -296,6 +338,7 @@ export class DataProduct extends PackageableElement {
   icon: DataProductIcon | undefined;
   accessPointGroups: AccessPointGroup[] = [];
   supportInfo: SupportInfo | undefined;
+  type: DataProductType | undefined;
 
   override accept_PackageableElementVisitor<T>(
     visitor: PackageableElementVisitor<T>,
@@ -313,6 +356,7 @@ export class DataProduct extends PackageableElement {
       this.icon ?? '',
       hashArray(this.accessPointGroups),
       this.supportInfo ?? '',
+      this.type ?? '',
       hashArray(this.stereotypes.map((val) => val.pointerHashCode)),
       hashArray(this.taggedValues),
     ]);

package/src/graph-manager/AbstractPureGraphManager.ts

@@ -170,6 +170,11 @@ export interface ExecutionOptions {
   abortController?: AbortController | undefined;
   preservedResponseHeadersList?: string[];
   tracingtags?: PlainObject | undefined;
+  /**
+   * If set will force lambda to append ->from(mapping/runtime)
+   * instead of providing in separate input property
+   */
+  forceFromExpression?: boolean | undefined;
 }
 
 export interface ServiceRegistrationOptions {

package/src/graph-manager/action/changeDetection/DataProductObserveHelper.ts

@@ -16,13 +16,17 @@
 
 import { makeObservable, observable, override } from 'mobx';
 import {
-  LakehouseAccessPoint,
-  type Email,
-  type SupportInfo,
   type AccessPoint,
   type AccessPointGroup,
   type DataProduct,
+  type DataProductIcon,
   type DataProductLink,
+  type Email,
+  type SupportInfo,
+  DataProductEmbeddedImageIcon,
+  DataProductLibraryIcon,
+  LakehouseAccessPoint,
+  UnknownDataProductIcon,
 } from '../../../graph/metamodel/pure/dataProduct/DataProduct.js';
 import {
   observe_Abstract_PackageableElement,
@@ -109,6 +113,26 @@ export const observe_AccessPointGroup = skipObserved(
   },
 );
 
+export const observe_DataProductIcon = skipObserved(
+  (metamodel: DataProductIcon): DataProductIcon => {
+    if (metamodel instanceof DataProductLibraryIcon) {
+      makeObservable(metamodel, {
+        libraryId: observable,
+        iconId: observable,
+      });
+    } else if (metamodel instanceof DataProductEmbeddedImageIcon) {
+      makeObservable(metamodel, {
+        imageUrl: observable,
+      });
+    } else if (metamodel instanceof UnknownDataProductIcon) {
+      makeObservable(metamodel, {
+        content: observable,
+      });
+    }
+    return metamodel;
+  },
+);
+
 export const observe_DataProduct = skipObserved(
   (metamodel: DataProduct): DataProduct => {
     observe_Abstract_PackageableElement(metamodel);
@@ -119,11 +143,15 @@ export const observe_DataProduct = skipObserved(
       title: observable,
       description: observable,
       supportInfo: observable,
+      icon: observable,
     });
 
     if (metamodel.supportInfo) {
       observe_SupportInfo(metamodel.supportInfo);
     }
+    if (metamodel.icon) {
+      observe_DataProductIcon(metamodel.icon);
+    }
     metamodel.accessPointGroups.forEach(observe_AccessPointGroup);
     return metamodel;
   },

package/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts

@@ -2694,7 +2694,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     lambda: RawLambda,
     runtime: Runtime | undefined,
     clientVersion: string | undefined,
-    parameterValues?: ParameterValue[],
+    options?: ExecutionOptions,
   ): V1_ExecuteInput =>
     this.createExecutionInputWithPureModelContext(
       graph.origin
@@ -2705,7 +2705,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
       runtime,
       clientVersion,
       new V1_ExecuteInput(),
-      parameterValues,
+      options,
     );
 
   public createLineageInput = (
@@ -2714,7 +2714,6 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     lambda: RawLambda,
     runtime: Runtime | undefined,
     clientVersion: string | undefined,
-    parameterValues?: ParameterValue[],
   ): V1_LineageInput => {
     const executionInput = this.createExecutionInput(
       graph,
@@ -2722,7 +2721,6 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
       lambda,
       runtime,
       clientVersion,
-      parameterValues,
     );
 
     const lineageInput = new V1_LineageInput();
@@ -2741,31 +2739,93 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     runtime: Runtime | undefined,
     clientVersion: string | undefined,
     executeInput: V1_ExecuteInput,
-    parameterValues?: ParameterValue[],
+    options?: ExecutionOptions | undefined,
   ): V1_ExecuteInput => {
+    let mappingPath = mapping?.path;
+    const lambdaToExecute = V1_transformRawLambda(
+      lambda,
+      new V1_GraphTransformerContextBuilder(
+        this.pluginManager.getPureProtocolProcessorPlugins(),
+      ).build(),
+    );
+    let runtimeToExecute = runtime
+      ? V1_transformRuntime(
+          runtime,
+          new V1_GraphTransformerContextBuilder(
+            this.pluginManager.getPureProtocolProcessorPlugins(),
+          ).build(),
+        )
+      : undefined;
+    if (options?.forceFromExpression) {
+      if (runtimeToExecute instanceof V1_RuntimePointer) {
+        const runtimePath = runtimeToExecute.runtime;
+        const body = lambdaToExecute.body;
+        if (body && runtimePath.length) {
+          // handles multi expression lambda (i.e let statement executions)
+          if (Array.isArray(body)) {
+            const expressions = body as object[];
+            if (expressions.length) {
+              const lastIdx = expressions.length - 1;
+              const lastBody = expressions[lastIdx];
+              if (lastBody) {
+                const fromExpression = this.__fromExpression(
+                  mappingPath,
+                  runtimePath,
+                  lastBody,
+                );
+                expressions[lastIdx] = fromExpression;
+                mappingPath = undefined;
+                runtimeToExecute = undefined;
+              }
+            }
+          } else {
+            const fromExpression = this.__fromExpression(
+              mappingPath,
+              runtimePath,
+              body,
+            );
+            lambdaToExecute.body = fromExpression;
+            mappingPath = undefined;
+            runtimeToExecute = undefined;
+          }
+        }
+      }
+    }
     return this.createExecutionInputWithPureModelContextWithV1(
       data,
-      mapping?.path,
-      V1_transformRawLambda(
-        lambda,
-        new V1_GraphTransformerContextBuilder(
-          this.pluginManager.getPureProtocolProcessorPlugins(),
-        ).build(),
-      ),
-      runtime
-        ? V1_transformRuntime(
-            runtime,
-            new V1_GraphTransformerContextBuilder(
-              this.pluginManager.getPureProtocolProcessorPlugins(),
-            ).build(),
-          )
-        : undefined,
+      mappingPath,
+      lambdaToExecute,
+      runtimeToExecute,
       clientVersion,
       executeInput,
-      parameterValues,
+      options?.parameterValues,
     );
   };
 
+  __fromExpression = (
+    mapping: string | undefined,
+    runtime: string,
+    body: object,
+  ): object => {
+    return {
+      _type: 'func',
+      function: 'from',
+      parameters: [
+        body,
+        mapping
+          ? {
+              _type: 'packageableElementPtr',
+              fullPath: mapping,
+            }
+          : undefined,
+        {
+          _type: 'packageableElementPtr',
+          fullPath: runtime,
+        },
+      ].filter(isNonNullable),
+    };
+  };
+
   private createExecutionInputWithPureModelContextWithV1 = (
     data: V1_PureModelContext,
     mapping: string | undefined,
@@ -2857,7 +2917,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
           this.engine.config.useDevClientProtocol
             ? V1_PureGraphManager.DEV_PROTOCOL_VERSION
             : V1_PureGraphManager.PROD_PROTOCOL_VERSION,
-          options?.parameterValues,
+          options,
         ),
       options,
       _report,
@@ -2915,7 +2975,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
       this.engine.config.useDevClientProtocol
         ? V1_PureGraphManager.DEV_PROTOCOL_VERSION
         : V1_PureGraphManager.PROD_PROTOCOL_VERSION,
-      options?.parameterValues,
+      options,
     );
     stopWatch.record(GRAPH_MANAGER_EVENT.V1_ENGINE_OPERATION_INPUT__SUCCESS);
     const stream = await this.engine.exportData(input, options, contentType);
@@ -2984,7 +3044,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
         testInfo.runtime,
         V1_PureGraphManager.PROD_PROTOCOL_VERSION,
         new V1_ExecuteInput(),
-        options?.parameterValues,
+        options,
       );
       const result = V1_buildExecutionResult(
         (await this.engine.runQuery(input, options)).executionResult,
@@ -3096,7 +3156,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
       runtime,
       V1_PureGraphManager.DEV_PROTOCOL_VERSION,
       testDataGenerationExecuteInput,
-      options?.parameterValues,
+      options,
     );
     testDataGenerationExecuteInput.parameters = parameters;
     testDataGenerationExecuteInput.hashStrings = Boolean(
@@ -3144,7 +3204,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
       runtime,
       V1_PureGraphManager.DEV_PROTOCOL_VERSION,
       testDataGenerationExecuteWithSeedInput,
-      options?.parameterValues,
+      options,
     );
     testDataGenerationExecuteWithSeedInput.hashStrings = Boolean(
       options?.anonymizeGeneratedData,

package/src/graph-manager/protocol/pure/v1/lakehouse/entitlements/V1_EntitlementsDataProduct.ts

@@ -74,5 +74,5 @@ export class V1_EntitlementsDataProductDetails {
 }
 
 export class V1_EntitlementsDataProductDetailsResponse {
-  dataProducts: V1_EntitlementsDataProductDetails[] = [];
+  dataProducts?: V1_EntitlementsDataProductDetails[];
 }

package/src/graph-manager/protocol/pure/v1/model/packageableElements/dataProduct/V1_DataProduct.ts

@@ -34,6 +34,11 @@ import type { V1_TaggedValue } from '../domain/V1_TaggedValue.js';
 
 export const V1_DATA_PRODUCT_ELEMENT_PROTOCOL_TYPE = 'dataProduct';
 
+export enum V1_DataProductTypeValue {
+  INTERNAL = 'internalDataProductType',
+  EXTERNAL = 'externalDataProductType',
+}
+
 export abstract class V1_AccessPoint implements Hashable {
   id!: string;
   description: string | undefined;
@@ -222,13 +227,29 @@ export abstract class V1_DataProductIcon implements Hashable {
   abstract get hashCode(): string;
 }
 
-export class V1_EmbeddedImageIcon
+export class V1_DataProductLibraryIcon
+  extends V1_DataProductIcon
+  implements Hashable
+{
+  libraryId!: string;
+  iconId!: string;
+
+  override get hashCode(): string {
+    return hashArray([
+      CORE_HASH_STRUCTURE.DATA_PRODUCT_ICON_LIBRARY,
+      this.libraryId,
+      this.iconId,
+    ]);
+  }
+}
+
+export class V1_DataProductEmbeddedImageIcon
   extends V1_DataProductIcon
   implements Hashable
 {
   imageUrl!: string; // base64 encoded image content
 
-  get hashCode(): string {
+  override get hashCode(): string {
     return hashArray([
       CORE_HASH_STRUCTURE.DATA_PRODUCT_ICON_EMBEDDED_IMAGE,
       this.imageUrl,
@@ -236,18 +257,40 @@ export class V1_EmbeddedImageIcon
   }
 }
 
-export class V1_LibraryIcon extends V1_DataProductIcon implements Hashable {
-  libraryId!: string;
-  iconId!: string;
-  get hashCode(): string {
+// handle incoming icons not yet modeled
+export class V1_UnknownDataProductIcon
+  extends V1_DataProductIcon
+  implements Hashable
+{
+  content!: PlainObject;
+
+  override get hashCode(): string {
     return hashArray([
-      CORE_HASH_STRUCTURE.DATA_PRODUCT_ICON_LIBRARY,
-      this.libraryId,
-      this.iconId,
+      CORE_HASH_STRUCTURE.INTERNAL__UNKNOWN_DATA_PRODUCT_ICON,
+      hashObjectWithoutSourceInformation(this.content),
     ]);
   }
 }
 
+export abstract class V1_DataProductType implements Hashable {
+  get hashCode(): string {
+    return hashArray([CORE_HASH_STRUCTURE.DATA_PRODUCT_TYPE]);
+  }
+}
+
+export class V1_InternalDataProductType extends V1_DataProductType {
+  _type = V1_DataProductTypeValue.INTERNAL;
+}
+
+export class V1_ExternalDataProductType extends V1_DataProductType {
+  _type = V1_DataProductTypeValue.EXTERNAL;
+  link!: V1_DataProductLink;
+
+  override get hashCode(): string {
+    return hashArray([CORE_HASH_STRUCTURE.DATA_PRODUCT_TYPE, this.link]);
+  }
+}
+
 export class V1_DataProduct extends V1_PackageableElement implements Hashable {
   title: string | undefined;
   description: string | undefined;
@@ -256,6 +299,7 @@ export class V1_DataProduct extends V1_PackageableElement implements Hashable {
   icon: V1_DataProductIcon | undefined;
   accessPointGroups: V1_AccessPointGroup[] = [];
   supportInfo: V1_SupportInfo | undefined;
+  type: V1_DataProductType | undefined;
   stereotypes: V1_StereotypePtr[] = [];
   taggedValues: V1_TaggedValue[] = [];
 
@@ -269,6 +313,7 @@ export class V1_DataProduct extends V1_PackageableElement implements Hashable {
       this.icon ?? '',
       hashArray(this.accessPointGroups),
       this.supportInfo ?? '',
+      this.type ?? '',
       hashArray(this.stereotypes),
       hashArray(this.taggedValues),
     ]);

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/from/V1_DataProductTransformer.ts

@@ -17,29 +17,37 @@
 import { UnsupportedOperationError } from '@finos/legend-shared';
 import {
   type AccessPoint,
-  LakehouseAccessPoint,
-  UnknownAccessPoint,
   type DataProduct,
-  LibraryIcon,
-  EmbeddedImageIcon,
+  type DataProductIcon,
+  DataProductEmbeddedImageIcon,
+  DataProductLibraryIcon,
+  LakehouseAccessPoint,
   ModelAccessPointGroup,
+  ExternalDataProductType,
+  InternalDataProductType,
+  UnknownAccessPoint,
+  UnknownDataProductIcon,
 } from '../../../../../../../graph/metamodel/pure/dataProduct/DataProduct.js';
 import {
   type V1_AccessPoint,
+  type V1_DataProductIcon,
+  type V1_DataProductRegion,
+  type V1_DeliveryFrequency,
   V1_AccessPointGroup,
   V1_DataProduct,
-  type V1_DeliveryFrequency,
+  V1_DataProductDiagram,
+  V1_DataProductEmbeddedImageIcon,
+  V1_DataProductLibraryIcon,
+  V1_DataProductRuntimeInfo,
+  V1_ElementScope,
   V1_Email,
-  V1_EmbeddedImageIcon,
   V1_LakehouseAccessPoint,
-  V1_LibraryIcon,
+  V1_ModelAccessPointGroup,
   V1_SupportInfo,
   V1_UnknownAccessPoint,
-  type V1_DataProductRegion,
-  V1_ModelAccessPointGroup,
-  V1_DataProductRuntimeInfo,
-  V1_ElementScope,
-  V1_DataProductDiagram,
+  V1_UnknownDataProductIcon,
+  V1_InternalDataProductType,
+  V1_ExternalDataProductType,
 } from '../../../model/packageableElements/dataProduct/V1_DataProduct.js';
 import { V1_initPackageableElement } from './V1_CoreTransformerHelper.js';
 import { V1_transformRawLambda } from './V1_RawValueSpecificationTransformer.js';
@@ -74,6 +82,28 @@ const transformAccessPoint = (
   );
 };
 
+const transformDataProductIcon = (
+  icon: DataProductIcon,
+): V1_DataProductIcon => {
+  if (icon instanceof DataProductLibraryIcon) {
+    const dataProductLibraryIcon = new V1_DataProductLibraryIcon();
+    dataProductLibraryIcon.libraryId = icon.libraryId;
+    dataProductLibraryIcon.iconId = icon.iconId;
+    return dataProductLibraryIcon;
+  } else if (icon instanceof DataProductEmbeddedImageIcon) {
+    const dataProductEmbeddedImageIcon = new V1_DataProductEmbeddedImageIcon();
+    dataProductEmbeddedImageIcon.imageUrl = icon.imageUrl;
+    return dataProductEmbeddedImageIcon;
+  } else if (icon instanceof UnknownDataProductIcon) {
+    const unknownDataProductIcon = new V1_UnknownDataProductIcon();
+    unknownDataProductIcon.content = icon.content;
+    return unknownDataProductIcon;
+  }
+  throw new UnsupportedOperationError(
+    `Can't transform data product icon type: ${icon}`,
+  );
+};
+
 export const V1_transformDataProduct = (
   element: DataProduct,
   context: V1_GraphTransformerContext,
@@ -88,16 +118,20 @@ export const V1_transformDataProduct = (
   product.coverageRegions = element.coverageRegions as
     | V1_DataProductRegion[]
     | undefined;
-  if (element.icon instanceof LibraryIcon) {
-    const libIcon = new V1_LibraryIcon();
-    libIcon.libraryId = element.icon.libraryId;
-    libIcon.iconId = element.icon.iconId;
-    product.icon = libIcon;
-  } else if (element.icon instanceof EmbeddedImageIcon) {
-    const embeddedIcon = new V1_EmbeddedImageIcon();
-    embeddedIcon.imageUrl = element.icon.imageUrl;
-    product.icon = embeddedIcon;
+  if (element.type instanceof InternalDataProductType) {
+    product.type = new V1_InternalDataProductType();
+  } else if (element.type instanceof ExternalDataProductType) {
+    const dataProductType = new V1_ExternalDataProductType();
+    dataProductType.link = element.type.link;
+    product.type = dataProductType;
   }
+
+  if (!element.icon) {
+    product.icon = undefined;
+  } else {
+    product.icon = transformDataProductIcon(element.icon);
+  }
+
   if (!element.supportInfo) {
     product.supportInfo = undefined;
   } else {
@@ -114,6 +148,7 @@ export const V1_transformDataProduct = (
     });
     product.supportInfo = supportInfo;
   }
+
   product.accessPointGroups = element.accessPointGroups.map(
     (metamodelGroup) => {
       if (metamodelGroup instanceof ModelAccessPointGroup) {
@@ -181,9 +216,11 @@ export const V1_transformDataProduct = (
       return group;
     },
   );
+
   product.stereotypes = element.stereotypes.map(V1_transformStereotype);
   product.taggedValues = element.taggedValues.map((taggedValue) =>
     V1_transformTaggedValue(taggedValue),
   );
+
   return product;
 };

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/to/V1_ElementSecondPassBuilder.ts

@@ -117,11 +117,17 @@ import {
 } from './helpers/V1_FunctionActivatorBuilderHelper.js';
 import type { V1_INTERNAL__UnknownElement } from '../../../model/packageableElements/V1_INTERNAL__UnknownElement.js';
 import type { V1_HostedService } from '../../../model/packageableElements/function/V1_HostedService.js';
-import { type V1_DataProduct } from '../../../model/packageableElements/dataProduct/V1_DataProduct.js';
+import {
+  V1_ExternalDataProductType,
+  V1_InternalDataProductType,
+  type V1_DataProduct,
+} from '../../../model/packageableElements/dataProduct/V1_DataProduct.js';
 import {
   type DataProduct_DeliveryFrequency,
   type DataProduct_Region,
   Email,
+  ExternalDataProductType,
+  InternalDataProductType,
   SupportInfo,
 } from '../../../../../../../graph/metamodel/pure/dataProduct/DataProduct.js';
 import {
@@ -726,6 +732,13 @@ export class V1_ElementSecondPassBuilder
       });
       dataProduct.supportInfo = supportInfo;
     }
+    if (element.type instanceof V1_InternalDataProductType) {
+      dataProduct.type = new InternalDataProductType();
+    } else if (element.type instanceof V1_ExternalDataProductType) {
+      const dataProductType = new ExternalDataProductType();
+      dataProductType.link = element.type.link;
+      dataProduct.type = dataProductType;
+    }
     dataProduct.deliveryFrequency = element.deliveryFrequency as
       | DataProduct_DeliveryFrequency
       | undefined;