@finos/legend-application-studio 28.21.29 → 28.21.30

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@finos/legend-application-studio",
-  "version": "28.21.29",
+  "version": "28.21.30",
   "description": "Legend Studio application core",
   "keywords": [
     "legend",
@@ -47,17 +47,17 @@
   },
   "dependencies": {
     "@dagrejs/dagre": "1.1.4",
-    "@finos/legend-application": "16.0.118",
-    "@finos/legend-art": "7.1.158",
-    "@finos/legend-code-editor": "2.0.193",
-    "@finos/legend-data-cube": "0.3.110",
-    "@finos/legend-extension-dsl-data-product": "0.0.103",
-    "@finos/legend-extension-dsl-diagram": "8.1.257",
-    "@finos/legend-graph": "32.6.24",
-    "@finos/legend-lego": "2.0.213",
-    "@finos/legend-query-builder": "4.18.32",
+    "@finos/legend-application": "16.0.119",
+    "@finos/legend-art": "7.1.159",
+    "@finos/legend-code-editor": "2.0.194",
+    "@finos/legend-data-cube": "0.3.111",
+    "@finos/legend-extension-dsl-data-product": "0.0.104",
+    "@finos/legend-extension-dsl-diagram": "8.1.258",
+    "@finos/legend-graph": "32.6.25",
+    "@finos/legend-lego": "2.0.214",
+    "@finos/legend-query-builder": "4.19.0",
     "@finos/legend-server-depot": "6.1.17",
-    "@finos/legend-server-lakehouse": "0.3.73",
+    "@finos/legend-server-lakehouse": "0.3.74",
     "@finos/legend-server-sdlc": "5.4.7",
     "@finos/legend-server-showcase": "0.2.68",
     "@finos/legend-shared": "11.0.27",

package/src/components/editor/editor-group/dataProduct/DataProductEditor.tsx

@@ -732,17 +732,6 @@ export const LakehouseDataProductAccessPointEditor = observer(
       );
     }, [lambdaEditorState, editorStore.applicationStore]);
 
-    // Load the relation columns for access points that already have sample
-    // values so mismatches surface on load. This only runs for rendered access
-    // points, keeping the number of concurrent calls bounded to the visible set.
-    useEffect(() => {
-      if (accessPointState.relationElementState !== undefined) {
-        flowResult(lambdaEditorState.updateLambdaRelationColumns()).catch(
-          editorStore.applicationStore.alertUnhandledError,
-        );
-      }
-    }, [accessPointState, lambdaEditorState, editorStore.applicationStore]);
-
     const handleDescriptionEdit = () => setEditingDescription(true);
     const handleDescriptionBlur = () => {
       setEditingDescription(false);
@@ -1785,6 +1774,20 @@ const AccessPointGroupEditor = observer(
       return () => intersectionObserver.disconnect();
     }, [accessPointCount, renderedCount]);
 
+    useEffect(() => {
+      flowResult(
+        productEditorState.batchUpdateLambdaRelationColumns(
+          lakehouseAccessPointStates.slice(0, renderedCount),
+        ),
+      ).catch(editorStore.applicationStore.alertUnhandledError);
+    }, [
+      renderedCount,
+      accessPointCount,
+      lakehouseAccessPointStates,
+      productEditorState,
+      editorStore.applicationStore.alertUnhandledError,
+    ]);
+
     const handleDescriptionEdit = () => setEditingDescription(true);
     const handleDescriptionBlur = () => {
       setEditingDescription(false);

package/src/components/editor/editor-group/ingest-editor/IngestDefinitionEditor.tsx

@@ -31,7 +31,9 @@ import {
   ModalHeaderActions,
   CustomSelectorInput,
   FlaskIcon,
+  SQLIcon,
   clsx,
+  PanelLoadingIndicator,
 } from '@finos/legend-art';
 import {
   INGEST_DEFINITION_TAB,
@@ -47,6 +49,7 @@ import { LEGEND_STUDIO_APPLICATION_NAVIGATION_CONTEXT_KEY } from '../../../../__
 import {
   LINEAGE_VIEW_MODE,
   LineageViewerContent,
+  IngestionDefinitionArtifactViewer,
 } from '@finos/legend-query-builder';
 import type { MatViewDataSet } from '@finos/legend-graph';
 import { IngestTestableEditor } from './testable/IngestTestableEditor.js';
@@ -213,6 +216,21 @@ export const IngestDefinitionEditor = observer(() => {
     }
   };
 
+  const viewSQL = () => {
+    if (
+      ingestDefinitionEditorState.ingest.TEMPORARY_MATVIEW_FUNCTION_DATA_SETS
+        ?.length
+    ) {
+      flowResult(ingestDefinitionEditorState.generateArtifact()).catch(
+        editorStore.applicationStore.alertUnhandledError,
+      );
+    } else {
+      editorStore.applicationStore.notificationService.notifyError(
+        'No MatView datasets available for artifact generation',
+      );
+    }
+  };
+
   return (
     <div className="ingest-definition-editor">
       <div className="panel">
@@ -266,9 +284,38 @@ export const IngestDefinitionEditor = observer(() => {
                           </div>
                         </button>
                       </div>
+                      {isValidForLineage && (
+                        <div className="btn__dropdown-combo btn__dropdown-combo--primary">
+                          <button
+                            className="btn__dropdown-combo__label"
+                            onClick={viewSQL}
+                            tabIndex={-1}
+                            disabled={
+                              ingestDefinitionEditorState
+                                .artifactGenerationState.isInProgress
+                            }
+                          >
+                            <SQLIcon className="btn__dropdown-combo__label__icon" />
+                            <div className="btn__dropdown-combo__label__title">
+                              {ingestDefinitionEditorState
+                                .artifactGenerationState.isInProgress
+                                ? 'Generating...'
+                                : 'View SQL'}
+                            </div>
+                          </button>
+                        </div>
+                      )}
                     </div>
                   </PanelHeaderActions>
                 </PanelHeader>
+                <PanelLoadingIndicator
+                  isLoading={
+                    ingestDefinitionEditorState.artifactGenerationState
+                      .isInProgress ||
+                    ingestDefinitionEditorState.lineageGenerationState
+                      .isInProgress
+                  }
+                />
                 <PanelContent>
                   <CodeEditor
                     inputValue={ingestDefinitionEditorState.textContent}
@@ -279,6 +326,16 @@ export const IngestDefinitionEditor = observer(() => {
                 <IngestLineageModal
                   ingestDefinitionEditorState={ingestDefinitionEditorState}
                 />
+                <IngestionDefinitionArtifactViewer
+                  artifact={ingestDefinitionEditorState.ingestionArtifact}
+                  onClose={() =>
+                    ingestDefinitionEditorState.setIngestionArtifact(undefined)
+                  }
+                  darkMode={
+                    !editorStore.applicationStore.layoutService
+                      .TEMPORARY__isLightColorThemeEnabled
+                  }
+                />
               </PanelContent>
             )}
             {selectedTab === INGEST_DEFINITION_TAB.TESTING && (

package/src/stores/editor/editor-state/element-editor-state/dataProduct/DataProductEditorState.ts

@@ -17,11 +17,13 @@
 import {
   type AppDirNode,
   type ArtifactGenerationExtensionResult,
+  type BatchLambdasRelationTypeResult,
   type DataProductElement,
   type IngestDefinition,
   type Mapping,
   type PackageableElement,
   type RawLambda,
+  type RelationTypeMetadata,
   type Stereotype,
   type StereotypeReference,
   type V1_AccessPointGroupInfo,
@@ -59,12 +61,6 @@ import {
   RelationElementsData,
   stub_Mapping,
   stub_RawLambda,
-  V1_GraphTransformerContextBuilder,
-  V1_LambdaReturnTypeInput,
-  V1_PureGraphManager,
-  V1_relationTypeModelSchema,
-  V1_RemoteEngine,
-  V1_transformRawLambda,
 } from '@finos/legend-graph';
 import type { EditorStore } from '../../../EditorStore.js';
 import { ElementEditorState } from '../ElementEditorState.js';
@@ -120,7 +116,6 @@ import type {
   AdhocDataProductDeployResponse,
   LakehouseIngestionManager,
 } from '@finos/legend-server-lakehouse';
-import { deserialize } from 'serializr';
 import { Diagram } from '@finos/legend-extension-dsl-diagram';
 import { LegendStudioTelemetryHelper } from '../../../../../__lib__/LegendStudioTelemetryHelper.js';
 import { onGeneratingDiagramFromMapping } from '../mapping/MappingEditorState.js';
@@ -195,7 +190,6 @@ export class AccessPointLambdaEditorState extends LambdaEditorState {
   }
 
   *updateLambdaRelationColumns(): GeneratorFn<void> {
-    // Prevent concurrent calls
     if (this.isUpdatingRelationColumns) {
       return;
     }
@@ -207,36 +201,11 @@ export class AccessPointLambdaEditorState extends LambdaEditorState {
 
     this.setIsUpdatingRelationColumns(true);
     try {
-      const model = guaranteeType(
-        this.editorStore.graphManagerState.graphManager,
-        V1_PureGraphManager,
-      ).getFullGraphModelData(this.editorStore.graphManagerState.graph);
-
-      const relationTypeInput = new V1_LambdaReturnTypeInput(
-        model,
-        V1_transformRawLambda(
+      const relationType =
+        (yield this.editorStore.graphManagerState.graphManager.getLambdaRelationType(
           this.val.accessPoint.func,
-          new V1_GraphTransformerContextBuilder(
-            this.editorStore.pluginManager.getPureProtocolProcessorPlugins(),
-          ).build(),
-        ),
-      );
-
-      const relationType = deserialize(
-        V1_relationTypeModelSchema,
-        (yield guaranteeType(
-          guaranteeType(
-            this.editorStore.graphManagerState.graphManager,
-            V1_PureGraphManager,
-          ).engine,
-          V1_RemoteEngine,
-        )
-          .getEngineServerClient()
-          .lambdaRelationType(
-            V1_LambdaReturnTypeInput.serialization.toJson(relationTypeInput),
-          )) as object,
-      );
-
+          this.editorStore.graphManagerState.graph,
+        )) as RelationTypeMetadata;
       this.setLambdaRelationColumns(
         relationType.columns.map((column) => column.name),
       );
@@ -1020,6 +989,7 @@ export class DataProductEditorState extends ElementEditorState {
       setDeployResponse: action,
       addAccessPoint: action,
       convertAccessPointsFuncObjects: flow,
+      batchUpdateLambdaRelationColumns: flow,
       selectedGroupState: observable,
       setSelectedGroupState: action,
       swapAccessPointGroups: action,
@@ -1135,6 +1105,59 @@ export class DataProductEditorState extends ElementEditorState {
     }
   }
 
+  *batchUpdateLambdaRelationColumns(
+    accessPointStates: LakehouseAccessPointState[],
+  ): GeneratorFn<void> {
+    const entries = accessPointStates
+      .filter(
+        (apState) =>
+          apState.relationElementState !== undefined &&
+          !apState.lambdaState.isUpdatingRelationColumns &&
+          apState.lambdaState.lastComputedLambdaHash !==
+            apState.accessPoint.func.hashCode,
+      )
+      .map((apState) => ({
+        apState,
+        hash: apState.accessPoint.func.hashCode,
+      }));
+    if (entries.length === 0) {
+      return;
+    }
+    entries.forEach((entry) =>
+      entry.apState.lambdaState.setIsUpdatingRelationColumns(true),
+    );
+    try {
+      const lambdas = new Map<string, RawLambda>(
+        entries.map((entry) => [
+          entry.apState.uuid,
+          entry.apState.accessPoint.func,
+        ]),
+      );
+      const { results } =
+        (yield this.editorStore.graphManagerState.graphManager.getBatchLambdasRelationType(
+          lambdas,
+          this.editorStore.graphManagerState.graph,
+        )) as BatchLambdasRelationTypeResult;
+      entries.forEach((entry) => {
+        const relationType = results.get(entry.apState.uuid);
+        entry.apState.lambdaState.setLambdaRelationColumns(
+          relationType
+            ? relationType.columns.map((column) => column.name)
+            : undefined,
+        );
+      });
+    } catch {
+      entries.forEach((entry) =>
+        entry.apState.lambdaState.setLambdaRelationColumns(undefined),
+      );
+    } finally {
+      entries.forEach((entry) => {
+        entry.apState.lambdaState.lastComputedLambdaHash = entry.hash;
+        entry.apState.lambdaState.setIsUpdatingRelationColumns(false);
+      });
+    }
+  }
+
   setSelectedGroupState(val: AccessPointGroupState | undefined): void {
     this.selectedGroupState = val;
   }

package/src/stores/editor/editor-state/element-editor-state/ingest/IngestDefinitionEditorState.ts

@@ -18,6 +18,8 @@ import {
   type MatViewDataSet,
   type PackageableElement,
   type V1_RawLineageModel,
+  type ArtifactGenerationExtensionResult,
+  type IngestionDefinitionArtifact,
   GRAPH_MANAGER_EVENT,
   IngestDefinition,
 } from '@finos/legend-graph';
@@ -62,6 +64,8 @@ export enum INGEST_DEFINITION_TAB {
   TESTING = 'Testing',
 }
 
+export const INGEST_DEFINITION_ARTIFACT_EXTENSION = 'ingestDefinition';
+
 const createEditorInitialConfiguration = (): EditorInitialConfiguration => {
   const config = new EditorInitialConfiguration();
   const ingest = new IngestElementEditorInitialConfiguration();
@@ -85,11 +89,20 @@ export const generateUrlToDeployOnOpen = (
 };
 
 const PARSER_SECTION = `###Lakehouse`;
+
+type CachedIngestArtifact = {
+  hashCode: string;
+  artifact: IngestionDefinitionArtifact;
+};
+
 export class IngestDefinitionEditorState extends ElementEditorState {
   selectedTab = INGEST_DEFINITION_TAB.DEFINITION;
   validateAndDeployResponse: ValidateAndDeploymentResponse | undefined;
   deploymentState = ActionState.create();
   lineageGenerationState = ActionState.create();
+  artifactGenerationState = ActionState.create();
+  ingestionArtifact: IngestionDefinitionArtifact | undefined;
+  cachedArtfact: CachedIngestArtifact | undefined;
   deployOnOpen = false;
   lineageState: LineageState;
   ingestTestableState: IngestTestableState;
@@ -106,13 +119,18 @@ export class IngestDefinitionEditorState extends ElementEditorState {
       deploymentState: observable,
       deployOnOpen: observable,
       setDeployOnOpen: observable,
+      cachedArtfact: observable,
       validateAndDeployResponse: observable,
+      ingestionArtifact: observable,
       deploymentResponse: computed,
       setSelectedTab: action,
       setValidateAndDeployResponse: action,
+      setIngestCachedArtifact: action,
+      setIngestionArtifact: action,
       init_with_deploy: flow,
       deploy: flow,
       generateLineage: flow,
+      generateArtifact: flow,
     });
     if (
       config?.elementEditorConfiguration instanceof
@@ -154,6 +172,14 @@ export class IngestDefinitionEditorState extends ElementEditorState {
     this.deployOnOpen = value;
   }
 
+  setIngestionArtifact(val: IngestionDefinitionArtifact | undefined): void {
+    this.ingestionArtifact = val;
+  }
+
+  setIngestCachedArtifact(val: CachedIngestArtifact | undefined): void {
+    this.cachedArtfact = val;
+  }
+
   *init_with_deploy(auth: AuthContextProps): GeneratorFn<void> {
     this.setDeployOnOpen(false);
     if (!auth.isAuthenticated) {
@@ -269,6 +295,62 @@ export class IngestDefinitionEditorState extends ElementEditorState {
     }
   }
 
+  *generateArtifact(): GeneratorFn<void> {
+    if (this.artifactGenerationState.isInProgress) {
+      this.editorStore.applicationStore.notificationService.notifyError(
+        'Artifact generation in progress already',
+      );
+      return;
+    }
+    const currentHashCode = this.ingest.hashCode;
+    if (this.cachedArtfact?.hashCode === currentHashCode) {
+      // reuse cached artifact if the ingest definition has not changed
+      this.setIngestionArtifact(this.cachedArtfact.artifact);
+      return;
+    }
+    try {
+      this.artifactGenerationState.inProgress();
+      const generatedArtifacts =
+        (yield this.editorStore.graphManagerState.graphManager.generateArtifacts(
+          this.editorStore.graphManagerState.graph,
+          this.editorStore.graphEditorMode.getGraphTextInputOption(),
+          [this.ingest.path],
+        )) as ArtifactGenerationExtensionResult;
+      const ingestArtifact = generatedArtifacts.values.find(
+        (artifact) =>
+          artifact.extension === INGEST_DEFINITION_ARTIFACT_EXTENSION,
+      );
+      const artifactContent =
+        ingestArtifact?.artifactsByExtensionElements[0]?.files[0]?.content;
+      if (!artifactContent) {
+        throw new Error(
+          `Could not find generated ingest definition artifact for '${this.ingest.path}'`,
+        );
+      }
+      const artifact =
+        this.editorStore.graphManagerState.graphManager.buildIngestDefinitionArtifact(
+          JSON.parse(artifactContent) as Record<PropertyKey, unknown>,
+          this.editorStore.graphManagerState.graph,
+        );
+      this.setIngestCachedArtifact({
+        hashCode: currentHashCode,
+        artifact,
+      });
+      this.setIngestionArtifact(artifact);
+    } catch (error) {
+      assertErrorThrown(error);
+      this.editorStore.applicationStore.logService.error(
+        LogEvent.create(GRAPH_MANAGER_EVENT.EXECUTION_FAILURE),
+        error,
+      );
+      this.editorStore.applicationStore.notificationService.notifyError(
+        `Failed to generate ingest artifact: ${error.message}`,
+      );
+    } finally {
+      this.artifactGenerationState.complete();
+    }
+  }
+
   reprocess(
     newElement: PackageableElement,
     editorStore: EditorStore,

package/src/stores/editor/editor-state/element-editor-state/mapping/MappingElementDecorator.ts

@@ -671,7 +671,10 @@ export class MappingElementDecorator implements SetImplementationVisitor<void> {
     ): RelationFunctionPropertyMapping[] => {
       const existingPropertyMappings = (propertyMappings ?? []).filter((pm) => {
         if (pm instanceof RelationFunctionPropertyMapping) {
-          return !isStubbed_RelationColumn(pm.column);
+          return (
+            pm.valueFn !== undefined ||
+            (pm.column !== undefined && !isStubbed_RelationColumn(pm.column))
+          );
         }
         return false;
       });
@@ -715,7 +718,9 @@ export class MappingElementDecorator implements SetImplementationVisitor<void> {
         propertyMappingsBeforeDecoration
           .filter(
             (propertyMapping) =>
-              !isStubbed_RelationColumn(propertyMapping.column),
+              propertyMapping.valueFn !== undefined ||
+              (propertyMapping.column !== undefined &&
+                !isStubbed_RelationColumn(propertyMapping.column)),
           )
           .filter(
             (propertyMapping) =>

package/src/stores/editor/editor-state/element-editor-state/mapping/RelationFunctionInstanceSetImplementationState.ts

@@ -85,7 +85,10 @@ export class RelationFunctionPropertyMappingState extends PropertyMappingState {
     pretty?: boolean | undefined;
     preserveCompilationError?: boolean | undefined;
   }): GeneratorFn<void> {
-    if (!isStubbed_RelationColumn(this.propertyMapping.column)) {
+    if (
+      this.propertyMapping.column &&
+      !isStubbed_RelationColumn(this.propertyMapping.column)
+    ) {
       this.setLambdaString(this.propertyMapping.column.name);
     } else {
       this.clearErrors();
@@ -170,7 +173,10 @@ export class RelationFunctionInstanceSetImplementationState extends InstanceSetI
 
   *convertPropertyMappingTransformObjects(): GeneratorFn<void> {
     this.propertyMappingStates.forEach((pm) => {
-      if (!isStubbed_RelationColumn(pm.propertyMapping.column)) {
+      if (
+        pm.propertyMapping.column &&
+        !isStubbed_RelationColumn(pm.propertyMapping.column)
+      ) {
         pm.setLambdaString(pm.propertyMapping.column.name);
       }
     });