@finos/legend-application-studio 28.18.130 → 28.18.132

package/src/stores/editor/editor-state/element-editor-state/dataProduct/DataProductEditorState.ts

@@ -18,6 +18,7 @@ import {
   DataProduct,
   LakehouseAccessPoint,
   type PackageableElement,
+  type IngestDefinition,
   type AccessPoint,
   stub_RawLambda,
   LakehouseTargetEnv,
@@ -27,6 +28,7 @@ import {
   GRAPH_MANAGER_EVENT,
   isStubbed_RawLambda,
   AccessPointGroup,
+  CodeCompletionResult,
 } from '@finos/legend-graph';
 import type { EditorStore } from '../../../EditorStore.js';
 import { ElementEditorState } from '../ElementEditorState.js';
@@ -39,6 +41,9 @@ import {
   LogEvent,
   deleteEntry,
   filterByType,
+  ActionState,
+  guaranteeNonNullable,
+  assertTrue,
 } from '@finos/legend-shared';
 import {
   dataProduct_addAccessPoint,
@@ -46,12 +51,19 @@ import {
   dataProduct_deleteAccessPoint,
 } from '../../../../graph-modifier/DSL_DataProduct_GraphModifierHelper.js';
 import { LambdaEditorState } from '@finos/legend-query-builder';
+import type { IngestionManager } from '../../../../ingestion/IngestionManager.js';
+import {
+  DataProductElementEditorInitialConfiguration,
+  EditorInitialConfiguration,
+} from '../ElementEditorInitialConfiguration.js';
+import { EXTERNAL_APPLICATION_NAVIGATION__generateUrlWithEditorConfig } from '../../../../../__lib__/LegendStudioNavigation.js';
+import type { AdhocDataProductDeployResponse } from '../../../../ingestion/AdhocDataProductDeployResponse.js';
 
 export class AccessPointState {
-  readonly state: DataProductEditorState;
+  readonly state: AccessPointGroupState;
   accessPoint: AccessPoint;
 
-  constructor(val: AccessPoint, editorState: DataProductEditorState) {
+  constructor(val: AccessPoint, editorState: AccessPointGroupState) {
     this.accessPoint = val;
     this.state = editorState;
   }
@@ -62,9 +74,11 @@ export class AccessPointLambdaEditorState extends LambdaEditorState {
   readonly val: LakehouseAccessPointState;
 
   constructor(val: LakehouseAccessPointState) {
-    super('', LAMBDA_PIPE);
+    super('', LAMBDA_PIPE, {
+      typeAheadEnabled: true,
+    });
     this.val = val;
-    this.editorStore = val.state.editorStore;
+    this.editorStore = val.state.state.editorStore;
   }
 
   override get lambdaId(): string {
@@ -104,7 +118,7 @@ export class AccessPointLambdaEditorState extends LambdaEditorState {
   }): GeneratorFn<void> {
     if (
       !isStubbed_RawLambda(this.val.accessPoint.func) &&
-      !this.val.state.isConvertingTransformLambdaObjects
+      !this.val.state.state.isConvertingTransformLambdaObjects
     ) {
       try {
         const lambdas = new Map<string, RawLambda>();
@@ -135,12 +149,33 @@ export class AccessPointLambdaEditorState extends LambdaEditorState {
       this.setLambdaString('');
     }
   }
+
+  override async getCodeComplete(input: string): Promise<CodeCompletionResult> {
+    try {
+      return (await this.editorStore.graphManagerState.graphManager.getCodeComplete(
+        input,
+        this.editorStore.graphManagerState.graph,
+        undefined,
+        {
+          ignoreElements: [this.val.state.state.product.path],
+        },
+      )) as unknown as CodeCompletionResult;
+    } catch (error) {
+      assertErrorThrown(error);
+      this.editorStore.applicationStore.logService.error(
+        LogEvent.create(GRAPH_MANAGER_EVENT.PARSING_FAILURE),
+        error,
+      );
+      return new CodeCompletionResult();
+    }
+  }
 }
+
 export class LakehouseAccessPointState extends AccessPointState {
   declare accessPoint: LakehouseAccessPoint;
   lambdaState: AccessPointLambdaEditorState;
 
-  constructor(val: LakehouseAccessPoint, editorState: DataProductEditorState) {
+  constructor(val: LakehouseAccessPoint, editorState: AccessPointGroupState) {
     super(val, editorState);
     makeObservable(this, {
       lambdaState: observable,
@@ -150,36 +185,122 @@ export class LakehouseAccessPointState extends AccessPointState {
   }
 }
 
+export class AccessPointGroupState {
+  readonly state: DataProductEditorState;
+  value: AccessPointGroup;
+  accessPointStates: AccessPointState[] = [];
+
+  constructor(val: AccessPointGroup, editorState: DataProductEditorState) {
+    this.value = val;
+    this.state = editorState;
+    this.accessPointStates = val.accessPoints.map((e) =>
+      this.buildAccessPointState(e),
+    );
+    makeObservable(this, {
+      value: observable,
+      accessPointStates: observable,
+      addAccessPoint: action,
+      deleteAccessPoint: action,
+    });
+  }
+
+  deleteAccessPoint(val: AccessPointState): void {
+    const state = this.accessPointStates.find((a) => a === val);
+    deleteEntry(this.accessPointStates, state);
+    dataProduct_deleteAccessPoint(this.value, val.accessPoint);
+  }
+
+  addAccessPoint(point: AccessPoint): void {
+    const accessPointState = this.buildAccessPointState(point);
+    addUniqueEntry(this.accessPointStates, accessPointState);
+    dataProduct_addAccessPoint(this.value, point);
+  }
+
+  buildAccessPointState(val: AccessPoint): AccessPointState {
+    if (val instanceof LakehouseAccessPoint) {
+      return new LakehouseAccessPointState(val, this);
+    }
+    return new AccessPointState(val, this);
+  }
+}
+
+const createEditorInitialConfiguration = (): EditorInitialConfiguration => {
+  const config = new EditorInitialConfiguration();
+  const ingest = new DataProductElementEditorInitialConfiguration();
+  ingest.deployOnOpen = true;
+  config.elementEditorConfiguration = ingest;
+  return config;
+};
+
+const editorInitialConfigToBase64 = (val: EditorInitialConfiguration): string =>
+  btoa(JSON.stringify(EditorInitialConfiguration.serialization.toJson(val)));
+
+export const generateUrlToDeployOnOpen = (
+  val: DataProductEditorState,
+): string => {
+  return val.editorStore.applicationStore.navigationService.navigator.generateAddress(
+    EXTERNAL_APPLICATION_NAVIGATION__generateUrlWithEditorConfig(
+      val.editorStore.editorMode.generateElementLink(val.product.path),
+      editorInitialConfigToBase64(createEditorInitialConfiguration()),
+    ),
+  );
+};
+
 export class DataProductEditorState extends ElementEditorState {
   accessPointModal = false;
-  accessPointStates: AccessPointState[] = [];
+  deploymentState = ActionState.create();
+  accessPointGroupStates: AccessPointGroupState[] = [];
   isConvertingTransformLambdaObjects = false;
+  deployOnOpen = false;
+  deployResponse: AdhocDataProductDeployResponse | undefined;
 
-  constructor(editorStore: EditorStore, element: PackageableElement) {
+  constructor(
+    editorStore: EditorStore,
+    element: PackageableElement,
+    config?: EditorInitialConfiguration,
+  ) {
     super(editorStore, element);
 
     makeObservable(this, {
       product: computed,
       accessPointModal: observable,
-      accessPointStates: observable,
-      deleteAccessPoint: observable,
+      accessPointGroupStates: observable,
       isConvertingTransformLambdaObjects: observable,
+      deploy: flow,
+      deployOnOpen: observable,
+      deployResponse: observable,
+      setDeployOnOpen: action,
+      setDeployResponse: action,
       setAccessPointModal: action,
       addAccessPoint: action,
       convertAccessPointsFuncObjects: flow,
     });
-    this.accessPointStates = this.product.accessPointGroups
-      .map((e) => e.accessPoints)
-      .flat()
-      .map((e) => this.buildAccessPointState(e));
+    this.accessPointGroupStates = this.product.accessPointGroups.map(
+      (e) => new AccessPointGroupState(e, this),
+    );
+    const elementConfig = config?.elementEditorConfiguration;
+    if (elementConfig instanceof DataProductElementEditorInitialConfiguration) {
+      this.deployOnOpen = elementConfig.deployOnOpen ?? false;
+    }
+  }
+
+  setDeployOnOpen(value: boolean): void {
+    this.deployOnOpen = value;
+  }
+
+  setDeployResponse(
+    response: AdhocDataProductDeployResponse | undefined,
+  ): void {
+    this.deployResponse = response;
   }
 
   *convertAccessPointsFuncObjects(): GeneratorFn<void> {
     const lambdas = new Map<string, RawLambda>();
     const index = new Map<string, LakehouseAccessPointState>();
-    const states = this.accessPointStates.filter(
-      filterByType(LakehouseAccessPointState),
-    );
+    const states = this.accessPointGroupStates
+      .map((e) => e.accessPointStates)
+      .flat()
+      .filter(filterByType(LakehouseAccessPointState));
     states.forEach((pm) => {
       if (!isStubbed_RawLambda(pm.accessPoint.func)) {
         lambdas.set(pm.lambdaState.lambdaId, pm.accessPoint.func);
@@ -211,50 +332,66 @@ export class DataProductEditorState extends ElementEditorState {
     }
   }
 
-  buildAccessPointState(val: AccessPoint): AccessPointState {
-    if (val instanceof LakehouseAccessPoint) {
-      return new LakehouseAccessPointState(val, this);
-    }
-    return new AccessPointState(val, this);
-  }
-
   setAccessPointModal(val: boolean): void {
     this.accessPointModal = val;
   }
 
-  deleteAccessPoint(val: AccessPointState): void {
-    const ap = val.accessPoint;
-    // find group
-    const group = this.product.accessPointGroups.find((g) =>
-      Boolean(g.accessPoints.find((v) => v === ap)),
-    );
-    if (group) {
-      dataProduct_deleteAccessPoint(group, ap);
-      deleteEntry(this.accessPointStates, val);
-    }
-  }
-
-  addAccessPoint(id: string): void {
+  addAccessPoint(
+    id: string,
+    description: string | undefined,
+    accessPointGroup: AccessPointGroupState | string,
+  ): void {
     const accesspoint = new LakehouseAccessPoint(
       id,
       LakehouseTargetEnv.Snowflake,
       stub_RawLambda(),
     );
-
-    const group =
-      this.product.accessPointGroups[0] ?? this.createBareGroupAndAdd();
-    dataProduct_addAccessPoint(group, accesspoint);
-    addUniqueEntry(
-      this.accessPointStates,
-      this.buildAccessPointState(accesspoint),
-    );
+    accesspoint.description = description;
+    const groupState =
+      accessPointGroup instanceof AccessPointGroupState
+        ? accessPointGroup
+        : this.createBareGroupAndAdd(accessPointGroup);
+    groupState.addAccessPoint(accesspoint);
+    addUniqueEntry(this.accessPointGroupStates, groupState);
   }
 
-  createBareGroupAndAdd(): AccessPointGroup {
+  createBareGroupAndAdd(id: string): AccessPointGroupState {
     const group = new AccessPointGroup();
-    group.id = 'default';
+    group.id = 'id';
     dataProduct_addAccessPointGroup(this.product, group);
-    return group;
+    return new AccessPointGroupState(group, this);
+  }
+
+  *deploy(token: string | undefined): GeneratorFn<void> {
+    try {
+      assertTrue(
+        this.validForDeployment,
+        'Data product definition is not valid for deployment',
+      );
+      this.deploymentState.inProgress();
+      // The grammar we provide will be for the current data product + all ingests (used for compilation)
+      const grammar =
+        (yield this.editorStore.graphManagerState.graphManager.elementsToPureCode(
+          [...this.editorStore.graphManagerState.graph.ingests, this.product],
+        )) as unknown as string;
+
+      const response = (yield guaranteeNonNullable(
+        this.ingestionManager,
+      ).deployDataProduct(
+        grammar,
+        guaranteeNonNullable(this.associatedIngest?.appDirDeployment),
+        this.deploymentState,
+        token,
+      )) as unknown as AdhocDataProductDeployResponse;
+      this.setDeployResponse(response);
+    } catch (error) {
+      assertErrorThrown(error);
+      this.editorStore.applicationStore.notificationService.notifyError(
+        `Ingest definition failed to deploy: ${error.message}`,
+      );
+    } finally {
+      this.deploymentState.complete();
+    }
   }
 
   get product(): DataProduct {
@@ -265,10 +402,37 @@ export class DataProductEditorState extends ElementEditorState {
     );
   }
 
+  get validForDeployment(): boolean {
+    return Boolean(
+      this.associatedIngest?.appDirDeployment && this.ingestionManager,
+    );
+  }
+
   get accessPoints(): AccessPoint[] {
     return this.product.accessPointGroups.map((e) => e.accessPoints).flat();
   }
 
+  get ingestionManager(): IngestionManager | undefined {
+    return this.editorStore.ingestionManager;
+  }
+
+  get deployValidationMessage(): string {
+    if (!this.associatedIngest?.appDirDeployment) {
+      return 'No app dir deployment found';
+    } else if (!this.ingestionManager) {
+      return 'No ingestion manager found';
+    }
+    return 'Deploy';
+  }
+
+  // We need to get the associated Ingest to get the app dir deployment
+  // We could do a more in depth check on the access point lambdas to check which ingest it uses but for now
+  // we will assume all ingests have the same DID
+  // we get the last one, to prioritize the ones in the current project followed by dependency ones
+  get associatedIngest(): IngestDefinition | undefined {
+    return this.editorStore.graphManagerState.graph.ingests.slice(-1)[0];
+  }
+
   override reprocess(
     newElement: PackageableElement,
     editorStore: EditorStore,

package/src/stores/editor/editor-state/element-editor-state/mapping/FlatDataInstanceSetImplementationState.ts

@@ -50,6 +50,7 @@ import {
   stub_RawLambda,
   isStubbed_RawLambda,
   SetImplementationExplicitReference,
+  type CodeCompletionResult,
 } from '@finos/legend-graph';
 import { MAPPING_ELEMENT_TYPE } from './MappingEditorState.js';
 
@@ -297,6 +298,14 @@ export class EmbeddedFlatDataInstanceSetImplementationState
       setImplementation.propertyMappings,
     );
   }
+  typeAheadEnabled = false;
+  getCodeComplete(input: string): Promise<CodeCompletionResult> {
+    throw new Error('Method not implemented.');
+  }
+
+  setTypeAhead(val: boolean): void {
+    this.typeAheadEnabled = val;
+  }
 
   get lambdaId(): string {
     throw new IllegalStateError(

package/src/stores/editor/editor-state/element-editor-state/mapping/relational/RelationalInstanceSetImplementationState.ts

@@ -43,6 +43,7 @@ import {
   buildSourceInformationSourceId,
   isStubbed_RawRelationalOperationElement,
   stub_RawRelationalOperationElement,
+  type CodeCompletionResult,
 } from '@finos/legend-graph';
 
 export class RelationalPropertyMappingState extends PropertyMappingState {
@@ -158,6 +159,7 @@ export class EmbeddedRelationalInstanceSetImplementationState
   declare instanceSetImplementationState: RelationalInstanceSetImplementationState;
   declare mappingElement: EmbeddedRelationalInstanceSetImplementation;
   declare propertyMapping: EmbeddedRelationalInstanceSetImplementation;
+  typeAheadEnabled = false;
 
   constructor(
     editorStore: EditorStore,
@@ -173,6 +175,10 @@ export class EmbeddedRelationalInstanceSetImplementationState
     );
   }
 
+  setTypeAhead(val: boolean): void {
+    this.typeAheadEnabled = val;
+  }
+
   get lambdaId(): string {
     throw new IllegalStateError(
       `Can't build lambda ID for embedded relational instance set implementation state`,
@@ -234,6 +240,10 @@ export class EmbeddedRelationalInstanceSetImplementationState
   }): GeneratorFn<void> {
     throw new UnsupportedOperationError();
   }
+
+  getCodeComplete(input: string): Promise<CodeCompletionResult> {
+    throw new Error('Method not implemented.');
+  }
 }
 
 export class RootRelationalInstanceSetImplementationState extends RelationalInstanceSetImplementationState {

package/src/stores/graph-modifier/DSL_DataProduct_GraphModifierHelper.ts

@@ -51,3 +51,9 @@ export const dataProduct_deleteAccessPointGroup = action(
     deleteEntry(product.accessPointGroups, accessPointGroup);
   },
 );
+
+export const dataProduct_setTitle = action(
+  (product: DataProduct, title: string) => {
+    product.title = title;
+  },
+);

package/src/stores/ingestion/AdhocDataProductDeployResponse.ts

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { PlainObject } from '@finos/legend-shared';
+
+export class AdhocDataProductDeployResponse {
+  content!: PlainObject;
+}
+
+export const createAdhocDataProductDeployResponse = (
+  json: PlainObject<AdhocDataProductDeployResponse>,
+): AdhocDataProductDeployResponse => {
+  const response = new AdhocDataProductDeployResponse();
+  response.content = json;
+  return response;
+};

package/src/stores/ingestion/IngestDeploymentServerClient.ts

@@ -26,9 +26,12 @@ import type {
   IngestDefinitionValidationResponse,
 } from './IngestionDeploymentResponse.js';
 import type { IngestDeploymentServerConfig } from '../../application/LegendIngestionConfiguration.js';
+import type { AdhocDataProductDeployResponse } from './AdhocDataProductDeployResponse.js';
 
 export class IngestDeploymentServerClient extends AbstractServerClient {
   environmentClassification: string;
+
+  private DATA_PRODUCT_URL = 'data-product';
   constructor(config: IngestDeploymentServerConfig) {
     super({
       baseUrl: config.ingestServerUrl,
@@ -49,6 +52,9 @@ export class IngestDeploymentServerClient extends AbstractServerClient {
     Authorization: `Bearer ${token}`,
   });
 
+  private _dataProduct = (): string =>
+    `${this.baseUrl}/${this.DATA_PRODUCT_URL}/api/entitlements/sdlc/deploy/definitions`;
+
   private _ingest = (): string =>
     `${this.baseUrl}/api/ingest/sdlc/deploy/definitions`;
 
@@ -80,4 +86,16 @@ export class IngestDeploymentServerClient extends AbstractServerClient {
       this._tokenWithTextPlain(token),
     );
   }
+
+  deployDataProduct(
+    fullGrammar: string,
+    token: string | undefined,
+  ): Promise<PlainObject<AdhocDataProductDeployResponse>> {
+    return this.post(
+      `${this._dataProduct()}`,
+      fullGrammar,
+      undefined,
+      this._tokenWithTextPlain(token),
+    );
+  }
 }

package/src/stores/ingestion/IngestionManager.ts

@@ -30,6 +30,10 @@ import {
   IngestDeploymentServerConfig,
   type LegendIngestionConfiguration,
 } from '../../application/LegendIngestionConfiguration.js';
+import {
+  createAdhocDataProductDeployResponse,
+  type AdhocDataProductDeployResponse,
+} from './AdhocDataProductDeployResponse.js';
 
 export class IngestionManager {
   private ingestDiscoveryServerClient: IngestDiscoveryServerClient;
@@ -109,6 +113,27 @@ export class IngestionManager {
     return fullResponse;
   }
 
+  async deployDataProduct(
+    grammarText: string,
+    appDirNode: AppDirNode,
+    actionState: ActionState | undefined,
+    token: string | undefined,
+  ): Promise<AdhocDataProductDeployResponse> {
+    actionState?.setMessage(
+      `Discovering associated data product environment for DID ${appDirNode.appDirId}...`,
+    );
+    await this.identifyIngestDeploymentServer(appDirNode, token);
+    actionState?.setMessage(
+      `Deploying data product with server ${this.ingestDeploymentServerClient.baseUrl ?? ''} for realm ${this.ingestDeploymentServerClient.environmentClassification}...`,
+    );
+    const deployResponse =
+      await this.ingestDeploymentServerClient.deployDataProduct(
+        grammarText,
+        token,
+      );
+    return createAdhocDataProductDeployResponse(deployResponse);
+  }
+
   private async _validate(
     ingestDefinition: string,
     appDirNode: AppDirNode | undefined,

package/tsconfig.json

@@ -205,6 +205,7 @@
     "./src/stores/graph-modifier/STO_FlatData_GraphModifierHelper.ts",
     "./src/stores/graph-modifier/STO_Relational_GraphModifierHelper.ts",
     "./src/stores/graph-modifier/Testable_GraphModifierHelper.ts",
+    "./src/stores/ingestion/AdhocDataProductDeployResponse.ts",
     "./src/stores/ingestion/IngestDeploymentServerClient.ts",
     "./src/stores/ingestion/IngestDiscoveryServerClient.ts",
     "./src/stores/ingestion/IngestionDeploymentResponse.ts",