@finos/legend-application-data-cube 0.5.6 → 0.6.0

package/src/components/builder/source/LakehouseConsumerDataCubeSourceBuilder.tsx

@@ -0,0 +1,149 @@
+/**
+ * Copyright (c) 2025-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { observer } from 'mobx-react-lite';
+import { FormTextInput } from '@finos/legend-data-cube';
+import { CustomSelectorInput } from '@finos/legend-art';
+import { useAuth } from 'react-oidc-context';
+import { useLegendDataCubeBuilderStore } from '../LegendDataCubeBuilderStoreProvider.js';
+import { assertErrorThrown, guaranteeNonNullable } from '@finos/legend-shared';
+import { useEffect } from 'react';
+import type { LakehouseConsumerDataCubeSourceBuilderState } from '../../../stores/builder/source/LakehouseConsumerDataCubeSourceBuilderState.js';
+
+export const LakehouseConsumerDataCubeSourceBuilder: React.FC<{
+  sourceBuilder: LakehouseConsumerDataCubeSourceBuilderState;
+}> = observer(({ sourceBuilder: state }) => {
+  const auth = useAuth();
+  const store = useLegendDataCubeBuilderStore();
+
+  useEffect(() => {
+    state.reset();
+    state.loadDataProducts();
+  }, [state]);
+
+  return (
+    <div className="flex h-full w-full">
+      <div className="m-3 flex w-full flex-col items-stretch gap-2 text-neutral-500">
+        {state.dataProducts.length > 0 && (
+          <div className="query-setup__wizard__group mt-3">
+            <div className="query-setup__wizard__group__title">
+              Data Product
+            </div>
+            <CustomSelectorInput
+              className="query-setup__wizard__selector text-nowrap"
+              options={state.dataProducts.map((dataProduct) => ({
+                label: guaranteeNonNullable(dataProduct.path),
+                value: guaranteeNonNullable(dataProduct.path),
+              }))}
+              disabled={false}
+              isLoading={false}
+              onChange={(newValue: { label: string; value: string } | null) => {
+                state.setSelectedDataProduct(newValue?.value ?? '');
+                state.fetchDataProductEnvironments(auth.user?.access_token);
+              }}
+              value={
+                state.selectedDataProduct
+                  ? {
+                      label: state.selectedDataProduct,
+                      value: state.selectedDataProduct,
+                    }
+                  : null
+              }
+              isClearable={false}
+              escapeClearsValue={true}
+            />
+          </div>
+        )}
+        {state.environments.length > 0 && (
+          <div className="query-setup__wizard__group mt-2">
+            <div className="query-setup__wizard__group__title">Environment</div>
+            <CustomSelectorInput
+              className="query-setup__wizard__selector"
+              options={state.environments.map((env) => ({
+                label: env,
+                value: env,
+              }))}
+              disabled={false}
+              isLoading={false}
+              onChange={(newValue: { label: string; value: string } | null) => {
+                const env = newValue?.value ?? '';
+                state.setSelectedEnvironment(env);
+                state.fetchAccessPoints();
+              }}
+              value={
+                state.selectedEnvironment
+                  ? {
+                      label: state.selectedEnvironment,
+                      value: state.selectedEnvironment,
+                    }
+                  : null
+              }
+              isClearable={false}
+              escapeClearsValue={true}
+            />
+          </div>
+        )}
+        {state.accessPoints.length > 0 && (
+          <div className="query-setup__wizard__group mt-2">
+            <div className="query-setup__wizard__group__title">
+              Access Points
+            </div>
+            <CustomSelectorInput
+              className="query-setup__wizard__selector"
+              options={state.accessPoints.map((accessPoint) => ({
+                label: accessPoint,
+                value: accessPoint,
+              }))}
+              disabled={false}
+              isLoading={false}
+              onChange={(newValue: { label: string; value: string } | null) => {
+                const accessPoint = newValue?.value ?? '';
+                state.setSelectedAccessPoint(accessPoint);
+                state
+                  .fetchEnvironment(auth.user?.access_token)
+                  .catch((error) => {
+                    assertErrorThrown(error);
+                    store.alertService.alertUnhandledError(error);
+                  });
+              }}
+              value={
+                state.selectedAccessPoint
+                  ? {
+                      label: state.selectedAccessPoint,
+                      value: state.selectedAccessPoint,
+                    }
+                  : null
+              }
+              isClearable={false}
+              escapeClearsValue={true}
+            />
+          </div>
+        )}
+        {state.selectedAccessPoint && (
+          <div className="query-setup__wizard__group mt-2">
+            <div className="query-setup__wizard__group__title">Warehouse</div>
+            <FormTextInput
+              className="w-full text-base text-black"
+              value={state.warehouse}
+              onChange={(event) => {
+                state.setWarehouse(event.target.value);
+              }}
+            />
+          </div>
+        )}
+      </div>
+    </div>
+  );
+});

package/src/stores/LegendDataCubeBaseStore.ts

@@ -36,6 +36,7 @@ import {
   guaranteeIsString,
 } from '@finos/legend-shared';
 import {
+  LakehouseContractServerClient,
   LakehouseIngestServerClient,
   LakehousePlatformServerClient,
 } from '@finos/legend-server-lakehouse';
@@ -68,6 +69,7 @@ export class LegendDataCubeBaseStore {
   readonly depotServerClient: DepotServerClient;
   readonly lakehousePlatformServerClient: LakehousePlatformServerClient;
   readonly lakehouseIngestServerClient: LakehouseIngestServerClient;
+  readonly lakehouseContractServerClient: LakehouseContractServerClient;
   readonly graphManager: V1_PureGraphManager;
   readonly remoteEngine: V1_RemoteEngine;
   readonly engineServerClient: V1_EngineServerClient;
@@ -102,6 +104,12 @@ export class LegendDataCubeBaseStore {
     this.lakehouseIngestServerClient.setTracerService(
       application.tracerService,
     );
+    this.lakehouseContractServerClient = new LakehouseContractServerClient({
+      baseUrl: this.application.config.lakehouseContractUrl,
+    });
+    this.lakehouseContractServerClient.setTracerService(
+      application.tracerService,
+    );
 
     this.graphManager = new V1_PureGraphManager(
       this.application.pluginManager,

package/src/stores/LegendDataCubeDataCubeEngine.ts

@@ -89,6 +89,9 @@ import {
   EXECUTION_SERIALIZATION_FORMAT,
   V1_LakehouseRuntime,
   V1_IngestDefinition,
+  V1_DataProduct,
+  V1_LakehouseAccessPoint,
+  V1_serializeRawValueSpecification,
 } from '@finos/legend-graph';
 import {
   _elementPtr,
@@ -152,9 +155,14 @@ import {
 import { QUERY_BUILDER_PURE_PATH } from '@finos/legend-query-builder';
 import {
   LAKEHOUSE_PRODUCER_DATA_CUBE_SOURCE_TYPE,
-  IngestDefinitionDataCubeSource,
-  RawIngestDefinitionDataCubeSource,
-} from './model/IngestDefinitionDataCubeSource.js';
+  LakehouseProducerDataCubeSource,
+  RawLakehouseProducerDataCubeSource,
+} from './model/LakehouseProducerDataCubeSource.js';
+import {
+  LAKEHOUSE_CONSUMER_DATA_CUBE_SOURCE_TYPE,
+  LakehouseConsumerDataCubeSource,
+  RawLakehouseConsumerDataCubeSource,
+} from './model/LakehouseConsumerDataCubeSource.js';
 
 export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
   private readonly _application: LegendDataCubeApplicationStore;
@@ -362,7 +370,7 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
       };
     } else if (source.type === LAKEHOUSE_PRODUCER_DATA_CUBE_SOURCE_TYPE) {
       const rawSource =
-        RawIngestDefinitionDataCubeSource.serialization.fromJson(source);
+        RawLakehouseProducerDataCubeSource.serialization.fromJson(source);
 
       return {
         ingestDefinition: {
@@ -372,6 +380,20 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
         },
         sourceType: source._type,
       };
+    } else if (source.type === LAKEHOUSE_CONSUMER_DATA_CUBE_SOURCE_TYPE) {
+      const rawSource =
+        RawLakehouseConsumerDataCubeSource.serialization.fromJson(source);
+
+      return {
+        dataProduct: {
+          environment: rawSource.environment,
+          warehouse: rawSource.warehouse,
+          path: rawSource.paths[0],
+          accessPoint: rawSource.paths[1],
+          dpCoordinates: rawSource.dpCoordinates,
+        },
+        sourceType: source._type,
+      };
     }
     return {};
   }
@@ -679,9 +701,9 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
       }
       case LAKEHOUSE_PRODUCER_DATA_CUBE_SOURCE_TYPE: {
         const rawSource =
-          RawIngestDefinitionDataCubeSource.serialization.fromJson(value);
+          RawLakehouseProducerDataCubeSource.serialization.fromJson(value);
 
-        const source = new IngestDefinitionDataCubeSource();
+        const source = new LakehouseProducerDataCubeSource();
 
         const query = new V1_ClassInstance();
         query.type = V1_ClassInstanceType.INGEST_ACCESSOR;
@@ -709,6 +731,33 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
         }
         return source;
       }
+      case LAKEHOUSE_CONSUMER_DATA_CUBE_SOURCE_TYPE: {
+        const rawSource =
+          RawLakehouseConsumerDataCubeSource.serialization.fromJson(value);
+
+        const source = new LakehouseConsumerDataCubeSource();
+
+        source.model = await this._synthesizeLakehouseConsumerPMCD(
+          rawSource,
+          source,
+        );
+
+        //TODO: add support for parameters
+        try {
+          source.columns = (
+            await this._getLambdaRelationType(
+              this.serializeValueSpecification(_lambda([], [source.query])),
+              source.model,
+            )
+          ).columns;
+        } catch (error) {
+          assertErrorThrown(error);
+          throw new Error(
+            `Can't get query result columns. Make sure the source query return a relation (i.e. typed TDS). Error: ${error.message}`,
+          );
+        }
+        return source;
+      }
       default:
         throw new UnsupportedOperationError(
           `Can't process query source of type '${value._type}'`,
@@ -789,7 +838,14 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
           model: context.model,
         })
       ).completions as CompletionItem[];
-    } else if (context instanceof IngestDefinitionDataCubeSource) {
+    } else if (context instanceof LakehouseProducerDataCubeSource) {
+      return (
+        await this._engineServerClient.completeCode({
+          codeBlock,
+          model: context.model,
+        })
+      ).completions as CompletionItem[];
+    } else if (context instanceof LakehouseConsumerDataCubeSource) {
       return (
         await this._engineServerClient.completeCode({
           codeBlock,
@@ -889,7 +945,9 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
         result = await this._runQuery(query, source.model, undefined, options);
       } else if (source instanceof UserDefinedFunctionDataCubeSource) {
         result = await this._runQuery(query, source.model, undefined, options);
-      } else if (source instanceof IngestDefinitionDataCubeSource) {
+      } else if (source instanceof LakehouseProducerDataCubeSource) {
+        result = await this._runQuery(query, source.model, undefined, options);
+      } else if (source instanceof LakehouseConsumerDataCubeSource) {
         result = await this._runQuery(query, source.model, undefined, options);
       } else if (source instanceof LegendQueryDataCubeSource) {
         const filteredParameterValues = await this._processLegendQueryParams(
@@ -1013,7 +1071,15 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
           undefined,
           options,
         )) as Response;
-      } else if (source instanceof IngestDefinitionDataCubeSource) {
+      } else if (source instanceof LakehouseProducerDataCubeSource) {
+        return (await this._runExportQuery(
+          query,
+          source.model,
+          format,
+          undefined,
+          options,
+        )) as Response;
+      } else if (source instanceof LakehouseConsumerDataCubeSource) {
         return (await this._runExportQuery(
           query,
           source.model,
@@ -1135,7 +1201,12 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
         DataCubeFunction.FROM,
         [_elementPtr(source.runtime)].filter(isNonNullable),
       );
-    } else if (source instanceof IngestDefinitionDataCubeSource) {
+    } else if (source instanceof LakehouseProducerDataCubeSource) {
+      return _function(
+        DataCubeFunction.FROM,
+        [_elementPtr(source.runtime)].filter(isNonNullable),
+      );
+    } else if (source instanceof LakehouseConsumerDataCubeSource) {
       return _function(
         DataCubeFunction.FROM,
         [_elementPtr(source.runtime)].filter(isNonNullable),
@@ -1259,7 +1330,9 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
       return this._getLambdaRelationType(query, source.model);
     } else if (source instanceof LocalFileDataCubeSource) {
       return this._getLambdaRelationType(query, source.model);
-    } else if (source instanceof IngestDefinitionDataCubeSource) {
+    } else if (source instanceof LakehouseProducerDataCubeSource) {
+      return this._getLambdaRelationType(query, source.model);
+    } else if (source instanceof LakehouseConsumerDataCubeSource) {
       return this._getLambdaRelationType(query, source.model);
     }
     throw new UnsupportedOperationError(
@@ -1550,8 +1623,8 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
   }
 
   private _synthesizeLakehouseProducerPMCD(
-    rawSource: RawIngestDefinitionDataCubeSource,
-    source: IngestDefinitionDataCubeSource,
+    rawSource: RawLakehouseProducerDataCubeSource,
+    source: LakehouseProducerDataCubeSource,
   ) {
     const runtime = new V1_LakehouseRuntime();
     runtime.warehouse = rawSource.warehouse;
@@ -1580,6 +1653,48 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
     return model;
   }
 
+  private async _synthesizeLakehouseConsumerPMCD(
+    rawSource: RawLakehouseConsumerDataCubeSource,
+    source: LakehouseConsumerDataCubeSource,
+  ) {
+    const pmcd = await this._depotServerClient.getPureModelContextData(
+      rawSource.dpCoordinates.groupId,
+      rawSource.dpCoordinates.artifactId,
+      rawSource.dpCoordinates.versionId,
+      true,
+    );
+    const deserializedPMCD = guaranteeType(
+      V1_deserializePureModelContext(pmcd),
+      V1_PureModelContextData,
+    );
+    const runtime = new V1_LakehouseRuntime();
+    runtime.warehouse = rawSource.warehouse;
+    runtime.environment = rawSource.environment;
+
+    const packageableRuntime = new V1_PackageableRuntime();
+    packageableRuntime.runtimeValue = runtime;
+    packageableRuntime.name = 'lakehouseConsumer';
+    packageableRuntime.package = 'runtime';
+    source.runtime = packageableRuntime.path;
+
+    deserializedPMCD.elements.push(packageableRuntime);
+
+    const dataProduct = guaranteeType(
+      deserializedPMCD.elements.find((pe) => pe.path === rawSource.paths[0]),
+      V1_DataProduct,
+    );
+    const accessPoint = guaranteeType(
+      dataProduct.accessPointGroups.map((group) =>
+        group.accessPoints.find((point) => point.id === rawSource.paths[1]),
+      )[0],
+      V1_LakehouseAccessPoint,
+    );
+    const lambda = V1_serializeRawValueSpecification(accessPoint.func);
+    source.query = this.deserializeValueSpecification(lambda);
+
+    return V1_serializePureModelContext(deserializedPMCD);
+  }
+
   // ---------------------------------- APPLICATION ----------------------------------
 
   override logDebug(message: string, ...data: unknown[]) {

package/src/stores/builder/LegendDataCubeBuilderStore.tsx

@@ -71,10 +71,11 @@ import { LocalFileDataCubeSourceLoaderState } from './source/LocalFileDataCubeSo
 import { LEGEND_DATACUBE_APP_EVENT } from '../../__lib__/LegendDataCubeEvent.js';
 import { LegendDataCubeQueryEditor } from '../../components/builder/LegendDataCubeQueryEditor.js';
 import type {
+  LakehouseContractServerClient,
   LakehouseIngestServerClient,
   LakehousePlatformServerClient,
 } from '@finos/legend-server-lakehouse';
-import { LAKEHOUSE_PRODUCER_DATA_CUBE_SOURCE_TYPE } from '../model/IngestDefinitionDataCubeSource.js';
+import { LAKEHOUSE_PRODUCER_DATA_CUBE_SOURCE_TYPE } from '../model/LakehouseProducerDataCubeSource.js';
 import { LakehouseProducerDataCubeSourceLoaderState } from './source/LakehouseProducerDataCubeSourceLoaderState.js';
 
 export class LegendDataCubeBuilderState {
@@ -134,6 +135,7 @@ export class LegendDataCubeBuilderStore {
   readonly depotServerClient: DepotServerClient;
   readonly platformServerClient: LakehousePlatformServerClient;
   readonly ingestServerClient: LakehouseIngestServerClient;
+  readonly contractServerClient: LakehouseContractServerClient;
   readonly engineServerClient: V1_EngineServerClient;
   readonly graphManager: V1_PureGraphManager;
   readonly taskService: DataCubeTaskService;
@@ -185,6 +187,7 @@ export class LegendDataCubeBuilderStore {
     this.engineServerClient = baseStore.engineServerClient;
     this.platformServerClient = baseStore.lakehousePlatformServerClient;
     this.ingestServerClient = baseStore.lakehouseIngestServerClient;
+    this.contractServerClient = baseStore.lakehouseContractServerClient;
     this.graphManager = baseStore.graphManager;
     this.taskService = baseStore.taskService;
     this.alertService = baseStore.alertService;

package/src/stores/builder/LegendDataCubeCreatorState.tsx

@@ -45,6 +45,7 @@ import { LocalFileDataCubeSourceBuilderState } from './source/LocalFileDataCubeS
 import { UserDefinedFunctionDataCubeSourceBuilderState } from './source/UserDefinedFunctionDataCubeSourceBuilderState.js';
 import { LEGEND_DATACUBE_APP_EVENT } from '../../__lib__/LegendDataCubeEvent.js';
 import { LakehouseProducerDataCubeSourceBuilderState } from './source/LakehouseProducerDataCubeSourceBuilderState.js';
+import { LakehouseConsumerDataCubeSourceBuilderState } from './source/LakehouseConsumerDataCubeSourceBuilderState.js';
 
 const DEFAULT_SOURCE_TYPE = LegendDataCubeSourceBuilderType.LEGEND_QUERY;
 
@@ -133,6 +134,15 @@ export class LegendDataCubeCreatorState {
           this._store.ingestServerClient,
           this._alertService,
         );
+      case LegendDataCubeSourceBuilderType.LAKEHOUSE_CONSUMER:
+        return new LakehouseConsumerDataCubeSourceBuilderState(
+          this._application,
+          this._engine,
+          this._store.depotServerClient,
+          this._store.platformServerClient,
+          this._store.contractServerClient,
+          this._alertService,
+        );
       default:
         throw new UnsupportedOperationError(
           `Can't create source builder for unsupported type '${type}'`,