@finos/legend-application-data-cube 0.5.6 → 0.6.0

package/src/stores/builder/source/LakehouseConsumerDataCubeSourceBuilderState.ts

@@ -0,0 +1,277 @@
+/**
+ * Copyright (c) 2025-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { action, flow, makeObservable, observable } from 'mobx';
+import {
+  LegendDataCubeSourceBuilderState,
+  LegendDataCubeSourceBuilderType,
+} from './LegendDataCubeSourceBuilderState.js';
+import {
+  assertErrorThrown,
+  guaranteeNonNullable,
+  type GeneratorFn,
+  type PlainObject,
+} from '@finos/legend-shared';
+import type { DataCubeAlertService } from '@finos/legend-data-cube';
+import type { LegendDataCubeApplicationStore } from '../../LegendDataCubeBaseStore.js';
+import type { LegendDataCubeDataCubeEngine } from '../../LegendDataCubeDataCubeEngine.js';
+import {
+  IngestDeploymentServerConfig,
+  type LakehousePlatformServerClient,
+  type LakehouseContractServerClient,
+} from '@finos/legend-server-lakehouse';
+import {
+  DepotScope,
+  VersionedProjectData,
+  type DepotServerClient,
+  type StoredSummaryEntity,
+} from '@finos/legend-server-depot';
+import {
+  CORE_PURE_PATH,
+  type V1_EntitlementsDataProductDetailsResponse,
+  type V1_EntitlementsDataProductDetails,
+} from '@finos/legend-graph';
+import { RawLakehouseConsumerDataCubeSource } from '../../model/LakehouseConsumerDataCubeSource.js';
+
+export class LakehouseConsumerDataCubeSourceBuilderState extends LegendDataCubeSourceBuilderState {
+  warehouse: string | undefined;
+  selectedDataProduct: string | undefined;
+  selectedAccessPoint: string | undefined;
+  selectedEnvironment: string | undefined;
+  paths: string[] = [];
+  ingestEnvironment: string | undefined;
+  dataProducts: StoredSummaryEntity[] = [];
+  dataProductMap: Record<string, V1_EntitlementsDataProductDetails> = {};
+  accessPoints: string[] = [];
+  environments: string[] = [];
+  dpCoordinates: VersionedProjectData | undefined;
+
+  private readonly _depotServerClient: DepotServerClient;
+  private readonly _platformServerClient: LakehousePlatformServerClient;
+  private readonly _contractServerClient: LakehouseContractServerClient;
+
+  constructor(
+    application: LegendDataCubeApplicationStore,
+    engine: LegendDataCubeDataCubeEngine,
+    depotServerClient: DepotServerClient,
+    platformServerClient: LakehousePlatformServerClient,
+    contractServerClient: LakehouseContractServerClient,
+    alertService: DataCubeAlertService,
+  ) {
+    super(application, engine, alertService);
+    this._platformServerClient = platformServerClient;
+    this._depotServerClient = depotServerClient;
+    this._contractServerClient = contractServerClient;
+
+    makeObservable(this, {
+      warehouse: observable,
+      dataProducts: observable,
+      selectedDataProduct: observable,
+      accessPoints: observable,
+      environments: observable,
+      selectedAccessPoint: observable,
+      selectedEnvironment: observable,
+      fetchDataProductEnvironments: flow,
+      loadDataProducts: flow,
+
+      setWarehouse: action,
+      setDataProducts: action,
+      setSelectedDataProduct: action,
+      setAccessPoints: action,
+      setEnvironments: action,
+      setSelectedAccessPoint: action,
+      setSelectedEnvironment: action,
+    });
+  }
+
+  setWarehouse(warehouse: string | undefined) {
+    this.warehouse = warehouse;
+  }
+
+  setDataProducts(dataProducts: StoredSummaryEntity[]) {
+    this.dataProducts = dataProducts;
+  }
+
+  setSelectedDataProduct(dataProduct: string | undefined) {
+    this.selectedDataProduct = dataProduct;
+  }
+
+  setAccessPoints(accessPoints: string[]) {
+    this.accessPoints = accessPoints;
+  }
+
+  setEnvironments(environments: string[]) {
+    this.environments = environments;
+  }
+
+  setSelectedAccessPoint(accessPoint: string | undefined) {
+    this.selectedAccessPoint = accessPoint;
+  }
+
+  setSelectedEnvironment(environment: string | undefined) {
+    this.selectedEnvironment = environment;
+  }
+
+  *loadDataProducts(): GeneratorFn<void> {
+    try {
+      this.setDataProducts(
+        (yield this._depotServerClient.getEntitiesSummaryByClassifier(
+          CORE_PURE_PATH.DATA_PRODUCT,
+          {
+            scope: DepotScope.RELEASES,
+            latest: true,
+            summary: true,
+          },
+        )) as StoredSummaryEntity[],
+      );
+    } catch (error) {
+      assertErrorThrown(error);
+      throw error;
+    }
+  }
+
+  *fetchDataProductEnvironments(
+    access_token: string | undefined,
+  ): GeneratorFn<void> {
+    this.resetDataProduct();
+    const selectedDp = guaranteeNonNullable(this.selectedDataProduct);
+    const dataProductResponse =
+      (yield this._contractServerClient.getDataProduct(
+        selectedDp.split('::').pop() ?? '',
+        access_token,
+      )) as V1_EntitlementsDataProductDetailsResponse;
+    if (dataProductResponse.dataProducts) {
+      this.dataProductMap = dataProductResponse.dataProducts
+        .filter((dp) => Boolean(dp.lakehouseEnvironment))
+        .reduce(
+          (acc, dp) => {
+            const envType = guaranteeNonNullable(
+              dp.lakehouseEnvironment,
+            ).type.valueOf();
+            acc[envType] = dp;
+            return acc;
+          },
+          {} as Record<string, V1_EntitlementsDataProductDetails>,
+        );
+
+      this.setEnvironments(Object.keys(this.dataProductMap));
+    }
+  }
+
+  fetchAccessPoints() {
+    this.resetEnvironment();
+    const selectedEnvironment = guaranteeNonNullable(this.selectedEnvironment);
+    const dataProduct = this.dataProductMap[selectedEnvironment];
+
+    if (
+      dataProduct?.origin &&
+      'group' in dataProduct.origin &&
+      'artifact' in dataProduct.origin &&
+      'version' in dataProduct.origin
+    ) {
+      const versionedData = new VersionedProjectData();
+      versionedData.groupId = dataProduct.origin.group as string;
+      versionedData.artifactId = dataProduct.origin.artifact as string;
+      versionedData.versionId = dataProduct.origin.version as string;
+      this.dpCoordinates = versionedData;
+    }
+    this.setAccessPoints(
+      dataProduct?.dataProduct.accessPoints.map(
+        (accessPoint) => accessPoint.name,
+      ) ?? [],
+    );
+  }
+
+  async fetchEnvironment(access_token: string | undefined) {
+    const selectedEnvironment = guaranteeNonNullable(this.selectedEnvironment);
+    const dataProduct = this.dataProductMap[selectedEnvironment];
+    const config = IngestDeploymentServerConfig.serialization.fromJson(
+      await this._platformServerClient.findProducerServer(
+        guaranteeNonNullable(dataProduct?.deploymentId),
+        'DEPLOYMENT',
+        access_token,
+      ),
+    );
+    const baseUrl = new URL(config.ingestServerUrl).hostname;
+    const subdomain = baseUrl.split('.')[0];
+    const parts = subdomain?.split('-');
+    const env = parts?.slice(0, -1).join('-');
+    this.ingestEnvironment = env;
+    this.setWarehouse('LAKEHOUSE_CONSUMER_DEFAULT_WH');
+  }
+
+  reset() {
+    this.setWarehouse(undefined);
+    this.setDataProducts([]);
+    this.setSelectedDataProduct(undefined);
+    this.setAccessPoints([]);
+    this.setEnvironments([]);
+    this.setSelectedAccessPoint(undefined);
+    this.setSelectedEnvironment(undefined);
+    this.dpCoordinates = undefined;
+  }
+
+  resetDataProduct() {
+    this.setWarehouse(undefined);
+    this.setAccessPoints([]);
+    this.setEnvironments([]);
+    this.setSelectedAccessPoint(undefined);
+    this.setSelectedEnvironment(undefined);
+    this.dpCoordinates = undefined;
+  }
+
+  resetEnvironment() {
+    this.setWarehouse(undefined);
+    this.setAccessPoints([]);
+    this.setSelectedAccessPoint(undefined);
+    this.dpCoordinates = undefined;
+  }
+
+  override get label(): LegendDataCubeSourceBuilderType {
+    return LegendDataCubeSourceBuilderType.LAKEHOUSE_CONSUMER;
+  }
+
+  override get isValid(): boolean {
+    return (
+      Boolean(this.warehouse) &&
+      Boolean(this.selectedAccessPoint) &&
+      Boolean(this.selectedDataProduct) &&
+      Boolean(this.selectedEnvironment) &&
+      Boolean(this.dpCoordinates)
+    );
+  }
+
+  override async generateSourceData(): Promise<PlainObject> {
+    // build data cube source
+    this.paths = [];
+    this.paths.push(
+      ...[
+        guaranteeNonNullable(this.selectedDataProduct),
+        guaranteeNonNullable(this.selectedAccessPoint),
+      ],
+    );
+
+    const rawSource = new RawLakehouseConsumerDataCubeSource();
+    rawSource.environment = guaranteeNonNullable(this.ingestEnvironment);
+    rawSource.dpCoordinates = guaranteeNonNullable(this.dpCoordinates);
+    rawSource.paths = this.paths;
+    rawSource.warehouse = guaranteeNonNullable(this.warehouse);
+
+    return Promise.resolve(
+      RawLakehouseConsumerDataCubeSource.serialization.toJson(rawSource),
+    );
+  }
+}

package/src/stores/builder/source/LakehouseProducerDataCubeSourceBuilderState.ts

@@ -23,7 +23,7 @@ import { guaranteeNonNullable, type PlainObject } from '@finos/legend-shared';
 import type { DataCubeAlertService } from '@finos/legend-data-cube';
 import type { LegendDataCubeApplicationStore } from '../../LegendDataCubeBaseStore.js';
 import type { LegendDataCubeDataCubeEngine } from '../../LegendDataCubeDataCubeEngine.js';
-import { RawIngestDefinitionDataCubeSource } from '../../model/IngestDefinitionDataCubeSource.js';
+import { RawLakehouseProducerDataCubeSource } from '../../model/LakehouseProducerDataCubeSource.js';
 import {
   IngestDeploymentServerConfig,
   ProducerEnvironment,
@@ -225,7 +225,7 @@ export class LakehouseProducerDataCubeSourceBuilderState extends LegendDataCubeS
 
     // build data cube source
     this.createPath();
-    const rawSource = new RawIngestDefinitionDataCubeSource();
+    const rawSource = new RawLakehouseProducerDataCubeSource();
     rawSource.ingestDefinitionUrn = guaranteeNonNullable(
       this.selectedIngestUrn,
     );
@@ -234,7 +234,7 @@ export class LakehouseProducerDataCubeSourceBuilderState extends LegendDataCubeS
     rawSource.warehouse = guaranteeNonNullable(this.warehouse);
 
     return Promise.resolve(
-      RawIngestDefinitionDataCubeSource.serialization.toJson(rawSource),
+      RawLakehouseProducerDataCubeSource.serialization.toJson(rawSource),
     );
   }
 }

package/src/stores/builder/source/LakehouseProducerDataCubeSourceLoaderState.ts

@@ -24,7 +24,7 @@ import type { LegendDataCubeDataCubeEngine } from '../../LegendDataCubeDataCubeE
 import { LegendDataCubeSourceLoaderState } from './LegendDataCubeSourceLoaderState.js';
 import type { DataCubeAlertService } from '@finos/legend-data-cube';
 import type { PersistentDataCube } from '@finos/legend-graph';
-import { RawIngestDefinitionDataCubeSource } from '../../model/IngestDefinitionDataCubeSource.js';
+import { RawLakehouseProducerDataCubeSource } from '../../model/LakehouseProducerDataCubeSource.js';
 import type { LakehouseIngestServerClient } from '@finos/legend-server-lakehouse';
 import { action, makeObservable, observable } from 'mobx';
 import { LegendDataCubeSourceBuilderType } from './LegendDataCubeSourceBuilderState.js';
@@ -84,7 +84,7 @@ export class LakehouseProducerDataCubeSourceLoaderState extends LegendDataCubeSo
   }
 
   reset() {
-    const rawSource = RawIngestDefinitionDataCubeSource.serialization.fromJson(
+    const rawSource = RawLakehouseProducerDataCubeSource.serialization.fromJson(
       this.sourceData,
     );
     this.setIngestDefinitionUrn(rawSource.ingestDefinitionUrn);
@@ -106,7 +106,7 @@ export class LakehouseProducerDataCubeSourceLoaderState extends LegendDataCubeSo
 
   override async load(source: PlainObject | undefined) {
     const deserializedSource =
-      RawIngestDefinitionDataCubeSource.serialization.fromJson(
+      RawLakehouseProducerDataCubeSource.serialization.fromJson(
         guaranteeNonNullable(source),
       );
 
@@ -116,7 +116,7 @@ export class LakehouseProducerDataCubeSourceLoaderState extends LegendDataCubeSo
       )[0] as PlainObject,
     );
 
-    return RawIngestDefinitionDataCubeSource.serialization.toJson(
+    return RawLakehouseProducerDataCubeSource.serialization.toJson(
       deserializedSource,
     );
   }

package/src/stores/builder/source/LegendDataCubeSourceBuilderState.ts

@@ -28,6 +28,7 @@ export enum LegendDataCubeSourceBuilderType {
   LOCAL_FILE = 'Local File',
   USER_DEFINED_FUNCTION = 'User Defined Function',
   LAKEHOUSE_PRODUCER = 'Lakehouse Producer',
+  LAKEHOUSE_CONSUMER = 'Lakehouse Consumer',
 }
 
 export abstract class LegendDataCubeSourceBuilderState {

package/src/stores/model/LakehouseConsumerDataCubeSource.ts

@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) 2025-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { DataCubeSource } from '@finos/legend-data-cube';
+import type { V1_PureModelContextComposite } from '@finos/legend-graph';
+import { VersionedProjectData } from '@finos/legend-server-depot';
+import {
+  SerializationFactory,
+  usingConstantValueSchema,
+  usingModelSchema,
+  type PlainObject,
+} from '@finos/legend-shared';
+import { createModelSchema, list, primitive } from 'serializr';
+
+export const LAKEHOUSE_CONSUMER_DATA_CUBE_SOURCE_TYPE = 'lakehouseConsumer';
+
+export class LakehouseConsumerDataCubeSource extends DataCubeSource {
+  model!: PlainObject<V1_PureModelContextComposite>;
+  runtime!: string;
+}
+
+export class RawLakehouseConsumerDataCubeSource {
+  dpCoordinates!: VersionedProjectData;
+  warehouse!: string;
+  environment!: string;
+  paths!: string[];
+
+  static readonly serialization = new SerializationFactory(
+    createModelSchema(RawLakehouseConsumerDataCubeSource, {
+      _type: usingConstantValueSchema(LAKEHOUSE_CONSUMER_DATA_CUBE_SOURCE_TYPE),
+      dpCoordinates: usingModelSchema(
+        VersionedProjectData.serialization.schema,
+      ),
+      warehouse: primitive(),
+      environment: primitive(),
+      paths: list(primitive()),
+    }),
+  );
+}

package/src/stores/model/{IngestDefinitionDataCubeSource.ts → LakehouseProducerDataCubeSource.ts}

@@ -25,19 +25,19 @@ import { createModelSchema, list, primitive } from 'serializr';
 
 export const LAKEHOUSE_PRODUCER_DATA_CUBE_SOURCE_TYPE = 'lakehouseProducer';
 
-export class IngestDefinitionDataCubeSource extends DataCubeSource {
+export class LakehouseProducerDataCubeSource extends DataCubeSource {
   model!: PlainObject<V1_PureModelContextData>;
   runtime!: string;
 }
 
-export class RawIngestDefinitionDataCubeSource {
+export class RawLakehouseProducerDataCubeSource {
   ingestDefinitionUrn!: string;
   warehouse!: string;
   ingestServerUrl!: string;
   paths!: string[];
 
   static readonly serialization = new SerializationFactory(
-    createModelSchema(RawIngestDefinitionDataCubeSource, {
+    createModelSchema(RawLakehouseProducerDataCubeSource, {
       _type: usingConstantValueSchema(LAKEHOUSE_PRODUCER_DATA_CUBE_SOURCE_TYPE),
       ingestDefinitionUrn: primitive(),
       warehouse: primitive(),

package/tsconfig.json

@@ -67,6 +67,7 @@
     "./src/stores/LegendDataCubeDataCubeEngine.ts",
     "./src/stores/LegendDataCubeDuckDBEngine.ts",
     "./src/stores/builder/source/FreeformTDSExpressionDataCubeSourceBuilderState.ts",
+    "./src/stores/builder/source/LakehouseConsumerDataCubeSourceBuilderState.ts",
     "./src/stores/builder/source/LakehouseProducerDataCubeSourceBuilderState.ts",
     "./src/stores/builder/source/LakehouseProducerDataCubeSourceLoaderState.ts",
     "./src/stores/builder/source/LegendDataCubeSourceBuilderState.ts",
@@ -75,7 +76,8 @@
     "./src/stores/builder/source/LocalFileDataCubeSourceBuilderState.ts",
     "./src/stores/builder/source/LocalFileDataCubeSourceLoaderState.ts",
     "./src/stores/builder/source/UserDefinedFunctionDataCubeSourceBuilderState.ts",
-    "./src/stores/model/IngestDefinitionDataCubeSource.ts",
+    "./src/stores/model/LakehouseConsumerDataCubeSource.ts",
+    "./src/stores/model/LakehouseProducerDataCubeSource.ts",
     "./src/stores/model/LegendQueryDataCubeSource.ts",
     "./src/stores/model/LocalFileDataCubeSource.ts",
     "./src/application/LegendDataCube.tsx",
@@ -92,6 +94,7 @@
     "./src/components/builder/LegendDataCubeSaver.tsx",
     "./src/components/builder/LegendDataCubeSourceViewer.tsx",
     "./src/components/builder/source/FreeformTDSExpressionDataCubeSourceBuilder.tsx",
+    "./src/components/builder/source/LakehouseConsumerDataCubeSourceBuilder.tsx",
     "./src/components/builder/source/LakehouseProducerDataCubeSourceBuilder.tsx",
     "./src/components/builder/source/LakehouseProducerDataCubeSourceLoader.tsx",
     "./src/components/builder/source/LegendDataCubeSourceLoader.tsx",

package/lib/stores/model/IngestDefinitionDataCubeSource.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"IngestDefinitionDataCubeSource.d.ts","sourceRoot":"","sources":["../../../src/stores/model/IngestDefinitionDataCubeSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EACL,oBAAoB,EAEpB,KAAK,WAAW,EACjB,MAAM,sBAAsB,CAAC;AAG9B,eAAO,MAAM,wCAAwC,sBAAsB,CAAC;AAE5E,qBAAa,8BAA+B,SAAQ,cAAc;IAChE,KAAK,EAAG,WAAW,CAAC,uBAAuB,CAAC,CAAC;IAC7C,OAAO,EAAG,MAAM,CAAC;CAClB;AAED,qBAAa,iCAAiC;IAC5C,mBAAmB,EAAG,MAAM,CAAC;IAC7B,SAAS,EAAG,MAAM,CAAC;IACnB,eAAe,EAAG,MAAM,CAAC;IACzB,KAAK,EAAG,MAAM,EAAE,CAAC;IAEjB,MAAM,CAAC,QAAQ,CAAC,aAAa,0DAQ3B;CACH"}

package/lib/stores/model/IngestDefinitionDataCubeSource.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"IngestDefinitionDataCubeSource.js","sourceRoot":"","sources":["../../../src/stores/model/IngestDefinitionDataCubeSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EACL,oBAAoB,EACpB,wBAAwB,GAEzB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAE/D,MAAM,CAAC,MAAM,wCAAwC,GAAG,mBAAmB,CAAC;AAE5E,MAAM,OAAO,8BAA+B,SAAQ,cAAc;IAChE,KAAK,CAAwC;IAC7C,OAAO,CAAU;CAClB;AAED,MAAM,OAAO,iCAAiC;IAC5C,mBAAmB,CAAU;IAC7B,SAAS,CAAU;IACnB,eAAe,CAAU;IACzB,KAAK,CAAY;IAEjB,MAAM,CAAU,aAAa,GAAG,IAAI,oBAAoB,CACtD,iBAAiB,CAAC,iCAAiC,EAAE;QACnD,KAAK,EAAE,wBAAwB,CAAC,wCAAwC,CAAC;QACzE,mBAAmB,EAAE,SAAS,EAAE;QAChC,SAAS,EAAE,SAAS,EAAE;QACtB,eAAe,EAAE,SAAS,EAAE;QAC5B,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC;KACzB,CAAC,CACH,CAAC"}