@finos/legend-application-data-cube 0.6.15 → 0.7.1

package/src/components/builder/source/LakehouseConsumerDataCubeSourceBuilder.tsx

@@ -20,11 +20,13 @@ import { useAuth } from 'react-oidc-context';
 import { guaranteeNonNullable } from '@finos/legend-shared';
 import { useEffect } from 'react';
 import type { LakehouseConsumerDataCubeSourceBuilderState } from '../../../stores/builder/source/LakehouseConsumerDataCubeSourceBuilderState.js';
+import { useLegendDataCubeBuilderStore } from '../LegendDataCubeBuilderStoreProvider.js';
 
 export const LakehouseConsumerDataCubeSourceBuilder: React.FC<{
   sourceBuilder: LakehouseConsumerDataCubeSourceBuilderState;
 }> = observer(({ sourceBuilder: state }) => {
   const auth = useAuth();
+  const store = useLegendDataCubeBuilderStore();
 
   useEffect(() => {
     state.reset();
@@ -50,7 +52,11 @@ export const LakehouseConsumerDataCubeSourceBuilder: React.FC<{
             isLoading={state.dataProductLoadingState.isInProgress}
             onChange={(newValue: { label: string; value: string } | null) => {
               state.setSelectedDataProduct(newValue?.value ?? '');
-              state.fetchDataProductEnvironments(auth.user?.access_token);
+              state
+                .fetchAccessPoints(auth.user?.access_token)
+                .catch((error) =>
+                  store.alertService.alertUnhandledError(error),
+                );
             }}
             value={
               state.selectedDataProduct
@@ -65,37 +71,6 @@ export const LakehouseConsumerDataCubeSourceBuilder: React.FC<{
             escapeClearsValue={true}
           />
         </div>
-        {state.ingestEnvironments.length > 0 && (
-          <div className="query-setup__wizard__group mt-2">
-            <div className="query-setup__wizard__group__title">
-              Ingest Environment
-            </div>
-            <CustomSelectorInput
-              className="query-setup__wizard__selector"
-              options={state.ingestEnvironments.map((env) => ({
-                label: env,
-                value: env,
-              }))}
-              disabled={false}
-              isLoading={false}
-              onChange={(newValue: { label: string; value: string } | null) => {
-                const env = newValue?.value ?? '';
-                state.setSelectedIngestEnvironment(env);
-                state.fetchAccessPoints();
-              }}
-              value={
-                state.selectedIngestEnvironment
-                  ? {
-                      label: state.selectedIngestEnvironment,
-                      value: state.selectedIngestEnvironment,
-                    }
-                  : null
-              }
-              isClearable={false}
-              escapeClearsValue={true}
-            />
-          </div>
-        )}
         {state.accessPoints.length > 0 && (
           <div className="query-setup__wizard__group mt-2">
             <div className="query-setup__wizard__group__title">

package/src/components/builder/source/LakehouseProducerDataCubeSourceBuilder.tsx

@@ -15,23 +15,37 @@
  */
 import { observer } from 'mobx-react-lite';
 import type { LakehouseProducerDataCubeSourceBuilderState } from '../../../stores/builder/source/LakehouseProducerDataCubeSourceBuilderState.js';
-import { FormButton, FormTextInput } from '@finos/legend-data-cube';
+import {
+  FormButton,
+  FormCheckbox,
+  FormTextInput,
+} from '@finos/legend-data-cube';
 import { CustomSelectorInput } from '@finos/legend-art';
 import { useAuth } from 'react-oidc-context';
 import { useLegendDataCubeBuilderStore } from '../LegendDataCubeBuilderStoreProvider.js';
 import { guaranteeNonNullable } from '@finos/legend-shared';
-import { useEffect } from 'react';
+import { useEffect, useState } from 'react';
 
 export const LakehouseProducerDataCubeSourceBuilder: React.FC<{
   sourceBuilder: LakehouseProducerDataCubeSourceBuilderState;
 }> = observer(({ sourceBuilder: state }) => {
   const auth = useAuth();
   const store = useLegendDataCubeBuilderStore();
+  const [isIcebergFlowSelected, setIsIcebergFlowSelected] = useState(true);
+
+  const toggleSetisIcebergEnabled = () => {
+    setIsIcebergFlowSelected(!isIcebergFlowSelected);
+    state.setEnableIceberg(!isIcebergFlowSelected);
+  };
 
   useEffect(() => {
     state.reset();
   }, [state]);
 
+  useEffect(() => {
+    state.setUserManagerSettings(auth.settings);
+  }, [state, auth]);
+
   function createUrnPairs(
     urns: string[],
   ): Record<string, string | undefined>[] {
@@ -70,6 +84,17 @@ export const LakehouseProducerDataCubeSourceBuilder: React.FC<{
             </FormButton>
           </div>
         </div>
+        {state.icebergEnabled && (
+          <div className="query-setup__wizard__group mt-2">
+            <div className="flex h-5 w-[calc(100%_-_40px)] overflow-x-auto">
+              <FormCheckbox
+                label="Use Iceberg"
+                checked={isIcebergFlowSelected}
+                onChange={toggleSetisIcebergEnabled}
+              />
+            </div>
+          </div>
+        )}
         {state.ingestUrns.length > 0 && (
           <div className="query-setup__wizard__group mt-3">
             <div className="query-setup__wizard__group__title">Ingest Urn</div>
@@ -130,18 +155,19 @@ export const LakehouseProducerDataCubeSourceBuilder: React.FC<{
             />
           </div>
         )}
-        {state.selectedTable && (
-          <div className="query-setup__wizard__group mt-2">
-            <div className="query-setup__wizard__group__title">Warehouse</div>
-            <FormTextInput
-              className="w-full text-base text-black"
-              value={state.warehouse}
-              onChange={(event) => {
-                state.setWarehouse(event.target.value);
-              }}
-            />
-          </div>
-        )}
+        {state.selectedTable &&
+          (!isIcebergFlowSelected || !state.icebergEnabled) && (
+            <div className="query-setup__wizard__group mt-2">
+              <div className="query-setup__wizard__group__title">Warehouse</div>
+              <FormTextInput
+                className="w-full text-base text-black"
+                value={state.warehouse}
+                onChange={(event) => {
+                  state.setWarehouse(event.target.value);
+                }}
+              />
+            </div>
+          )}
       </div>
     </div>
   );

package/src/components/builder/source/LakehouseProducerDataCubeSourceLoader.tsx

@@ -33,6 +33,10 @@ export const LakehouseProducerDataCubeSourceLoader = observer(
       partialSourceLoader.reset();
     }, [partialSourceLoader]);
 
+    useEffect(() => {
+      partialSourceLoader.setUserManagerSettings(auth.settings);
+    }, [partialSourceLoader, auth]);
+
     return (
       <div className="flex h-full w-full">
         <div className="m-3 flex w-full flex-col items-stretch gap-2 text-neutral-500">

package/src/stores/LegendDataCubeDataCubeEngine.ts

@@ -156,6 +156,7 @@ import { QUERY_BUILDER_PURE_PATH } from '@finos/legend-query-builder';
 import {
   LAKEHOUSE_PRODUCER_DATA_CUBE_SOURCE_TYPE,
   LakehouseProducerDataCubeSource,
+  LakehouseProducerIcebergCachedDataCubeSource,
   RawLakehouseProducerDataCubeSource,
 } from './model/LakehouseProducerDataCubeSource.js';
 import {
@@ -198,6 +199,7 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
   // ---------------------------------- IMPLEMENTATION ----------------------------------
 
   override getDataFromSource(source?: DataCubeSource): PlainObject {
+    // TODO: add lakehouse sources
     if (source instanceof LegendQueryDataCubeSource) {
       const queryInfo = source.info;
       return {
@@ -378,6 +380,8 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
           warehouse: rawSource.warehouse,
           ingestServerUrl: rawSource.ingestServerUrl,
         },
+        icebergCatalog: rawSource.icebergConfig?.catalogUrl,
+        paths: rawSource.paths,
         sourceType: source._type,
       };
     } else if (source.type === LAKEHOUSE_CONSUMER_DATA_CUBE_SOURCE_TYPE) {
@@ -459,62 +463,7 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
               // TODO: confirm this is in accordance to engine
               // check if we have a duckdb enum mapping
               // See https://duckdb.org/docs/sql/data_types/overview.html
-              switch (col[1] as string) {
-                case 'BIT': {
-                  column.type = new V1_Bit();
-                  break;
-                }
-                case 'BOOLEAN': {
-                  // TODO: understand why boolean is not present in relationalDataType
-                  column.type = new V1_VarChar();
-                  break;
-                }
-                case 'DATE': {
-                  column.type = new V1_Date();
-                  break;
-                }
-                case 'DECIMAL': {
-                  column.type = new V1_Decimal();
-                  break;
-                }
-                case 'DOUBLE': {
-                  column.type = new V1_Double();
-                  break;
-                }
-                case 'FLOAT': {
-                  column.type = new V1_Float();
-                  break;
-                }
-                case 'INTEGER': {
-                  column.type = new V1_Integer();
-                  break;
-                }
-                case 'TININT': {
-                  column.type = new V1_TinyInt();
-                  break;
-                }
-                case 'SMALLINT': {
-                  column.type = new V1_SmallInt();
-                  break;
-                }
-                case 'BIGINT': {
-                  column.type = new V1_BigInt();
-                  break;
-                }
-                case 'TIMESTAMP': {
-                  column.type = new V1_Timestamp();
-                  break;
-                }
-                case 'VARCHAR': {
-                  column.type = new V1_VarChar();
-                  break;
-                }
-                default: {
-                  throw new UnsupportedOperationError(
-                    `Can't ingest local file data: failed to find matching relational data type for DuckDB type '${col[1]}' when synthesizing table definition`,
-                  );
-                }
-              }
+              this._getColumnType(col, column);
               return column;
             }),
           });
@@ -703,33 +652,87 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
         const rawSource =
           RawLakehouseProducerDataCubeSource.serialization.fromJson(value);
 
-        const source = new LakehouseProducerDataCubeSource();
-
-        const query = new V1_ClassInstance();
-        query.type = V1_ClassInstanceType.INGEST_ACCESSOR;
-        const ingestAccesor = new V1_RelationStoreAccessor();
-        ingestAccesor.path = rawSource.paths;
-        ingestAccesor.metadata = false;
-        query.value = ingestAccesor;
-        source.query = query;
-
-        const model = this._synthesizeLakehouseProducerPMCD(rawSource, source);
-        source.model = V1_serializePureModelContextData(model);
+        if (
+          rawSource.icebergConfig?.icebergRef &&
+          rawSource.icebergConfig.catalogUrl
+        ) {
+          const source = new LakehouseProducerIcebergCachedDataCubeSource();
+          const tableCatalog = this._duckDBEngine.retrieveCatalogTable(
+            rawSource.icebergConfig.icebergRef,
+          );
 
-        try {
-          source.columns = (
-            await this._getLambdaRelationType(
-              this.serializeValueSpecification(_lambda([], [source.query])),
-              source.model,
-            )
-          ).columns;
-        } catch (error) {
-          assertErrorThrown(error);
-          throw new Error(
-            `Can't get query result columns. Make sure the source query return a relation (i.e. typed TDS). Error: ${error.message}`,
+          const { model, database, schema, table, runtime } =
+            this._synthesizeMinimalModelContext({
+              schemaName: tableCatalog.schemaName,
+              tableName: tableCatalog.tableName,
+              tableColumns: tableCatalog.columns.map((col) => {
+                const column = new V1_Column();
+                column.name = col[0] as string;
+                // TODO: confirm this is in accordance to engine
+                // check if we have a duckdb enum mapping
+                // See https://duckdb.org/docs/sql/data_types/overview.html
+                this._getColumnType(col, column);
+                return column;
+              }),
+            });
+
+          source.db = database.path;
+          source.model = model;
+          source.table = table.name;
+          source.schema = schema.name;
+          source.runtime = runtime.path;
+
+          const query = new V1_ClassInstance();
+          query.type = V1_ClassInstanceType.RELATION_STORE_ACCESSOR;
+          const storeAccessor = new V1_RelationStoreAccessor();
+          storeAccessor.path = [source.db, source.schema, source.table];
+          query.value = storeAccessor;
+          source.query = query;
+          try {
+            source.columns = (
+              await this._getLambdaRelationType(
+                this.serializeValueSpecification(_lambda([], [source.query])),
+                source.model,
+              )
+            ).columns;
+          } catch (error) {
+            assertErrorThrown(error);
+            throw new Error(
+              `Can't get query result columns. Make sure the source query return a relation (i.e. typed TDS). Error: ${error.message}`,
+            );
+          }
+          return source;
+        } else {
+          const source = new LakehouseProducerDataCubeSource();
+
+          const query = new V1_ClassInstance();
+          query.type = V1_ClassInstanceType.INGEST_ACCESSOR;
+          const ingestAccesor = new V1_RelationStoreAccessor();
+          ingestAccesor.path = rawSource.paths;
+          ingestAccesor.metadata = false;
+          query.value = ingestAccesor;
+          source.query = query;
+
+          const model = this._synthesizeLakehouseProducerPMCD(
+            rawSource,
+            source,
           );
+          source.model = V1_serializePureModelContextData(model);
+          try {
+            source.columns = (
+              await this._getLambdaRelationType(
+                this.serializeValueSpecification(_lambda([], [source.query])),
+                source.model,
+              )
+            ).columns;
+          } catch (error) {
+            assertErrorThrown(error);
+            throw new Error(
+              `Can't get query result columns. Make sure the source query return a relation (i.e. typed TDS). Error: ${error.message}`,
+            );
+          }
+          return source;
         }
-        return source;
       }
       case LAKEHOUSE_CONSUMER_DATA_CUBE_SOURCE_TYPE: {
         const rawSource =
@@ -1019,6 +1022,35 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
           result: await this._duckDBEngine.runSQLQuery(sql),
           executionTime: endTime - startTime,
         };
+      } else if (
+        source instanceof LakehouseProducerIcebergCachedDataCubeSource
+      ) {
+        const executionPlan = await this._generateExecutionPlan(
+          query,
+          source.model,
+          [],
+          // NOTE: for local file, we're using DuckDB, but its protocol models
+          // are not available in the latest production protocol version V1_33_0, so
+          // we have to force using VX_X_X
+          // once we either cut another protocol version or backport the DuckDB models
+          // to V1_33_0, we will can remove this
+          { ...options, clientVersion: PureClientVersion.VX_X_X },
+        );
+        const sql = guaranteeNonNullable(
+          executionPlan instanceof V1_SimpleExecutionPlan
+            ? executionPlan.rootExecutionNode.executionNodes
+                .filter(filterByType(V1_SQLExecutionNode))
+                .at(-1)?.sqlQuery
+            : undefined,
+          `Can't process execution plan: failed to extract generated SQL`,
+        );
+        const endTime = performance.now();
+        return {
+          executedQuery: await queryCodePromise,
+          executedSQL: sql,
+          result: await this._duckDBEngine.runSQLQuery(sql),
+          executionTime: endTime - startTime,
+        };
       } else {
         throw new UnsupportedOperationError(
           `Can't execute query with unsupported source`,
@@ -1204,6 +1236,11 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
         DataCubeFunction.FROM,
         [_elementPtr(source.runtime)].filter(isNonNullable),
       );
+    } else if (source instanceof LakehouseProducerIcebergCachedDataCubeSource) {
+      return _function(
+        DataCubeFunction.FROM,
+        [_elementPtr(source.runtime)].filter(isNonNullable),
+      );
     } else if (source instanceof LakehouseProducerDataCubeSource) {
       return _function(
         DataCubeFunction.FROM,
@@ -1360,6 +1397,8 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
       return this._getLambdaRelationType(query, source.model);
     } else if (source instanceof LakehouseProducerDataCubeSource) {
       return this._getLambdaRelationType(query, source.model);
+    } else if (source instanceof LakehouseProducerIcebergCachedDataCubeSource) {
+      return this._getLambdaRelationType(query, source.model);
     } else if (source instanceof LakehouseConsumerDataCubeSource) {
       return this._getLambdaRelationType(query, source.model);
     }
@@ -1590,6 +1629,82 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
     return { dbReference, columnNames };
   }
 
+  async ingestIcebergTable(
+    warehouse: string,
+    paths: string[],
+    catalogApi: string,
+    refId?: string,
+    token?: string,
+  ) {
+    const { dbReference } = await this._duckDBEngine.ingestIcebergTable(
+      warehouse,
+      paths,
+      catalogApi,
+      refId,
+      token,
+    );
+    return { dbReference };
+  }
+
+  private _getColumnType(col: string[], column: V1_Column) {
+    switch (col[1] as string) {
+      case 'BIT': {
+        column.type = new V1_Bit();
+        break;
+      }
+      case 'BOOLEAN': {
+        // TODO: understand why boolean is not present in relationalDataType
+        column.type = new V1_VarChar();
+        break;
+      }
+      case 'DATE': {
+        column.type = new V1_Date();
+        break;
+      }
+      case 'DECIMAL': {
+        column.type = new V1_Decimal();
+        break;
+      }
+      case 'DOUBLE': {
+        column.type = new V1_Double();
+        break;
+      }
+      case 'FLOAT': {
+        column.type = new V1_Float();
+        break;
+      }
+      case 'INTEGER': {
+        column.type = new V1_Integer();
+        break;
+      }
+      case 'TININT': {
+        column.type = new V1_TinyInt();
+        break;
+      }
+      case 'SMALLINT': {
+        column.type = new V1_SmallInt();
+        break;
+      }
+      case 'BIGINT': {
+        column.type = new V1_BigInt();
+        break;
+      }
+      case 'TIMESTAMP': {
+        column.type = new V1_Timestamp();
+        break;
+      }
+      case 'VARCHAR': {
+        column.type = new V1_VarChar();
+        break;
+      }
+      default: {
+        throw new UnsupportedOperationError(
+          `Can't ingest local file data: failed to find matching relational data type for DuckDB type '${col[1]}' when synthesizing table definition`,
+        );
+      }
+    }
+  }
+
   private _synthesizeMinimalModelContext(data: {
     schemaName: string;
     tableName: string;

package/src/stores/LegendDataCubeDuckDBEngine.ts

@@ -106,6 +106,12 @@ export class LegendDataCubeDuckDBEngine {
     const database = new duckdb.AsyncDuckDB(logger, worker);
     await database.instantiate(bundle.mainModule, bundle.pthreadWorker);
     this._database = database;
+
+    const connection = await this.database.connect();
+    await connection.query(`SET builtin_httpfs = false;`);
+    await connection.query(
+      `INSTALL iceberg FROM 'https://nightly-extensions.duckdb.org/iceberg/caca3ac6';`,
+    );
   }
 
   async cache(result: TDSExecutionResult) {
@@ -265,6 +271,72 @@ export class LegendDataCubeDuckDBEngine {
     };
   }
 
+  async ingestIcebergTable(
+    warehouse: string,
+    paths: string[],
+    catalogApi: string,
+    refId?: string,
+    token?: string,
+  ) {
+    if (!isNullable(refId) && this._catalog.has(refId)) {
+      guaranteeNonNullable(this._catalog.get(refId));
+      return {
+        dbReference: refId,
+      };
+    }
+
+    const schemaName = LegendDataCubeDuckDBEngine.DUCKDB_DEFAULT_SCHEMA_NAME;
+    LegendDataCubeDuckDBEngine.ingestFileTableCounter += 1;
+    const tableName = `${LegendDataCubeDuckDBEngine.INGEST_TABLE_NAME_PREFIX}${LegendDataCubeDuckDBEngine.ingestFileTableCounter}`;
+
+    const connection = await this.database.connect();
+
+    const secret = `CREATE OR REPLACE SECRET iceberg_secret (
+      TYPE ICEBERG,
+      TOKEN '${token}'
+    );`;
+    await connection.query(secret);
+
+    const catalog = `ATTACH OR REPLACE '${warehouse}' AS iceberg_catalog (
+      TYPE iCEBERG,
+      SECRET iceberg_secret,
+      ENDPOINT '${catalogApi}',
+      SUPPORT_NESTED_NAMESPACES true
+    );`;
+    await connection.query(catalog);
+
+    const selectQuery = `SELECT * from iceberg_catalog."${paths[0]}.${paths[1]}".${paths[2]};`;
+    const results = await connection.query(selectQuery);
+
+    await connection.insertArrowTable(results, {
+      name: tableName,
+      create: true,
+      schema: schemaName,
+    });
+
+    const describeQuery = `DESCRIBE ${schemaName}.${tableName};`;
+    const describeResult = await connection.query(describeQuery);
+
+    const tableSpec = describeResult
+      .toArray()
+      .map((spec) => [
+        spec[LegendDataCubeDuckDBEngine.COLUMN_NAME] as string,
+        spec[LegendDataCubeDuckDBEngine.COLUMN_TYPE] as string,
+      ]);
+    await connection.close();
+
+    const ref = isNullable(refId) ? uuid() : refId;
+    this._catalog.set(ref, {
+      schemaName,
+      tableName,
+      columns: tableSpec,
+    } satisfies DuckDBCatalogTable);
+
+    return {
+      dbReference: ref,
+    };
+  }
+
   async runSQLQuery(sql: string) {
     const connection = await this.database.connect();
     const result = await connection.query(sql);
@@ -374,7 +446,7 @@ export class LegendDataCubeDuckDBEngine {
   }
 }
 
-type DuckDBCatalogTable = {
+export type DuckDBCatalogTable = {
   schemaName: string;
   tableName: string;
   columns: string[][];