@finos/legend-application-data-cube 0.3.2 → 0.3.4

package/src/components/builder/source/LegendDataCubeSourceLoader.tsx

@@ -0,0 +1,111 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { observer } from 'mobx-react-lite';
+import { FormButton } from '@finos/legend-data-cube';
+import { useLegendDataCubeBuilderStore } from '../LegendDataCubeBuilderStoreProvider.js';
+import { LocalFileDataCubeSourceLoaderState } from '../../../stores/builder/source/LocalFileDataCubeSourceLoaderState.js';
+import { LocalFileDataCubePartialSourceLoader } from './LocalFileDataCubeSourceLoader.js';
+import { DataCubeIcon } from '@finos/legend-art';
+import { formatDistanceToNow } from '@finos/legend-shared';
+import type { LegendDataCubeSourceLoaderState } from '../../../stores/builder/source/LegendDataCubeSourceLoaderState.js';
+import { LegendDataCubeBlockingWindow } from '../../LegendDataCubeBlockingWindow.js';
+
+export const LegendDataCubeSourceLoader = observer(
+  (props: { state: LegendDataCubeSourceLoaderState }) => {
+    const { state } = props;
+    const store = useLegendDataCubeBuilderStore();
+    const persistentDataCube = state.persistentDataCube;
+
+    return (
+      <>
+        <div className="h-[calc(100%_-_40px)] w-full px-2 pt-2">
+          <div className="h-full w-full border border-neutral-300 bg-white">
+            <div className="h-full w-full select-none p-2">
+              <div className="relative mb-0.5 flex h-[42px] w-full border border-neutral-200 bg-neutral-100">
+                <div className="w-full">
+                  <div className="h-6 w-4/5 overflow-hidden text-ellipsis whitespace-nowrap px-1.5 leading-6">
+                    {persistentDataCube.name}
+                  </div>
+                  <div className="flex h-[18px] items-start justify-between px-1.5">
+                    <div className="flex">
+                      <DataCubeIcon.ClockEdit className="text-sm text-neutral-500" />
+                      <div className="ml-1 text-sm text-neutral-500">
+                        {persistentDataCube.lastUpdatedAt
+                          ? formatDistanceToNow(
+                              new Date(persistentDataCube.lastUpdatedAt),
+                              {
+                                includeSeconds: true,
+                                addSuffix: true,
+                              },
+                            )
+                          : '(unknown)'}
+                      </div>
+                    </div>
+                    <div className="flex">
+                      <DataCubeIcon.User className="text-sm text-neutral-500" />
+                      <div className="ml-1 text-sm text-neutral-500">
+                        {persistentDataCube.owner}
+                      </div>
+                    </div>
+                  </div>
+                </div>
+              </div>
+              <div className="flex h-10 w-full items-center">
+                <div className="flex h-full w-32 flex-shrink-0 items-center text-sm">
+                  Source Type:
+                  <div className="pl-3">{state.label}</div>
+                </div>
+              </div>
+              <div className="h-[calc(100%_-_41px)] w-full overflow-auto">
+                {state instanceof LocalFileDataCubeSourceLoaderState && (
+                  <LocalFileDataCubePartialSourceLoader
+                    partialSourceLoader={state}
+                  />
+                )}
+              </div>
+            </div>
+          </div>
+        </div>
+        <div className="flex h-10 items-center justify-end px-2">
+          <FormButton onClick={state.display.onClose}>Cancel</FormButton>
+          <FormButton
+            className="ml-2"
+            disabled={!state.isValid || state.finalizeState.isInProgress}
+            onClick={() => {
+              state.finalize().catch((error) => {
+                store.alertService.alertUnhandledError(error);
+              });
+            }}
+          >
+            OK
+          </FormButton>
+        </div>
+      </>
+    );
+  },
+);
+
+export const LegendDataCubeSourceLoaderBlockingWindow = observer(() => {
+  const store = useLegendDataCubeBuilderStore();
+
+  if (!store.sourceLoader) {
+    return null;
+  }
+  return (
+    <LegendDataCubeBlockingWindow windowState={store.sourceLoader.display} />
+  );
+});

package/src/components/builder/source/LocalFileDataCubeSourceBuilder.tsx

@@ -30,8 +30,7 @@ export const LocalFileDataCubeSourceBuilder = observer(
           type={AlertType.WARNING}
           text={`Currently, support for local file comes with the following limitations:
 - Only CSV files are supported, but not all variants of CSV files are supported (required header row, comma delimiter, single escape quote).
-- Data from uploaded file will not be stored nor shared.
-- DataCube created with local file source cannot be saved.`}
+- Data from uploaded file will not be stored nor shared.`}
         />
         <div className="mt-2 flex h-6 w-full items-center text-neutral-500">
           <input

package/src/components/builder/source/LocalFileDataCubeSourceLoader.tsx

@@ -0,0 +1,58 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { observer } from 'mobx-react-lite';
+import { AlertType, FormAlert, FormCodeEditor } from '@finos/legend-data-cube';
+import { CODE_EDITOR_LANGUAGE } from '@finos/legend-code-editor';
+import type { LocalFileDataCubeSourceLoaderState } from '../../../stores/builder/source/LocalFileDataCubeSourceLoaderState.js';
+
+export const LocalFileDataCubePartialSourceLoader = observer(
+  (props: { partialSourceLoader: LocalFileDataCubeSourceLoaderState }) => {
+    const { partialSourceLoader } = props;
+
+    return (
+      <div className="h-full w-full">
+        <FormAlert
+          message="Local file support is experimental"
+          type={AlertType.WARNING}
+          text={`Currently, support for local file comes with the following limitations:
+- Only CSV files are supported, but not all variants of CSV files are supported (required header row, comma delimiter, single escape quote).
+- Data from uploaded file will not be stored nor shared.`}
+        />
+        <div className="mt-2 flex h-6 w-full items-center text-neutral-500">
+          <input
+            type="file"
+            onChange={(event) => {
+              partialSourceLoader.processFile(event.target.files?.[0]);
+            }}
+            className="w-full"
+          />
+        </div>
+        {partialSourceLoader.previewText !== undefined && (
+          <div className="mt-2 h-40">
+            <FormCodeEditor
+              value={partialSourceLoader.previewText}
+              language={CODE_EDITOR_LANGUAGE.TEXT}
+              isReadOnly={true}
+              hidePadding={true}
+              title="Data Preview"
+            />
+          </div>
+        )}
+      </div>
+    );
+  },
+);

package/src/stores/LegendDataCubeBaseStore.ts

@@ -23,8 +23,9 @@ import type { LegendDataCubePluginManager } from '../application/LegendDataCubeP
 import { DepotServerClient } from '@finos/legend-server-depot';
 import type { LegendDataCubeApplicationConfig } from '../application/LegendDataCubeApplicationConfig.js';
 import {
-  V1_EngineServerClient,
+  type V1_EngineServerClient,
   V1_PureGraphManager,
+  V1_RemoteEngine,
 } from '@finos/legend-graph';
 import {
   ActionState,
@@ -61,6 +62,7 @@ export class LegendDataCubeBaseStore {
   readonly pluginManager: LegendDataCubePluginManager;
   readonly depotServerClient: DepotServerClient;
   readonly graphManager: V1_PureGraphManager;
+  readonly remoteEngine: V1_RemoteEngine;
   readonly engineServerClient: V1_EngineServerClient;
 
   readonly engine: LegendDataCubeDataCubeEngine;
@@ -116,11 +118,15 @@ export class LegendDataCubeBaseStore {
       } satisfies DataCubeSetting<string>,
     ];
 
-    this.engineServerClient = new V1_EngineServerClient({
-      baseUrl: this.getEngineServerBaseUrlSettingValue(),
-      queryBaseUrl: this.application.config.engineQueryServerUrl,
-      enableCompression: this.getEngineEnableCompressionSettingValue(),
-    });
+    this.remoteEngine = new V1_RemoteEngine(
+      {
+        baseUrl: this.getEngineServerBaseUrlSettingValue(),
+        queryBaseUrl: this.application.config.engineQueryServerUrl,
+        enableCompression: this.getEngineEnableCompressionSettingValue(),
+      },
+      application.logService,
+    );
+    this.engineServerClient = this.remoteEngine.getEngineServerClient();
     this.engineServerClient.setTracerService(application.tracerService);
 
     this.engine = new LegendDataCubeDataCubeEngine(
@@ -200,6 +206,7 @@ export class LegendDataCubeBaseStore {
           },
         },
         {
+          engine: this.remoteEngine,
           tracerService: this.application.tracerService,
         },
       );

package/src/stores/LegendDataCubeDataCubeEngine.ts

@@ -82,6 +82,8 @@ import {
   V1_deserializePureModelContext,
   type V1_ConcreteFunctionDefinition,
   V1_deserializeValueSpecification,
+  LET_TOKEN,
+  V1_AppliedFunction,
 } from '@finos/legend-graph';
 import {
   _elementPtr,
@@ -101,6 +103,8 @@ import {
   DataCubeExecutionError,
   RawUserDefinedFunctionDataCubeSource,
   ADHOC_FUNCTION_DATA_CUBE_SOURCE_TYPE,
+  UserDefinedFunctionDataCubeSource,
+  DataCubeQueryFilterOperator,
 } from '@finos/legend-data-cube';
 import {
   isNonNullable,
@@ -199,12 +203,87 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
           RawLocalFileQueryDataCubeSource.serialization.fromJson(value);
         const source = new LocalFileDataCubeSource();
         source.fileName = rawSource.fileName;
-        source.count = rawSource.count;
-        source.db = rawSource.db;
-        source.model = rawSource.model;
-        source.runtime = rawSource.runtime;
-        source.schema = rawSource.schema;
-        source.table = rawSource.table;
+        source.fileFormat = rawSource.fileFormat;
+
+        const tableCatalog = this._duckDBEngine.retrieveCatalogTable(
+          rawSource._ref,
+        );
+
+        const { model, database, schema, table, runtime } =
+          this._synthesizeMinimalModelContext({
+            schemaName: tableCatalog.schemaName,
+            tableName: tableCatalog.tableName,
+            tableColumns: tableCatalog.columns.map((col) => {
+              const column = new V1_Column();
+              column.name = col[0] as string;
+              // TODO: confirm this is in accordance to engine
+              // check if we have a duckdb enum mapping
+              // See https://duckdb.org/docs/sql/data_types/overview.html
+              switch (col[1] as string) {
+                case 'BIT': {
+                  column.type = new V1_Bit();
+                  break;
+                }
+                case 'BOOLEAN': {
+                  // TODO: understand why boolean is not present in relationalDataType
+                  column.type = new V1_VarChar();
+                  break;
+                }
+                case 'DATE': {
+                  column.type = new V1_Date();
+                  break;
+                }
+                case 'DECIMAL': {
+                  column.type = new V1_Decimal();
+                  break;
+                }
+                case 'DOUBLE': {
+                  column.type = new V1_Double();
+                  break;
+                }
+                case 'FLOAT': {
+                  column.type = new V1_Float();
+                  break;
+                }
+                case 'INTEGER': {
+                  column.type = new V1_Integer();
+                  break;
+                }
+                case 'TININT': {
+                  column.type = new V1_TinyInt();
+                  break;
+                }
+                case 'SMALLINT': {
+                  column.type = new V1_SmallInt();
+                  break;
+                }
+                case 'BIGINT': {
+                  column.type = new V1_BigInt();
+                  break;
+                }
+                case 'TIMESTAMP': {
+                  column.type = new V1_Timestamp();
+                  break;
+                }
+                case 'VARCHAR': {
+                  column.type = new V1_VarChar();
+                  break;
+                }
+                default: {
+                  throw new UnsupportedOperationError(
+                    `Can't ingest local file data: failed to find matching relational data type for DuckDB type '${col[1]}' when synthesizing table definition`,
+                  );
+                }
+              }
+              return column;
+            }),
+          });
+
+        source.db = database.path;
+        source.model = model;
+        source.table = table.name;
+        source.schema = schema.name;
+        source.runtime = runtime.path;
 
         const query = new V1_ClassInstance();
         query.type = V1_ClassInstanceType.RELATION_STORE_ACCESSOR;
@@ -244,7 +323,8 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
           );
         }
 
-        const source = new AdhocQueryDataCubeSource();
+        const source = new UserDefinedFunctionDataCubeSource();
+        source.functionPath = rawSource.functionPath;
         source.runtime = rawSource.runtime;
         source.model = rawSource.model;
         if (deserializedModel.sdlcInfo instanceof V1_LegendSDLC) {
@@ -327,32 +407,6 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
           ),
         );
         source.query = at(source.lambda.body, 0);
-        // use the default parameter values from the query
-        //
-        // TODO?: we should probably allow configuring the parameters?
-        // this would mean we need to create first-class support for parameters in DataCube component
-        const parameterValues = await Promise.all(
-          source.lambda.parameters.map(async (parameter) => {
-            if (parameter.genericType?.rawType instanceof V1_PackageableType) {
-              const paramValue = new V1_ParameterValue();
-              paramValue.name = parameter.name;
-              const type = parameter.genericType.rawType.fullPath;
-              const defaultValue = queryInfo.defaultParameterValues?.find(
-                (val) => val.name === parameter.name,
-              )?.content;
-              paramValue.value =
-                defaultValue !== undefined
-                  ? await this.parseValueSpecification(defaultValue)
-                  : {
-                      _type: V1_deserializeRawValueSpecificationType(type),
-                      value: _defaultPrimitiveTypeValue(type),
-                    };
-              return paramValue;
-            }
-            return undefined;
-          }),
-        );
-        source.parameterValues = parameterValues.filter(isNonNullable);
         try {
           source.columns = (
             await this._getLambdaRelationType(
@@ -366,6 +420,59 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
             `Can't get query result columns. Make sure the saved query return a relation (i.e. typed TDS). Error: ${error.message}`,
           );
         }
+        // To handle parameter value with function calls we
+        // 1. Separate the parameters with function calls from regular parameters
+        // 2. Add let statements for function parameter values and store them in the source's letParameterValueSpec
+        // 3. Prepend the let statements to the lambda body when we execute the query
+        const letFuncs: V1_ValueSpecification[] = [];
+        const parameterValues = (
+          await Promise.all(
+            source.lambda.parameters.map(async (parameter) => {
+              if (
+                parameter.genericType?.rawType instanceof V1_PackageableType
+              ) {
+                const type = parameter.genericType.rawType.fullPath;
+                const defaultValueString =
+                  queryInfo.defaultParameterValues?.find(
+                    (val) => val.name === parameter.name,
+                  )?.content;
+                const defaultValueSpec =
+                  defaultValueString !== undefined
+                    ? await this.parseValueSpecification(defaultValueString)
+                    : {
+                        _type: V1_deserializeRawValueSpecificationType(type),
+                        value: _defaultPrimitiveTypeValue(type),
+                      };
+                if (defaultValueSpec instanceof V1_AppliedFunction) {
+                  const letFunc = guaranteeType(
+                    this.deserializeValueSpecification(
+                      await this._engineServerClient.grammarToJSON_lambda(
+                        `${LET_TOKEN} ${parameter.name} ${DataCubeQueryFilterOperator.EQUAL} ${defaultValueString}`,
+                        '',
+                        undefined,
+                        undefined,
+                        false,
+                      ),
+                    ),
+                    V1_Lambda,
+                  );
+                  letFuncs.push(...letFunc.body);
+                } else {
+                  const paramValue = new V1_ParameterValue();
+                  paramValue.name = parameter.name;
+                  paramValue.value = defaultValueSpec;
+                  return paramValue;
+                }
+              }
+              return undefined;
+            }),
+          )
+        ).filter(isNonNullable);
+        source.letParameterValueSpec = letFuncs;
+        source.parameterValues = parameterValues;
+        source.lambda.parameters = source.lambda.parameters.filter((param) =>
+          parameterValues.find((p) => p.name === param.name),
+        );
         return source;
       }
       default:
@@ -434,6 +541,13 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
           model: source.model,
         })
       ).completions as CompletionItem[];
+    } else if (source instanceof UserDefinedFunctionDataCubeSource) {
+      return (
+        await this._engineServerClient.completeCode({
+          codeBlock,
+          model: source.model,
+        })
+      ).completions as CompletionItem[];
     } else if (source instanceof LegendQueryDataCubeSource) {
       return (
         await this._engineServerClient.completeCode({
@@ -524,8 +638,11 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
     try {
       if (source instanceof AdhocQueryDataCubeSource) {
         result = await this._runQuery(query, source.model, undefined, options);
+      } else if (source instanceof UserDefinedFunctionDataCubeSource) {
+        result = await this._runQuery(query, source.model, undefined, options);
       } else if (source instanceof LegendQueryDataCubeSource) {
         query.parameters = source.lambda.parameters;
+        query.body = [...source.letParameterValueSpec, ...query.body];
         result = await this._runQuery(
           query,
           source.model,
@@ -598,19 +715,14 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
         TDSExecutionResult,
         `Can't process execution result: expected tabular data set format`,
       );
-      const endTime = performance.now();
-      const queryCode = await queryCodePromise;
-      const sql = guaranteeNonNullable(
-        result.activities?.[0] instanceof RelationalExecutionActivities
-          ? result.activities[0].sql
-          : undefined,
-        `Can't process execution result: failed to extract generated SQL`,
-      );
       return {
         result: result,
-        executedQuery: queryCode,
-        executedSQL: sql,
-        executionTime: endTime - startTime,
+        executedQuery: await queryCodePromise,
+        executedSQL:
+          result.activities?.at(-1) instanceof RelationalExecutionActivities
+            ? (result.activities.at(-1) as RelationalExecutionActivities).sql
+            : undefined,
+        executionTime: performance.now() - startTime,
       };
     } catch (error) {
       assertErrorThrown(error);
@@ -631,6 +743,11 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
         DataCubeFunction.FROM,
         [_elementPtr(source.runtime)].filter(isNonNullable),
       );
+    } else if (source instanceof UserDefinedFunctionDataCubeSource) {
+      return _function(
+        DataCubeFunction.FROM,
+        [_elementPtr(source.runtime)].filter(isNonNullable),
+      );
     } else if (source instanceof LegendQueryDataCubeSource) {
       return _function(
         DataCubeFunction.FROM,
@@ -748,6 +865,8 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
   ) {
     if (source instanceof AdhocQueryDataCubeSource) {
       return this._getLambdaRelationType(query, source.model);
+    } else if (source instanceof UserDefinedFunctionDataCubeSource) {
+      return this._getLambdaRelationType(query, source.model);
     } else if (source instanceof LegendQueryDataCubeSource) {
       return this._getLambdaRelationType(query, source.model);
     } else if (source instanceof CachedDataCubeSource) {
@@ -853,93 +972,10 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
     }
   }
 
-  async ingestLocalFileData(
-    data: string,
-    format: string,
-  ): Promise<DataCubeSource | undefined> {
-    const {
-      schema: schemaName,
-      table: tableName,
-      tableSpec,
-    } = await this._duckDBEngine.ingestLocalFileData(data, format);
-
-    const { model, database, schema, table, runtime } =
-      this._synthesizeMinimalModelContext({
-        schemaName,
-        tableName,
-        tableColumns: tableSpec.map((col) => {
-          const column = new V1_Column();
-          column.name = col[0] as string;
-          // TODO: confirm this is in accordance to engine
-          // check if we have a duckdb enum mapping
-          // See https://duckdb.org/docs/sql/data_types/overview.html
-          switch (col[1] as string) {
-            case 'BIT': {
-              column.type = new V1_Bit();
-              break;
-            }
-            case 'BOOLEAN': {
-              // TODO: understand why boolean is not present in relationalDataType
-              column.type = new V1_VarChar();
-              break;
-            }
-            case 'DATE': {
-              column.type = new V1_Date();
-              break;
-            }
-            case 'DECIMAL': {
-              column.type = new V1_Decimal();
-              break;
-            }
-            case 'DOUBLE': {
-              column.type = new V1_Double();
-              break;
-            }
-            case 'FLOAT': {
-              column.type = new V1_Float();
-              break;
-            }
-            case 'INTEGER': {
-              column.type = new V1_Integer();
-              break;
-            }
-            case 'TININT': {
-              column.type = new V1_TinyInt();
-              break;
-            }
-            case 'SMALLINT': {
-              column.type = new V1_SmallInt();
-              break;
-            }
-            case 'BIGINT': {
-              column.type = new V1_BigInt();
-              break;
-            }
-            case 'TIMESTAMP': {
-              column.type = new V1_Timestamp();
-              break;
-            }
-            case 'VARCHAR': {
-              column.type = new V1_VarChar();
-              break;
-            }
-            default: {
-              throw new UnsupportedOperationError(
-                `Can't ingest local file data: failed to find matching relational data type for DuckDB type '${col[1]}' when synthesizing table definition`,
-              );
-            }
-          }
-          return column;
-        }),
-      });
-
-    const source = new LocalFileDataCubeSource();
-    source.model = model;
-    source.runtime = runtime.path;
-    source.db = database.path;
-    source.schema = schema.name;
-    source.table = table.name;
-    return source;
+  async ingestLocalFileData(data: string, format: string, refId?: string) {
+    const { dbReference, columnNames } =
+      await this._duckDBEngine.ingestLocalFileData(data, format, refId);
+    return { dbReference, columnNames };
   }
 
   private _synthesizeMinimalModelContext(data: {