@finos/legend-application-data-cube 0.3.2 → 0.3.3

package/src/components/builder/source/LocalFileDataCubeSourceBuilder.tsx

@@ -30,8 +30,7 @@ export const LocalFileDataCubeSourceBuilder = observer(
           type={AlertType.WARNING}
           text={`Currently, support for local file comes with the following limitations:
 - Only CSV files are supported, but not all variants of CSV files are supported (required header row, comma delimiter, single escape quote).
-- Data from uploaded file will not be stored nor shared.
-- DataCube created with local file source cannot be saved.`}
+- Data from uploaded file will not be stored nor shared.`}
         />
         <div className="mt-2 flex h-6 w-full items-center text-neutral-500">
           <input

package/src/components/builder/source/loader/LocalFileDataCubePartialSourceLoader.tsx

@@ -0,0 +1,60 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { observer } from 'mobx-react-lite';
+import { AlertType, FormAlert, FormCodeEditor } from '@finos/legend-data-cube';
+import { CODE_EDITOR_LANGUAGE } from '@finos/legend-code-editor';
+import type { LocalFileDataCubePartialSourceLoaderState } from '../../../../stores/builder/source/loader/LocalFileDataCubePartialSourceLoaderState.js';
+
+export const LocalFileDataCubePartialSourceLoader = observer(
+  (props: {
+    partialSourceLoader: LocalFileDataCubePartialSourceLoaderState;
+  }) => {
+    const { partialSourceLoader } = props;
+
+    return (
+      <div className="h-full w-full">
+        <FormAlert
+          message="Local file support is experimental"
+          type={AlertType.WARNING}
+          text={`Currently, support for local file comes with the following limitations:
+- Only CSV files are supported, but not all variants of CSV files are supported (required header row, comma delimiter, single escape quote).
+- Data from uploaded file will not be stored nor shared.`}
+        />
+        <div className="mt-2 flex h-6 w-full items-center text-neutral-500">
+          <input
+            type="file"
+            onChange={(event) => {
+              partialSourceLoader.processFile(event.target.files?.[0]);
+            }}
+            className="w-full"
+          />
+        </div>
+        {partialSourceLoader.previewText !== undefined && (
+          <div className="mt-2 h-40">
+            <FormCodeEditor
+              value={partialSourceLoader.previewText}
+              language={CODE_EDITOR_LANGUAGE.TEXT}
+              isReadOnly={true}
+              hidePadding={true}
+              title="Data Preview"
+            />
+          </div>
+        )}
+      </div>
+    );
+  },
+);

package/src/stores/LegendDataCubeBaseStore.ts

@@ -23,8 +23,9 @@ import type { LegendDataCubePluginManager } from '../application/LegendDataCubeP
 import { DepotServerClient } from '@finos/legend-server-depot';
 import type { LegendDataCubeApplicationConfig } from '../application/LegendDataCubeApplicationConfig.js';
 import {
-  V1_EngineServerClient,
+  type V1_EngineServerClient,
   V1_PureGraphManager,
+  V1_RemoteEngine,
 } from '@finos/legend-graph';
 import {
   ActionState,
@@ -61,6 +62,7 @@ export class LegendDataCubeBaseStore {
   readonly pluginManager: LegendDataCubePluginManager;
   readonly depotServerClient: DepotServerClient;
   readonly graphManager: V1_PureGraphManager;
+  readonly remoteEngine: V1_RemoteEngine;
   readonly engineServerClient: V1_EngineServerClient;
 
   readonly engine: LegendDataCubeDataCubeEngine;
@@ -116,11 +118,15 @@ export class LegendDataCubeBaseStore {
       } satisfies DataCubeSetting<string>,
     ];
 
-    this.engineServerClient = new V1_EngineServerClient({
-      baseUrl: this.getEngineServerBaseUrlSettingValue(),
-      queryBaseUrl: this.application.config.engineQueryServerUrl,
-      enableCompression: this.getEngineEnableCompressionSettingValue(),
-    });
+    this.remoteEngine = new V1_RemoteEngine(
+      {
+        baseUrl: this.getEngineServerBaseUrlSettingValue(),
+        queryBaseUrl: this.application.config.engineQueryServerUrl,
+        enableCompression: this.getEngineEnableCompressionSettingValue(),
+      },
+      application.logService,
+    );
+    this.engineServerClient = this.remoteEngine.getEngineServerClient();
     this.engineServerClient.setTracerService(application.tracerService);
 
     this.engine = new LegendDataCubeDataCubeEngine(
@@ -200,6 +206,7 @@ export class LegendDataCubeBaseStore {
           },
         },
         {
+          engine: this.remoteEngine,
           tracerService: this.application.tracerService,
         },
       );

package/src/stores/LegendDataCubeDataCubeEngine.ts

@@ -82,6 +82,8 @@ import {
   V1_deserializePureModelContext,
   type V1_ConcreteFunctionDefinition,
   V1_deserializeValueSpecification,
+  LET_TOKEN,
+  V1_AppliedFunction,
 } from '@finos/legend-graph';
 import {
   _elementPtr,
@@ -101,6 +103,8 @@ import {
   DataCubeExecutionError,
   RawUserDefinedFunctionDataCubeSource,
   ADHOC_FUNCTION_DATA_CUBE_SOURCE_TYPE,
+  UserDefinedFunctionDataCubeSource,
+  DataCubeQueryFilterOperator,
 } from '@finos/legend-data-cube';
 import {
   isNonNullable,
@@ -199,12 +203,87 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
           RawLocalFileQueryDataCubeSource.serialization.fromJson(value);
         const source = new LocalFileDataCubeSource();
         source.fileName = rawSource.fileName;
-        source.count = rawSource.count;
-        source.db = rawSource.db;
-        source.model = rawSource.model;
-        source.runtime = rawSource.runtime;
-        source.schema = rawSource.schema;
-        source.table = rawSource.table;
+        source.fileFormat = rawSource.fileFormat;
+
+        const tableCatalog = this._duckDBEngine.retrieveCatalogTable(
+          rawSource._ref,
+        );
+
+        const { model, database, schema, table, runtime } =
+          this._synthesizeMinimalModelContext({
+            schemaName: tableCatalog.schemaName,
+            tableName: tableCatalog.tableName,
+            tableColumns: tableCatalog.columns.map((col) => {
+              const column = new V1_Column();
+              column.name = col[0] as string;
+              // TODO: confirm this is in accordance to engine
+              // check if we have a duckdb enum mapping
+              // See https://duckdb.org/docs/sql/data_types/overview.html
+              switch (col[1] as string) {
+                case 'BIT': {
+                  column.type = new V1_Bit();
+                  break;
+                }
+                case 'BOOLEAN': {
+                  // TODO: understand why boolean is not present in relationalDataType
+                  column.type = new V1_VarChar();
+                  break;
+                }
+                case 'DATE': {
+                  column.type = new V1_Date();
+                  break;
+                }
+                case 'DECIMAL': {
+                  column.type = new V1_Decimal();
+                  break;
+                }
+                case 'DOUBLE': {
+                  column.type = new V1_Double();
+                  break;
+                }
+                case 'FLOAT': {
+                  column.type = new V1_Float();
+                  break;
+                }
+                case 'INTEGER': {
+                  column.type = new V1_Integer();
+                  break;
+                }
+                case 'TININT': {
+                  column.type = new V1_TinyInt();
+                  break;
+                }
+                case 'SMALLINT': {
+                  column.type = new V1_SmallInt();
+                  break;
+                }
+                case 'BIGINT': {
+                  column.type = new V1_BigInt();
+                  break;
+                }
+                case 'TIMESTAMP': {
+                  column.type = new V1_Timestamp();
+                  break;
+                }
+                case 'VARCHAR': {
+                  column.type = new V1_VarChar();
+                  break;
+                }
+                default: {
+                  throw new UnsupportedOperationError(
+                    `Can't ingest local file data: failed to find matching relational data type for DuckDB type '${col[1]}' when synthesizing table definition`,
+                  );
+                }
+              }
+              return column;
+            }),
+          });
+
+        source.db = database.path;
+        source.model = model;
+        source.table = table.name;
+        source.schema = schema.name;
+        source.runtime = runtime.path;
 
         const query = new V1_ClassInstance();
         query.type = V1_ClassInstanceType.RELATION_STORE_ACCESSOR;
@@ -244,7 +323,8 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
           );
         }
 
-        const source = new AdhocQueryDataCubeSource();
+        const source = new UserDefinedFunctionDataCubeSource();
+        source.functionPath = rawSource.functionPath;
         source.runtime = rawSource.runtime;
         source.model = rawSource.model;
         if (deserializedModel.sdlcInfo instanceof V1_LegendSDLC) {
@@ -327,32 +407,6 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
           ),
         );
         source.query = at(source.lambda.body, 0);
-        // use the default parameter values from the query
-        //
-        // TODO?: we should probably allow configuring the parameters?
-        // this would mean we need to create first-class support for parameters in DataCube component
-        const parameterValues = await Promise.all(
-          source.lambda.parameters.map(async (parameter) => {
-            if (parameter.genericType?.rawType instanceof V1_PackageableType) {
-              const paramValue = new V1_ParameterValue();
-              paramValue.name = parameter.name;
-              const type = parameter.genericType.rawType.fullPath;
-              const defaultValue = queryInfo.defaultParameterValues?.find(
-                (val) => val.name === parameter.name,
-              )?.content;
-              paramValue.value =
-                defaultValue !== undefined
-                  ? await this.parseValueSpecification(defaultValue)
-                  : {
-                      _type: V1_deserializeRawValueSpecificationType(type),
-                      value: _defaultPrimitiveTypeValue(type),
-                    };
-              return paramValue;
-            }
-            return undefined;
-          }),
-        );
-        source.parameterValues = parameterValues.filter(isNonNullable);
         try {
           source.columns = (
             await this._getLambdaRelationType(
@@ -366,6 +420,59 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
             `Can't get query result columns. Make sure the saved query return a relation (i.e. typed TDS). Error: ${error.message}`,
           );
         }
+        // To handle parameter value with function calls we
+        // 1. Separate the parameters with function calls from regular parameters
+        // 2. Add let statements for function parameter values and store them in the source's letParameterValueSpec
+        // 3. Prepend the let statements to the lambda body when we execute the query
+        const letFuncs: V1_ValueSpecification[] = [];
+        const parameterValues = (
+          await Promise.all(
+            source.lambda.parameters.map(async (parameter) => {
+              if (
+                parameter.genericType?.rawType instanceof V1_PackageableType
+              ) {
+                const type = parameter.genericType.rawType.fullPath;
+                const defaultValueString =
+                  queryInfo.defaultParameterValues?.find(
+                    (val) => val.name === parameter.name,
+                  )?.content;
+                const defaultValueSpec =
+                  defaultValueString !== undefined
+                    ? await this.parseValueSpecification(defaultValueString)
+                    : {
+                        _type: V1_deserializeRawValueSpecificationType(type),
+                        value: _defaultPrimitiveTypeValue(type),
+                      };
+                if (defaultValueSpec instanceof V1_AppliedFunction) {
+                  const letFunc = guaranteeType(
+                    this.deserializeValueSpecification(
+                      await this._engineServerClient.grammarToJSON_lambda(
+                        `${LET_TOKEN} ${parameter.name} ${DataCubeQueryFilterOperator.EQUAL} ${defaultValueString}`,
+                        '',
+                        undefined,
+                        undefined,
+                        false,
+                      ),
+                    ),
+                    V1_Lambda,
+                  );
+                  letFuncs.push(...letFunc.body);
+                } else {
+                  const paramValue = new V1_ParameterValue();
+                  paramValue.name = parameter.name;
+                  paramValue.value = defaultValueSpec;
+                  return paramValue;
+                }
+              }
+              return undefined;
+            }),
+          )
+        ).filter(isNonNullable);
+        source.letParameterValueSpec = letFuncs;
+        source.parameterValues = parameterValues;
+        source.lambda.parameters = source.lambda.parameters.filter((param) =>
+          parameterValues.find((p) => p.name === param.name),
+        );
         return source;
       }
       default:
@@ -434,6 +541,13 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
           model: source.model,
         })
       ).completions as CompletionItem[];
+    } else if (source instanceof UserDefinedFunctionDataCubeSource) {
+      return (
+        await this._engineServerClient.completeCode({
+          codeBlock,
+          model: source.model,
+        })
+      ).completions as CompletionItem[];
     } else if (source instanceof LegendQueryDataCubeSource) {
       return (
         await this._engineServerClient.completeCode({
@@ -524,8 +638,11 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
     try {
       if (source instanceof AdhocQueryDataCubeSource) {
         result = await this._runQuery(query, source.model, undefined, options);
+      } else if (source instanceof UserDefinedFunctionDataCubeSource) {
+        result = await this._runQuery(query, source.model, undefined, options);
       } else if (source instanceof LegendQueryDataCubeSource) {
         query.parameters = source.lambda.parameters;
+        query.body = [...source.letParameterValueSpec, ...query.body];
         result = await this._runQuery(
           query,
           source.model,
@@ -631,6 +748,11 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
         DataCubeFunction.FROM,
         [_elementPtr(source.runtime)].filter(isNonNullable),
       );
+    } else if (source instanceof UserDefinedFunctionDataCubeSource) {
+      return _function(
+        DataCubeFunction.FROM,
+        [_elementPtr(source.runtime)].filter(isNonNullable),
+      );
     } else if (source instanceof LegendQueryDataCubeSource) {
       return _function(
         DataCubeFunction.FROM,
@@ -748,6 +870,8 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
   ) {
     if (source instanceof AdhocQueryDataCubeSource) {
       return this._getLambdaRelationType(query, source.model);
+    } else if (source instanceof UserDefinedFunctionDataCubeSource) {
+      return this._getLambdaRelationType(query, source.model);
     } else if (source instanceof LegendQueryDataCubeSource) {
       return this._getLambdaRelationType(query, source.model);
     } else if (source instanceof CachedDataCubeSource) {
@@ -853,93 +977,10 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
     }
   }
 
-  async ingestLocalFileData(
-    data: string,
-    format: string,
-  ): Promise<DataCubeSource | undefined> {
-    const {
-      schema: schemaName,
-      table: tableName,
-      tableSpec,
-    } = await this._duckDBEngine.ingestLocalFileData(data, format);
-
-    const { model, database, schema, table, runtime } =
-      this._synthesizeMinimalModelContext({
-        schemaName,
-        tableName,
-        tableColumns: tableSpec.map((col) => {
-          const column = new V1_Column();
-          column.name = col[0] as string;
-          // TODO: confirm this is in accordance to engine
-          // check if we have a duckdb enum mapping
-          // See https://duckdb.org/docs/sql/data_types/overview.html
-          switch (col[1] as string) {
-            case 'BIT': {
-              column.type = new V1_Bit();
-              break;
-            }
-            case 'BOOLEAN': {
-              // TODO: understand why boolean is not present in relationalDataType
-              column.type = new V1_VarChar();
-              break;
-            }
-            case 'DATE': {
-              column.type = new V1_Date();
-              break;
-            }
-            case 'DECIMAL': {
-              column.type = new V1_Decimal();
-              break;
-            }
-            case 'DOUBLE': {
-              column.type = new V1_Double();
-              break;
-            }
-            case 'FLOAT': {
-              column.type = new V1_Float();
-              break;
-            }
-            case 'INTEGER': {
-              column.type = new V1_Integer();
-              break;
-            }
-            case 'TININT': {
-              column.type = new V1_TinyInt();
-              break;
-            }
-            case 'SMALLINT': {
-              column.type = new V1_SmallInt();
-              break;
-            }
-            case 'BIGINT': {
-              column.type = new V1_BigInt();
-              break;
-            }
-            case 'TIMESTAMP': {
-              column.type = new V1_Timestamp();
-              break;
-            }
-            case 'VARCHAR': {
-              column.type = new V1_VarChar();
-              break;
-            }
-            default: {
-              throw new UnsupportedOperationError(
-                `Can't ingest local file data: failed to find matching relational data type for DuckDB type '${col[1]}' when synthesizing table definition`,
-              );
-            }
-          }
-          return column;
-        }),
-      });
-
-    const source = new LocalFileDataCubeSource();
-    source.model = model;
-    source.runtime = runtime.path;
-    source.db = database.path;
-    source.schema = schema.name;
-    source.table = table.name;
-    return source;
+  async ingestLocalFileData(data: string, format: string, refId?: string) {
+    const { dbReference, columnNames } =
+      await this._duckDBEngine.ingestLocalFileData(data, format, refId);
+    return { dbReference, columnNames };
   }
 
   private _synthesizeMinimalModelContext(data: {

package/src/stores/LegendDataCubeDuckDBEngine.ts

@@ -29,7 +29,9 @@ import {
   assertNonNullable,
   csvStringify,
   guaranteeNonNullable,
+  isNullable,
   UnsupportedOperationError,
+  uuid,
 } from '@finos/legend-shared';
 import type { CachedDataCubeSource } from '@finos/legend-data-cube';
 import { Type } from 'apache-arrow';
@@ -48,6 +50,7 @@ export class LegendDataCubeDuckDBEngine {
   // Options for creating csv using papa parser: https://www.papaparse.com/docs#config
   private static readonly ESCAPE_CHAR = `'`;
   private static readonly QUOTE_CHAR = `'`;
+  private _catalog: Map<string, DuckDBCatalogTable> = new Map();
 
   private _database?: duckdb.AsyncDuckDB | undefined;
 
@@ -58,6 +61,13 @@ export class LegendDataCubeDuckDBEngine {
     );
   }
 
+  retrieveCatalogTable(ref: string) {
+    return guaranteeNonNullable(
+      this._catalog.get(ref),
+      `Can't find reference ${ref}`,
+    );
+  }
+
   async initialize() {
     // Initialize DuckDB with WASM
     // See: https://duckdb.org/docs/api/wasm/instantiation.html
@@ -99,35 +109,90 @@ export class LegendDataCubeDuckDBEngine {
     const table = `${LegendDataCubeDuckDBEngine.CACHE_TABLE_NAME_PREFIX}${LegendDataCubeDuckDBEngine.cacheTableCounter}`;
     const csvFileName = `${LegendDataCubeDuckDBEngine.CACHE_FILE_NAME}${LegendDataCubeDuckDBEngine.cacheTableCounter}.csv`;
 
+    const columns: string[] = [];
     const columnNames: string[] = [];
-    result.builder.columns.forEach((col) => columnNames.push(col.name));
+    result.builder.columns.forEach((col) => {
+      let colType: string;
+      switch (col.type as string) {
+        case PRIMITIVE_TYPE.BINARY: {
+          colType = 'BIT';
+          break;
+        }
+        case PRIMITIVE_TYPE.BOOLEAN: {
+          colType = 'BOOLEAN';
+          break;
+        }
+        case PRIMITIVE_TYPE.NUMBER: {
+          colType = 'DOUBLE';
+          break;
+        }
+        case PRIMITIVE_TYPE.INTEGER: {
+          colType = 'INTEGER';
+          break;
+        }
+        // TODO: we need precision and scale
+        case PRIMITIVE_TYPE.DECIMAL: {
+          colType = 'DECIMAL';
+          break;
+        }
+        case PRIMITIVE_TYPE.FLOAT: {
+          colType = 'FLOAT';
+          break;
+        }
+        case PRIMITIVE_TYPE.STRICTDATE:
+        case PRIMITIVE_TYPE.DATE: {
+          colType = 'DATE';
+          break;
+        }
+        case PRIMITIVE_TYPE.DATETIME: {
+          colType = 'TIMESTAMP';
+          break;
+        }
+        case PRIMITIVE_TYPE.STRING: {
+          colType = 'VARCHAR';
+          break;
+        }
+        default: {
+          throw new UnsupportedOperationError(
+            `Can't initialize cache: failed to find matching DuckDB type for Pure type '${col.type}'`,
+          );
+        }
+      }
+      columns.push(`"${col.name}" ${colType}`);
+      columnNames.push(col.name);
+    });
 
     const data = result.result.rows.map((row) => row.values);
 
-    const csvContent = csvStringify([columnNames, ...data], {
-      escapeChar: LegendDataCubeDuckDBEngine.ESCAPE_CHAR,
-      quoteChar: LegendDataCubeDuckDBEngine.QUOTE_CHAR,
-    });
+    const csvContent = csvStringify([columnNames, ...data]);
     await this.database.registerFileText(csvFileName, csvContent);
 
     const connection = await this.database.connect();
+
+    // we create our own table schema becuase of date type conversions from arrow to duckDB data types
+    const CREATE_TABLE_SQL = `CREATE TABLE ${schema}.${table} (${columns.join(',')})`;
+    await connection.query(CREATE_TABLE_SQL);
+
     await connection.insertCSVFromPath(csvFileName, {
       schema: schema,
       name: table,
-      create: true,
-      header: true,
+      create: false,
+      header: true, // we add header and get it to autodetect otherwise we would have to provide column details with arrow datatypes
       detect: true,
-      dateFormat: 'YYYY-MM-DD',
-      timestampFormat: 'YYYY-MM-DD', // make sure Date is not auto-converted to timestamp
-      escape: LegendDataCubeDuckDBEngine.ESCAPE_CHAR,
-      quote: LegendDataCubeDuckDBEngine.QUOTE_CHAR,
     });
     await connection.close();
 
     return { schema, table, rowCount: result.result.rows.length };
   }
 
-  async ingestLocalFileData(data: string, format: string) {
+  async ingestLocalFileData(data: string, format: string, refId?: string) {
+    if (!isNullable(refId) && this._catalog.has(refId)) {
+      const dbDetails = guaranteeNonNullable(this._catalog.get(refId));
+      return {
+        dbReference: refId,
+        columnNames: dbDetails.columns.map((col) => col[0] as string),
+      };
+    }
     const schema = LegendDataCubeDuckDBEngine.DUCKDB_DEFAULT_SCHEMA_NAME;
     LegendDataCubeDuckDBEngine.ingestFileTableCounter += 1;
     const table = `${LegendDataCubeDuckDBEngine.INGEST_TABLE_NAME_PREFIX}${LegendDataCubeDuckDBEngine.ingestFileTableCounter}`;
@@ -145,7 +210,7 @@ export class LegendDataCubeDuckDBEngine {
           header: true,
           detect: true,
           dateFormat: 'YYYY-MM-DD',
-          timestampFormat: 'YYYY-MM-DD', // make sure Date is not auto-converted to timestamp
+          timestampFormat: 'YYYY-MM-DD hh:mm:ss[.zzzzzzzzz][+-TT[:tt]]', // make sure Date is not auto-converted to timestamp
           escape: LegendDataCubeDuckDBEngine.ESCAPE_CHAR,
           quote: LegendDataCubeDuckDBEngine.QUOTE_CHAR,
         });
@@ -161,12 +226,22 @@ export class LegendDataCubeDuckDBEngine {
     const tableSpec = (await connection.query(`DESCRIBE ${schema}.${table}`))
       .toArray()
       .map((spec) => [
-        spec[LegendDataCubeDuckDBEngine.COLUMN_NAME],
-        spec[LegendDataCubeDuckDBEngine.COLUMN_TYPE],
+        spec[LegendDataCubeDuckDBEngine.COLUMN_NAME] as string,
+        spec[LegendDataCubeDuckDBEngine.COLUMN_TYPE] as string,
       ]);
     await connection.close();
 
-    return { schema, table, tableSpec };
+    const ref = isNullable(refId) ? uuid() : refId;
+    this._catalog.set(ref, {
+      schemaName: schema,
+      tableName: table,
+      columns: tableSpec,
+    } satisfies DuckDBCatalogTable);
+
+    return {
+      dbReference: ref,
+      columnNames: tableSpec.map((spec) => spec[0] as string),
+    };
   }
 
   async runSQLQuery(sql: string) {
@@ -210,13 +285,16 @@ export class LegendDataCubeDuckDBEngine {
           col.type = PRIMITIVE_TYPE.BOOLEAN;
           break;
         }
-        case Type.Timestamp:
         case Type.Date:
         case Type.DateDay:
         case Type.DateMillisecond: {
           col.type = PRIMITIVE_TYPE.DATE;
           break;
         }
+        case Type.Timestamp: {
+          col.type = PRIMITIVE_TYPE.DATETIME;
+          break;
+        }
         case Type.Utf8:
         case Type.LargeUtf8: {
           col.type = PRIMITIVE_TYPE.STRING;
@@ -269,3 +347,9 @@ export class LegendDataCubeDuckDBEngine {
     await this._database?.terminate();
   }
 }
+
+type DuckDBCatalogTable = {
+  schemaName: string;
+  tableName: string;
+  columns: string[][];
+};