@finos/legend-application-data-cube 0.3.2 → 0.3.3

package/src/stores/builder/source/loader/LocalFileDataCubePartialSourceLoaderState.ts

@@ -0,0 +1,217 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  ActionState,
+  csvStringify,
+  guaranteeNonNullable,
+  IllegalStateError,
+  parseCSVFile,
+  type PlainObject,
+} from '@finos/legend-shared';
+import { makeObservable, observable, action } from 'mobx';
+import type { LegendDataCubeApplicationStore } from '../../../LegendDataCubeBaseStore.js';
+import type { LegendDataCubeDataCubeEngine } from '../../../LegendDataCubeDataCubeEngine.js';
+import {
+  LocalFileDataCubeSourceFormat,
+  RawLocalFileQueryDataCubeSource,
+} from '../../../model/LocalFileDataCubeSource.js';
+import { LegendDataCubePartialSourceLoaderState } from './LegendDataCubePartialSourceLoaderState.js';
+import { LegendDataCubeSourceLoaderType } from '../../LegendDataCubeSourceLoaderState.js';
+
+export class LocalFileDataCubePartialSourceLoaderState extends LegendDataCubePartialSourceLoaderState {
+  readonly processState = ActionState.create();
+
+  fileName?: string | undefined;
+  fileFormat?: LocalFileDataCubeSourceFormat | undefined;
+  // NOTE: type string is suitable for CSV/Excel, etc. but will not be appropriate
+  // for other format that we want to support, e.g. arrow/parquet
+  fileData?: string | undefined;
+  previewText?: string | undefined;
+  rowCount?: number | undefined;
+  columnNames?: string[] | undefined;
+
+  constructor(
+    application: LegendDataCubeApplicationStore,
+    engine: LegendDataCubeDataCubeEngine,
+  ) {
+    super(application, engine);
+
+    makeObservable(this, {
+      fileName: observable,
+      setFileName: action,
+
+      columnNames: observable,
+      setColumnNames: action,
+
+      fileFormat: observable,
+      setFileFormat: action,
+
+      fileData: observable,
+      setFileData: action,
+
+      previewText: observable,
+      setPreviewText: action,
+
+      rowCount: observable,
+      setRowCount: action,
+    });
+  }
+
+  setFileName(fileName: string | undefined) {
+    this.fileName = fileName;
+  }
+
+  setColumnNames(columnNames: string[] | undefined) {
+    this.columnNames = columnNames;
+  }
+
+  setFileFormat(format: LocalFileDataCubeSourceFormat | undefined) {
+    this.fileFormat = format;
+  }
+
+  setFileData(data: string | undefined) {
+    this.fileData = data;
+  }
+
+  setRowCount(count: number | undefined) {
+    this.rowCount = count;
+  }
+
+  setPreviewText(text: string | undefined) {
+    this.previewText = text;
+  }
+
+  override initialize(): void {
+    this.setFileName(undefined);
+    this.setColumnNames(undefined);
+    this.setFileFormat(undefined);
+    this.setFileData(undefined);
+    this.setRowCount(undefined);
+    this.setPreviewText(undefined);
+  }
+
+  processFile(file: File | undefined) {
+    this.setFileName(undefined);
+    this.setColumnNames(undefined);
+    this.setFileFormat(undefined);
+    this.setFileData(undefined);
+    this.setRowCount(undefined);
+    this.setPreviewText(undefined);
+
+    if (!file) {
+      return;
+    }
+
+    this.processState.inProgress();
+
+    const fileName = file.name;
+    const fileFormat = fileName.split('.').pop();
+
+    switch (fileFormat?.toLowerCase()) {
+      case LocalFileDataCubeSourceFormat.CSV.toLowerCase(): {
+        parseCSVFile(file, {
+          complete: (result) => {
+            this.setFileData(
+              csvStringify(result.data, { escapeChar: `'`, quoteChar: `'` }),
+            );
+            this.setColumnNames(
+              Object.keys(result.data.at(0) as object).filter(
+                (key) => key !== '',
+              ),
+            );
+            this.setFileName(fileName);
+            this.setFileFormat(LocalFileDataCubeSourceFormat.CSV);
+            this.setRowCount(result.data.length);
+            this.setPreviewText(
+              csvStringify(result.data.slice(0, 100), {
+                escapeChar: `'`,
+                quoteChar: `'`,
+              }),
+            );
+          },
+          header: true,
+          dynamicTyping: false,
+          skipEmptyLines: true,
+        });
+        break;
+      }
+      default: {
+        this.processState.complete();
+        throw new IllegalStateError(
+          `Can't process file with format '${fileFormat}'`,
+        );
+      }
+    }
+
+    this.processState.complete();
+  }
+
+  override get isValid(): boolean {
+    return Boolean(this.fileData);
+  }
+
+  override get label(): LegendDataCubeSourceLoaderType {
+    return LegendDataCubeSourceLoaderType.LOCAL_FILE;
+  }
+
+  override async load(source: PlainObject | undefined): Promise<PlainObject> {
+    const deserializedSource =
+      RawLocalFileQueryDataCubeSource.serialization.fromJson(
+        guaranteeNonNullable(source),
+      );
+
+    if (
+      !this.fileData ||
+      !this.fileName ||
+      !this.fileFormat ||
+      this.rowCount === undefined
+    ) {
+      throw new IllegalStateError(
+        `Can't load source data: file data and information is not set`,
+      );
+    }
+
+    const intersectingColumns = guaranteeNonNullable(
+      this.columnNames?.filter((col) =>
+        deserializedSource.columnNames.includes(col),
+      ),
+    );
+    if (intersectingColumns.length !== deserializedSource.columnNames.length) {
+      throw new Error(
+        `Columns mismatch: Expected [${deserializedSource.columnNames.join(',')}], got [${this.columnNames?.join(',')}]`,
+      );
+    }
+
+    const tableDetails = guaranteeNonNullable(
+      await this._engine.ingestLocalFileData(
+        this.fileData,
+        this.fileFormat,
+        deserializedSource._ref,
+      ),
+      `Can't load source data: failed to ingest data from local file`,
+    );
+
+    // TODO: do a type check for columns
+    const rawSource = new RawLocalFileQueryDataCubeSource();
+    rawSource.fileName = this.fileName;
+    rawSource.fileFormat = this.fileFormat;
+    rawSource._ref = tableDetails.dbReference;
+    rawSource.columnNames = tableDetails.columnNames;
+
+    return RawLocalFileQueryDataCubeSource.serialization.toJson(rawSource);
+  }
+}

package/src/stores/model/LegendQueryDataCubeSource.ts

@@ -19,6 +19,7 @@ import type {
   QueryInfo,
   V1_Lambda,
   V1_ParameterValue,
+  V1_ValueSpecification,
 } from '@finos/legend-graph';
 import {
   SerializationFactory,
@@ -36,6 +37,7 @@ export class LegendQueryDataCubeSource extends DataCubeSource {
   runtime!: string;
   model!: PlainObject;
   parameterValues: V1_ParameterValue[] = [];
+  letParameterValueSpec: V1_ValueSpecification[] = [];
 }
 
 export class RawLegendQueryDataCubeSource {

package/src/stores/model/LocalFileDataCubeSource.ts

@@ -15,13 +15,13 @@
  */
 
 import { DataCubeSource } from '@finos/legend-data-cube';
-import { type V1_PureModelContextData } from '@finos/legend-graph';
+import type { V1_PureModelContextData } from '@finos/legend-graph';
 import {
   SerializationFactory,
   usingConstantValueSchema,
   type PlainObject,
 } from '@finos/legend-shared';
-import { createModelSchema, primitive, raw } from 'serializr';
+import { createModelSchema, list, primitive } from 'serializr';
 
 export const LOCAL_FILE_QUERY_DATA_CUBE_SOURCE_TYPE = 'localFile';
 
@@ -36,32 +36,23 @@ export class LocalFileDataCubeSource extends DataCubeSource {
   db!: string;
   schema!: string;
   table!: string;
-  count!: number;
   fileName!: string;
   fileFormat!: LocalFileDataCubeSourceFormat;
 }
 
 export class RawLocalFileQueryDataCubeSource {
-  model!: PlainObject<V1_PureModelContextData>;
-  runtime!: string;
-  db!: string;
-  schema!: string;
-  table!: string;
-  count!: number;
   fileName!: string;
   fileFormat!: LocalFileDataCubeSourceFormat;
+  _ref!: string;
+  columnNames!: string[];
 
   static readonly serialization = new SerializationFactory(
     createModelSchema(RawLocalFileQueryDataCubeSource, {
       _type: usingConstantValueSchema(LOCAL_FILE_QUERY_DATA_CUBE_SOURCE_TYPE),
-      count: primitive(),
-      db: primitive(),
       fileFormat: primitive(),
       fileName: primitive(),
-      model: raw(),
-      runtime: primitive(),
-      schema: primitive(),
-      table: primitive(),
+      _ref: primitive(),
+      columnNames: list(primitive()),
     }),
   );
 }

package/tsconfig.json

@@ -60,6 +60,7 @@
     "./src/application/LegendDataCubeApplicationConfig.ts",
     "./src/application/LegendDataCubeApplicationPlugin.ts",
     "./src/application/LegendDataCubePluginManager.ts",
+    "./src/application/__test-utils__/LegendDataCubeApplicationTestUtils.ts",
     "./src/stores/DuckDBWASM.d.ts",
     "./src/stores/LegendDataCubeBaseStore.ts",
     "./src/stores/LegendDataCubeDataCubeEngine.ts",
@@ -69,26 +70,32 @@
     "./src/stores/builder/source/LegendQueryDataCubeSourceBuilderState.ts",
     "./src/stores/builder/source/LocalFileDataCubeSourceBuilderState.ts",
     "./src/stores/builder/source/UserDefinedFunctionDataCubeSourceBuilderState.ts",
+    "./src/stores/builder/source/loader/LegendDataCubePartialSourceLoaderState.ts",
+    "./src/stores/builder/source/loader/LocalFileDataCubePartialSourceLoaderState.ts",
     "./src/stores/model/LegendQueryDataCubeSource.ts",
     "./src/stores/model/LocalFileDataCubeSource.ts",
     "./src/application/LegendDataCube.tsx",
     "./src/components/LegendDataCubeBlockingWindow.tsx",
     "./src/components/LegendDataCubeFrameworkProvider.tsx",
     "./src/components/LegendDataCubeWebApplication.tsx",
+    "./src/components/__test-utils__/LegendDataCubeStoreTestUtils.tsx",
     "./src/components/builder/LegendDataCubeBuilder.tsx",
     "./src/components/builder/LegendDataCubeBuilderStoreProvider.tsx",
     "./src/components/builder/LegendDataCubeCreator.tsx",
     "./src/components/builder/LegendDataCubeDeleteConfirmation.tsx",
     "./src/components/builder/LegendDataCubeLoader.tsx",
+    "./src/components/builder/LegendDataCubePartialSourceLoader.tsx",
     "./src/components/builder/LegendDataCubeSaver.tsx",
     "./src/components/builder/LegendDataCubeSourceViewer.tsx",
     "./src/components/builder/source/AdhocQueryDataCubeSourceBuilder.tsx",
     "./src/components/builder/source/LegendQueryDataCubeSourceBuilder.tsx",
     "./src/components/builder/source/LocalFileDataCubeSourceBuilder.tsx",
     "./src/components/builder/source/UserDefinedFunctionDataCubeSourceBuilder.tsx",
+    "./src/components/builder/source/loader/LocalFileDataCubePartialSourceLoader.tsx",
     "./src/stores/builder/LegendDataCubeBuilderStore.tsx",
     "./src/stores/builder/LegendDataCubeCreatorState.tsx",
-    "./src/stores/builder/LegendDataCubeLoaderState.tsx"
+    "./src/stores/builder/LegendDataCubeLoaderState.tsx",
+    "./src/stores/builder/LegendDataCubeSourceLoaderState.tsx"
   ],
   "include": [
     "src/**/*.ts",