@finos/legend-application-studio 28.21.23 → 28.21.25

package/src/stores/editor/editor-state/element-editor-state/testable/LakehouseTestableUtils.ts

@@ -0,0 +1,224 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Shared utilities for Lakehouse test state:
+ * used by both DataProductTestableState and IngestTestableState.
+ */
+
+import {
+  type Accessor,
+  type AccessorOwner,
+  type AbstractPureGraphManager,
+  type BaseDataResolver,
+  DataProduct,
+  IngestDefinition,
+  RelationElement,
+  RelationElementsData,
+  RelationRowTestData,
+  getAccessorItemLabelForElement,
+  observe_RelationElement,
+  observe_RelationRowTestData,
+  type PackageableElement,
+} from '@finos/legend-graph';
+import { noop } from '@finos/legend-shared';
+import { runInAction } from 'mobx';
+import type { EditorStore } from '../../../EditorStore.js';
+import { RelationElementsDataState } from '../data/EmbeddedDataState.js';
+
+// ─── Pure helpers ─────────────────────────────────────────────────────────────
+
+export const createEmptyRelationElement = (
+  itemId: string,
+  columns: string[] = [],
+): RelationElement => {
+  const row = observe_RelationRowTestData(new RelationRowTestData());
+  row.values = columns.map(() => '');
+  const relationElement = new RelationElement();
+  relationElement.paths = [itemId];
+  relationElement.columns = columns;
+  relationElement.rows = [row];
+  return observe_RelationElement(relationElement);
+};
+
+export const buildRelationElementsDataWithColumns = (
+  accessors: Accessor[],
+): RelationElementsData => {
+  const relData = new RelationElementsData();
+  relData.relationElements = accessors.map((acc) => {
+    const itemId = acc.accessor || 'UNKNOWN';
+    const columns = acc.relationType.columns.map((col) => col.name);
+    return createEmptyRelationElement(itemId, columns);
+  });
+  return relData;
+};
+
+export const isIngestOrDataProductAccessor = (
+  accessor: Accessor,
+): accessor is Accessor =>
+  accessor.parentElement instanceof DataProduct ||
+  accessor.parentElement instanceof IngestDefinition;
+
+export interface ElementDataItem {
+  id: string;
+  label: string;
+}
+
+export const getElementDataItems = (
+  element: PackageableElement,
+  graphManager: AbstractPureGraphManager,
+): ElementDataItem[] => {
+  if (element instanceof DataProduct) {
+    return element.accessPointGroups
+      .flatMap((g) => g.accessPoints)
+      .map((ap) => ({ id: ap.id, label: ap.id }));
+  }
+  if (element instanceof IngestDefinition) {
+    return graphManager
+      .getIngestDefinitionDatasetNames(element)
+      .map((name) => ({ id: name, label: name }));
+  }
+  return [];
+};
+
+// ─── Shared element test data state ──────────────────────────────────────────
+
+/**
+ * Wraps a single BaseDataResolver within a test suite's testData array.
+ * Shared between DataProduct and Ingest test suite states — both populate
+ * test data by wiring BaseDataResolver entries to PackageableElements.
+ */
+export class LakehouseElementTestDataState {
+  readonly testData: BaseDataResolver;
+  readonly editorStore: EditorStore;
+  readonly relationElementsDataState: RelationElementsDataState | undefined;
+
+  constructor(testData: BaseDataResolver, editorStore: EditorStore) {
+    this.testData = testData;
+    this.editorStore = editorStore;
+    if (testData.data instanceof RelationElementsData) {
+      this.relationElementsDataState = new RelationElementsDataState(
+        editorStore,
+        testData.data,
+      );
+      this.initAccessorOptions();
+    }
+  }
+
+  get element(): PackageableElement {
+    return this.testData.element.value;
+  }
+
+  get itemLabel(): string {
+    return getAccessorItemLabelForElement(this.element as AccessorOwner);
+  }
+
+  private initAccessorOptions(): void {
+    const dataState = this.relationElementsDataState;
+    if (!dataState) {
+      return;
+    }
+    this.refreshAccessorOptions(dataState).catch(noop);
+    dataState.setRefreshAccessorOptions(() =>
+      this.refreshAccessorOptions(dataState),
+    );
+  }
+
+  private async refreshAccessorOptions(
+    dataState: RelationElementsDataState,
+  ): Promise<void> {
+    const element = this.element;
+    const graphManager = this.editorStore.graphManagerState.graphManager;
+    const graph = this.editorStore.graphManagerState.graph;
+    const items = getElementDataItems(element, graphManager);
+
+    if (items.length === 0) {
+      dataState.setAccessorOptions(undefined, undefined);
+      return;
+    }
+
+    const options = await Promise.all(
+      items.map(async (item) => {
+        let columns: string[] = [];
+        try {
+          if (element instanceof IngestDefinition) {
+            const accessor =
+              await graphManager.createAccessorFromPackageableElement(
+                element,
+                graph,
+                { schemaName: undefined, tableName: item.id },
+              );
+            if (accessor) {
+              columns = accessor.relationType.columns.map((c) => c.name);
+            }
+          } else if (element instanceof DataProduct) {
+            const accessor = await graphManager.buildDataProductAccessor(
+              element,
+              graph,
+              { tableName: item.id },
+            );
+            if (accessor) {
+              columns = accessor.relationType.columns.map((c) => c.name);
+            }
+          }
+        } catch {
+          // best-effort column resolution
+        }
+        return { label: item.label, value: item.id, columns };
+      }),
+    );
+
+    runInAction(() => {
+      dataState.setAccessorOptions(options, this.itemLabel);
+      const columnsByItem = new Map(
+        options
+          .filter((o) => o.columns.length > 0)
+          .map((o) => [o.value, o.columns]),
+      );
+      for (const relState of dataState.relationElementStates) {
+        const rel = relState.relationElement;
+        if (rel.columns.length === 0) {
+          const key = rel.paths[rel.paths.length - 1];
+          const cols = key ? columnsByItem.get(key) : undefined;
+          if (cols) {
+            rel.columns = cols;
+          }
+        }
+      }
+    });
+  }
+}
+
+// ─── Shared interface for TestDataState ───────────────────────────────────────
+
+/**
+ * Structural interface satisfied by both DataProductTestDataState and
+ * IngestTestDataState. Used to type the shared LakehouseTestDataEditor
+ * component without importing the concrete state classes.
+ */
+export interface LakehouseTestDataStateBase {
+  elementTestDataStates: LakehouseElementTestDataState[];
+  selectedElementTestDataState: LakehouseElementTestDataState | undefined;
+  setSelectedElementTestDataState(
+    val: LakehouseElementTestDataState | undefined,
+  ): void;
+  availableElementsToAdd: PackageableElement[];
+  showAddElementModal: boolean;
+  setShowAddElementModal(val: boolean): void;
+  addElement(path: string): void;
+  deleteElement(state: LakehouseElementTestDataState): void;
+  editorStore: EditorStore;
+}

package/src/stores/workspace-setup/WorkspaceSetupStore.ts

@@ -27,7 +27,9 @@ import {
   guaranteeNonNullable,
   guaranteeType,
   exactSearch,
+  returnUndefOnError,
 } from '@finos/legend-shared';
+import { parseProjectIdentifier } from '@finos/legend-storage';
 import { generateSetupRoute } from '../../__lib__/LegendStudioNavigation.js';
 import {
   type SDLCServerClient,
@@ -43,7 +45,6 @@ import {
 import type { LegendStudioApplicationStore } from '../LegendStudioBaseStore.js';
 import {
   DEFAULT_TAB_SIZE,
-  DEFAULT_TYPEAHEAD_SEARCH_LIMIT,
   DEFAULT_TYPEAHEAD_SEARCH_MINIMUM_SEARCH_LENGTH,
 } from '@finos/legend-application';
 import {
@@ -63,13 +64,33 @@ interface ImportProjectSuccessReport {
   reviewUrl: string;
 }
 
+export interface ProjectSearchResult {
+  projects: Project[];
+  projectMatchedById?: Project | undefined;
+}
+
+const PROD_PROJECT_ID_PREFIX = 'PROD';
+const cleanProjectIdInput = (searchText: string): string | undefined => {
+  const parsed = returnUndefOnError(() =>
+    parseProjectIdentifier(searchText.trim()),
+  );
+  if (parsed === undefined) {
+    return undefined;
+  }
+  return `${parsed.prefix ?? PROD_PROJECT_ID_PREFIX}-${parsed.id}`;
+};
+
+export const PROJECT_SEARCH_FETCH_LIMIT = 30;
+
 export class WorkspaceSetupStore {
   readonly applicationStore: LegendStudioApplicationStore;
 
   readonly sdlcServerClient: SDLCServerClient;
   readonly initState = ActionState.create();
 
-  projects: Project[] = [];
+  projectSearchResult: ProjectSearchResult = { projects: [] };
+  private projectSearchSyncCounter = 0;
+  private searchCache = new Map<string, ProjectSearchResult>();
   currentProject?: Project | undefined;
   currentProjectConfigurationStatus?: ProjectConfigurationStatus | undefined;
   loadProjectsState = ActionState.create();
@@ -107,7 +128,7 @@ export class WorkspaceSetupStore {
     sdlcServerClient: SDLCServerClient,
   ) {
     makeObservable(this, {
-      projects: observable,
+      projectSearchResult: observable,
       currentProject: observable,
       currentProjectConfigurationStatus: observable,
       importProjectSuccessReport: observable,
@@ -134,6 +155,7 @@ export class WorkspaceSetupStore {
       removeRecentProject: action,
       removeRecentWorkspace: action,
       clearRecents: action,
+      filterListOnInput: action,
       initialize: flow,
       loadProjects: flow,
       loadSandboxProject: flow,
@@ -432,13 +454,82 @@ export class WorkspaceSetupStore {
     }
   }
 
+  private findPrefixCache(searchText: string): ProjectSearchResult | undefined {
+    let bestKey: string | undefined;
+    let bestEntry: ProjectSearchResult | undefined;
+    for (const [key, entry] of this.searchCache) {
+      const isStrictPrefix =
+        key.length < searchText.length && searchText.startsWith(key);
+      if (!isStrictPrefix) {
+        continue;
+      }
+      if (bestKey === undefined || key.length > bestKey.length) {
+        bestKey = key;
+        bestEntry = entry;
+      }
+    }
+    if (!bestEntry) {
+      return undefined;
+    }
+
+    const normalizedSearch = searchText.toLowerCase();
+    const projectMatchesSearch = (project: Project): boolean =>
+      project.name.toLowerCase().includes(normalizedSearch);
+    return {
+      projects: bestEntry.projects.filter(projectMatchesSearch),
+      projectMatchedById:
+        bestEntry.projectMatchedById &&
+        projectMatchesSearch(bestEntry.projectMatchedById)
+          ? bestEntry.projectMatchedById
+          : undefined,
+    };
+  }
+
+  filterListOnInput(searchText: string): void {
+    if (searchText.length === 0) {
+      return;
+    }
+    const exact = this.searchCache.get(searchText);
+    if (exact) {
+      this.projectSearchResult = exact;
+      return;
+    }
+    const narrowed = this.findPrefixCache(searchText);
+    if (narrowed) {
+      this.projectSearchResult = narrowed;
+    }
+  }
+
   *loadProjects(searchText: string): GeneratorFn<void> {
     const isValidSearchString =
       searchText.length >= DEFAULT_TYPEAHEAD_SEARCH_MINIMUM_SEARCH_LENGTH;
+    const searchSyncCounter = ++this.projectSearchSyncCounter;
+
+    const cached = this.searchCache.get(searchText);
+    if (cached) {
+      this.projectSearchResult = cached;
+      this.loadProjectsState.pass();
+      return;
+    }
+
+    this.projectSearchResult = { projects: this.projectSearchResult.projects };
     this.loadProjectsState.inProgress();
+
+    if (searchText.length > 0) {
+      const narrowed = this.findPrefixCache(searchText);
+      if (narrowed) {
+        this.projectSearchResult = narrowed;
+      }
+      if (searchText.length <= DEFAULT_TYPEAHEAD_SEARCH_MINIMUM_SEARCH_LENGTH) {
+        this.loadProjectsState.pass();
+        return;
+      }
+    }
+
     try {
-      this.projects = (
-        (yield this.sdlcServerClient.getProjects(
+      const cleanedProjectId = cleanProjectIdInput(searchText);
+      const [searchResults, projectByIdResult] = (yield Promise.all([
+        this.sdlcServerClient.getProjects(
           undefined,
           // We apply an exact search on the input text because we show exact searches with
           // custom selector. This avoids losing some results with the additional filtering.
@@ -448,11 +539,33 @@ export class WorkspaceSetupStore {
           // `loadSandboxProject`); exclude them here so they do not appear
           // in the main workspace setup project picker.
           [SANDBOX_SDLC_TAG],
-          DEFAULT_TYPEAHEAD_SEARCH_LIMIT,
-        )) as PlainObject<Project>[]
-      ).map((v) => Project.serialization.fromJson(v));
+          PROJECT_SEARCH_FETCH_LIMIT,
+        ),
+        cleanedProjectId
+          ? this.sdlcServerClient
+              .getProject(cleanedProjectId)
+              .catch(() => undefined)
+          : Promise.resolve(undefined),
+      ])) as [PlainObject<Project>[], PlainObject<Project> | undefined];
+
+      if (searchSyncCounter !== this.projectSearchSyncCounter) {
+        return;
+      }
+
+      const projects = searchResults.map((v) =>
+        Project.serialization.fromJson(v),
+      );
+      const projectMatchedById = projectByIdResult
+        ? Project.serialization.fromJson(projectByIdResult)
+        : undefined;
+
+      this.projectSearchResult = { projects, projectMatchedById };
+      this.searchCache.set(searchText, this.projectSearchResult);
       this.loadProjectsState.pass();
     } catch (error) {
+      if (searchSyncCounter !== this.projectSearchSyncCounter) {
+        return;
+      }
       assertErrorThrown(error);
       this.applicationStore.notificationService.notifyError(error);
       this.loadProjectsState.fail();
@@ -742,6 +855,8 @@ export class WorkspaceSetupStore {
         `Project '${name}' is succesfully created`,
       );
 
+      this.searchCache.clear();
+
       yield flowResult(this.changeProject(createdProject));
 
       this.setShowCreateProjectModal(false);
@@ -780,6 +895,8 @@ export class WorkspaceSetupStore {
         reviewUrl: importReview.webURL,
       });
 
+      this.searchCache.clear();
+
       yield flowResult(this.changeProject(report.project));
     } catch (error) {
       assertErrorThrown(error);

package/tsconfig.json

@@ -144,6 +144,7 @@
     "./src/stores/editor/editor-state/element-editor-state/function-activator/SnowflakeM2MUdfFunctionActivatorEditorState.ts",
     "./src/stores/editor/editor-state/element-editor-state/function-activator/testable/FunctionTestableState.ts",
     "./src/stores/editor/editor-state/element-editor-state/ingest/IngestDefinitionEditorState.ts",
+    "./src/stores/editor/editor-state/element-editor-state/ingest/IngestTestableState.ts",
     "./src/stores/editor/editor-state/element-editor-state/mapping/FlatDataInstanceSetImplementationState.ts",
     "./src/stores/editor/editor-state/element-editor-state/mapping/MappingDiagramGenerator.ts",
     "./src/stores/editor/editor-state/element-editor-state/mapping/MappingEditorState.ts",
@@ -168,6 +169,7 @@
     "./src/stores/editor/editor-state/element-editor-state/service/testable/ServiceTestDataState.ts",
     "./src/stores/editor/editor-state/element-editor-state/service/testable/ServiceTestEditorState.ts",
     "./src/stores/editor/editor-state/element-editor-state/service/testable/ServiceTestableState.ts",
+    "./src/stores/editor/editor-state/element-editor-state/testable/LakehouseTestableUtils.ts",
     "./src/stores/editor/editor-state/element-editor-state/testable/TestAssertionState.ts",
     "./src/stores/editor/editor-state/element-editor-state/testable/TestableEditorState.ts",
     "./src/stores/editor/editor-state/end-to-end-workflow-state/EndToEndWorkflowEditorState.ts",
@@ -296,6 +298,7 @@
     "./src/components/editor/editor-group/function-activator/SnowflakeM2MUdfFunctionActivatorEditor.tsx",
     "./src/components/editor/editor-group/function-activator/testable/FunctionTestableEditor.tsx",
     "./src/components/editor/editor-group/ingest-editor/IngestDefinitionEditor.tsx",
+    "./src/components/editor/editor-group/ingest-editor/testable/IngestTestableEditor.tsx",
     "./src/components/editor/editor-group/mapping-editor/ClassMappingEditor.tsx",
     "./src/components/editor/editor-group/mapping-editor/EnumerationMappingEditor.tsx",
     "./src/components/editor/editor-group/mapping-editor/FlatDataPropertyMappingEditor.tsx",
@@ -331,6 +334,7 @@
     "./src/components/editor/editor-group/service-editor/testable/ServiceTestDataEditor.tsx",
     "./src/components/editor/editor-group/service-editor/testable/ServiceTestableEditor.tsx",
     "./src/components/editor/editor-group/service-editor/testable/ServiceTestsEditor.tsx",
+    "./src/components/editor/editor-group/testable/LakehouseTestableEditor.tsx",
     "./src/components/editor/editor-group/testable/TestableSharedComponents.tsx",
     "./src/components/editor/editor-group/uml-editor/AssociationEditor.tsx",
     "./src/components/editor/editor-group/uml-editor/ClassEditor.tsx",