@finos/legend-application-query 13.8.28 → 13.8.30

package/src/components/ingest/IngestQueryBuilderSetupPanel.tsx

@@ -0,0 +1,204 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { useApplicationStore } from '@finos/legend-application';
+import {
+  AnchorLinkIcon,
+  CogIcon,
+  compareLabelFn,
+  ControlledDropdownMenu,
+  createFilter,
+  CustomSelectorInput,
+  MenuContent,
+  MenuContentItem,
+  MenuContentItemIcon,
+  MenuContentItemLabel,
+  MoreVerticalIcon,
+  PanelHeader,
+  PanelHeaderActionItem,
+  PanelHeaderActions,
+} from '@finos/legend-art';
+import { observer } from 'mobx-react-lite';
+import { useState } from 'react';
+import type { IngestLegendQueryBuilderState } from '../../stores/ingest/IngestLegendQueryBuilderState.js';
+import { LakehouseRuntimeConfigModal } from '../shared/LakehouseRuntimeConfigModal.js';
+
+type DataSetOption = {
+  label: string;
+  value: { schemaName?: string | undefined; tableName: string };
+};
+
+type IngestOption = {
+  label: string;
+  value: string;
+};
+
+/**
+ * Setup panel for ingest-backed queries. Mirrors the data-product setup
+ * panel's header actions (copy deep-link, edit lakehouse runtime) but
+ * skips the source / runtime selectors since the ingest flow is fully
+ * resolved from the route + adhoc runtime. The data set selector is kept
+ * so the user can switch between datasets of the same ingest definition.
+ */
+const IngestQueryBuilderSetupPanelContent = observer(
+  (props: { queryBuilderState: IngestLegendQueryBuilderState }) => {
+    const { queryBuilderState } = props;
+    const applicationStore = useApplicationStore();
+    const [isLakehouseConfigModalOpen, setIsLakehouseConfigModalOpen] =
+      useState(false);
+    const lakehouseRuntime = queryBuilderState.lakehouseRuntime;
+    const showLakehouseConfigButton = Boolean(lakehouseRuntime);
+    const darkMode =
+      !applicationStore.layoutService.TEMPORARY__isLightColorThemeEnabled;
+
+    // data set selector (driven by the ingest definition's datasets via
+    // `AccessorQueryBuilderState.accessorsOptions`).
+    const dataSetOptions =
+      queryBuilderState.accessorsOptions.sort(compareLabelFn);
+    const selectedDataSetOption = queryBuilderState.sourceAccessor
+      ? (dataSetOptions.find(
+          (opt) =>
+            opt.value.tableName ===
+              queryBuilderState.sourceAccessor?.accessor &&
+            opt.value.schemaName === queryBuilderState.sourceAccessor.schema,
+        ) ?? null)
+      : null;
+    const changeDataSet = async (val: DataSetOption): Promise<void> => {
+      await queryBuilderState.changeAccessor(val.value);
+    };
+
+    // ingest selector (paths fetched up front; only one ingest is built into
+    // the graph at a time — see `IngestQueryCreatorStore.swapIngestDefinition`).
+    const ingestOptions: IngestOption[] = queryBuilderState.allIngestPaths
+      .map((path) => ({ label: path, value: path }))
+      .sort(compareLabelFn);
+    const selectedIngestOption: IngestOption | null =
+      ingestOptions.find(
+        (opt) => opt.value === queryBuilderState.ingestDefinition.path,
+      ) ?? null;
+    const changeIngest = async (val: IngestOption): Promise<void> => {
+      await queryBuilderState.changeIngestDefinition(val.value);
+    };
+    const ingestFilterOption = createFilter({
+      ignoreCase: true,
+      ignoreAccents: false,
+      stringify: (option: { data: IngestOption }): string => option.data.value,
+    });
+
+    return (
+      <div className="query-builder__setup__config-group">
+        <PanelHeader title="properties">
+          <PanelHeaderActions>
+            <PanelHeaderActionItem
+              title="copy ingest query set up link to clipboard"
+              onClick={() => queryBuilderState.copyIngestQueryLinkToClipBoard()}
+            >
+              <AnchorLinkIcon />
+            </PanelHeaderActionItem>
+            {showLakehouseConfigButton && (
+              <ControlledDropdownMenu
+                className="panel__header__action query-builder__setup__config-group__header__dropdown-trigger"
+                title="Show Settings..."
+                content={
+                  <MenuContent>
+                    <MenuContentItem
+                      onClick={() => setIsLakehouseConfigModalOpen(true)}
+                    >
+                      <MenuContentItemIcon>
+                        <CogIcon />
+                      </MenuContentItemIcon>
+                      <MenuContentItemLabel>
+                        Lakehouse Runtime Configuration
+                      </MenuContentItemLabel>
+                    </MenuContentItem>
+                  </MenuContent>
+                }
+                menuProps={{
+                  anchorOrigin: {
+                    vertical: 'bottom',
+                    horizontal: 'right',
+                  },
+                  transformOrigin: {
+                    vertical: 'top',
+                    horizontal: 'right',
+                  },
+                }}
+              >
+                <MoreVerticalIcon className="query-builder__icon__more-options" />
+              </ControlledDropdownMenu>
+            )}
+          </PanelHeaderActions>
+        </PanelHeader>
+        <div className="query-builder__setup__config-group__content">
+          <div className="query-builder__setup__config-group__item">
+            <label
+              className="btn--sm query-builder__setup__config-group__item__label"
+              title="ingest definition"
+              htmlFor="query-builder__setup__ingest-selector"
+            >
+              Ingest
+            </label>
+            <CustomSelectorInput
+              inputId="query-builder__setup__ingest-selector"
+              className="panel__content__form__section__dropdown query-builder__setup__config-group__item__selector"
+              placeholder="Choose an ingest..."
+              options={ingestOptions}
+              onChange={(val: IngestOption): void => {
+                changeIngest(val).catch(() => undefined);
+              }}
+              value={selectedIngestOption}
+              darkMode={darkMode}
+              disabled={queryBuilderState.isSwappingIngest}
+              filterOption={ingestFilterOption}
+            />
+          </div>
+          <div className="query-builder__setup__config-group__item">
+            <label
+              className="btn--sm query-builder__setup__config-group__item__label"
+              title="data set"
+              htmlFor="query-builder__setup__ingest-dataset-selector"
+            >
+              Data Set
+            </label>
+            <CustomSelectorInput
+              inputId="query-builder__setup__ingest-dataset-selector"
+              className="panel__content__form__section__dropdown query-builder__setup__config-group__item__selector"
+              placeholder="Choose a data set..."
+              options={dataSetOptions}
+              onChange={(val: DataSetOption): void => {
+                changeDataSet(val).catch(() => undefined);
+              }}
+              value={selectedDataSetOption}
+              darkMode={darkMode}
+            />
+          </div>
+        </div>
+        <LakehouseRuntimeConfigModal
+          lakehouseRuntime={lakehouseRuntime}
+          open={isLakehouseConfigModalOpen}
+          onClose={() => setIsLakehouseConfigModalOpen(false)}
+          darkMode={darkMode}
+        />
+      </div>
+    );
+  },
+);
+
+export const renderIngestQueryBuilderSetupPanelContent = (
+  queryBuilderState: IngestLegendQueryBuilderState,
+): React.ReactNode => (
+  <IngestQueryBuilderSetupPanelContent queryBuilderState={queryBuilderState} />
+);

package/src/components/ingest/IngestQueryCreator.tsx

@@ -0,0 +1,75 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { observer, useLocalObservable } from 'mobx-react-lite';
+import { parseGAVCoordinates } from '@finos/legend-storage';
+import { guaranteeNonNullable } from '@finos/legend-shared';
+import {
+  useLegendQueryApplicationStore,
+  useLegendQueryBaseStore,
+} from '../LegendQueryFrameworkProvider.js';
+import {
+  INGEST_QUERY_CREATOR_ROUTE_PATTERN_TOKEN,
+  type IngestQueryCreatorPathParams,
+} from '../../__lib__/LegendQueryNavigation.js';
+import { IngestQueryCreatorStore } from '../../stores/ingest/IngestQueryCreatorStore.js';
+import { QueryEditorStoreContext } from '../QueryEditorStoreProvider.js';
+import { QueryEditor } from '../QueryEditor.js';
+import { useParams } from '@finos/legend-application/browser';
+
+/**
+ * Route handler for `LEGEND_QUERY_ROUTE_PATTERN.INGEST_QUERY`.
+ *
+ * Parses `gav`, `ingestDefinitionPath` and `dataSet` from the URL, builds an
+ * {@link IngestQueryCreatorStore}, and renders the shared {@link QueryEditor}
+ * against it. The store fetches only the ingest definition entity from Depot
+ * — no full graph build.
+ */
+export const IngestQueryCreator = observer(() => {
+  const applicationStore = useLegendQueryApplicationStore();
+  const baseStore = useLegendQueryBaseStore();
+  const params = useParams<IngestQueryCreatorPathParams>();
+
+  const gav = guaranteeNonNullable(
+    params[INGEST_QUERY_CREATOR_ROUTE_PATTERN_TOKEN.GAV],
+  );
+  const ingestDefinitionPath = guaranteeNonNullable(
+    params[INGEST_QUERY_CREATOR_ROUTE_PATTERN_TOKEN.INGEST_DEFINITION_PATH],
+  );
+  const dataSet = guaranteeNonNullable(
+    params[INGEST_QUERY_CREATOR_ROUTE_PATTERN_TOKEN.DATA_SET],
+  );
+  const { groupId, artifactId, versionId } = parseGAVCoordinates(gav);
+
+  const store = useLocalObservable(
+    () =>
+      new IngestQueryCreatorStore(
+        applicationStore,
+        baseStore.depotServerClient,
+        groupId,
+        artifactId,
+        versionId,
+        ingestDefinitionPath,
+        dataSet,
+      ),
+  );
+
+  return (
+    <QueryEditorStoreContext.Provider value={store}>
+      <QueryEditor />
+    </QueryEditorStoreContext.Provider>
+  );
+});

package/src/components/shared/LakehouseRuntimeConfigModal.tsx

@@ -0,0 +1,148 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { useApplicationStore } from '@finos/legend-application';
+import {
+  CogIcon,
+  Dialog,
+  Modal,
+  ModalBody,
+  ModalFooter,
+  ModalFooterButton,
+  ModalHeader,
+  ModalHeaderActions,
+  ModalTitle,
+  TimesIcon,
+} from '@finos/legend-art';
+import type { LakehouseRuntime } from '@finos/legend-graph';
+import { observer } from 'mobx-react-lite';
+import { useEffect, useState } from 'react';
+import { LegendQueryUserDataHelper } from '../../__lib__/LegendQueryUserDataHelper.js';
+
+/**
+ * Modal for editing a {@link LakehouseRuntime}'s environment and warehouse.
+ * Reused by both the data-product and ingest query builder setup panels.
+ */
+export const LakehouseRuntimeConfigModal = observer(
+  (props: {
+    lakehouseRuntime: LakehouseRuntime | undefined;
+    open: boolean;
+    onClose: () => void;
+    darkMode: boolean;
+  }) => {
+    const { lakehouseRuntime, open, onClose, darkMode } = props;
+    const [env, setEnv] = useState(lakehouseRuntime?.environment ?? '');
+    const [warehouse, setWarehouse] = useState(
+      lakehouseRuntime?.warehouse ?? '',
+    );
+
+    // sync local state when the modal opens or the runtime changes
+    useEffect(() => {
+      if (open && lakehouseRuntime) {
+        setEnv(lakehouseRuntime.environment ?? '');
+        setWarehouse(lakehouseRuntime.warehouse ?? '');
+      }
+    }, [open, lakehouseRuntime]);
+
+    const applicationStore = useApplicationStore();
+
+    const handleApply = (): void => {
+      if (lakehouseRuntime) {
+        const newEnv = env || undefined;
+        const newWarehouse = warehouse || undefined;
+        const hasChanged =
+          newEnv !== lakehouseRuntime.environment ||
+          newWarehouse !== lakehouseRuntime.warehouse;
+        lakehouseRuntime.environment = newEnv;
+        lakehouseRuntime.warehouse = newWarehouse;
+        if (hasChanged) {
+          LegendQueryUserDataHelper.persistLakehouseUserInfo(
+            applicationStore.userDataService,
+            {
+              env: newEnv,
+              snowflakeWarehouse: newWarehouse,
+            },
+          );
+        }
+      }
+      onClose();
+    };
+
+    if (!lakehouseRuntime) {
+      return null;
+    }
+
+    return (
+      <Dialog onClose={onClose} open={open}>
+        <Modal darkMode={darkMode}>
+          <ModalHeader>
+            <ModalTitle
+              icon={<CogIcon />}
+              title="Lakehouse Runtime Configuration"
+            />
+            <ModalHeaderActions>
+              <button
+                className="modal__header__action"
+                tabIndex={-1}
+                onClick={onClose}
+              >
+                <TimesIcon />
+              </button>
+            </ModalHeaderActions>
+          </ModalHeader>
+          <ModalBody>
+            <div className="panel__content__form__section">
+              <div className="panel__content__form__section__header__label">
+                Environment
+              </div>
+              <input
+                className="panel__content__form__section__input input--dark input--small"
+                spellCheck={false}
+                value={env}
+                placeholder="(optional)"
+                onChange={(e) => setEnv(e.target.value)}
+              />
+            </div>
+            <div className="panel__content__form__section">
+              <div className="panel__content__form__section__header__label">
+                Warehouse
+              </div>
+              <input
+                className="panel__content__form__section__input input--dark input--small"
+                spellCheck={false}
+                value={warehouse}
+                placeholder="(optional)"
+                onChange={(e) => setWarehouse(e.target.value)}
+              />
+            </div>
+          </ModalBody>
+          <ModalFooter>
+            <ModalFooterButton
+              text="Apply"
+              onClick={handleApply}
+              type="primary"
+            />
+            <ModalFooterButton
+              text="Cancel"
+              onClick={onClose}
+              type="secondary"
+            />
+          </ModalFooter>
+        </Modal>
+      </Dialog>
+    );
+  },
+);

package/src/stores/QueryEditorStore.ts

@@ -76,8 +76,10 @@ import {
   QueryDataProductNativeExecutionContextInfo,
   QueryDataProductModelAccessExecutionContextInfo,
   QueryDataProductLakehouseExecutionContextInfo,
+  QueryIngestExecutionContextInfo,
   ModelAccessPointGroup,
   type DataProduct,
+  IngestDefinition,
   LakehouseRuntime,
   V1_DATA_PRODUCT_ELEMENT_PROTOCOL_TYPE,
   V1_DataProductArtifact,
@@ -152,6 +154,11 @@ import { generateDataSpaceQueryCreatorRoute } from '../__lib__/DSL_DataSpace_Leg
 import { hasDataSpaceInfoBeenVisited } from '../__lib__/LegendQueryUserDataSpaceHelper.js';
 import { LegendQueryDataSpaceQueryBuilderState } from './data-space/query-builder/LegendQueryDataSpaceQueryBuilderState.js';
 import { LegendQueryDataProductQueryBuilderState } from './data-product/query-builder/LegendQueryDataProductQueryBuilderState.js';
+import { IngestLegendQueryBuilderState } from './ingest/IngestLegendQueryBuilderState.js';
+import {
+  fetchIngestEntitiesByClassifier,
+  swapIngestInGraph,
+} from './ingest/IngestQueryGraphHelper.js';
 import { DataProductSelectorState } from './data-space/DataProductSelectorState.js';
 import {
   decorateEnvWithRealm,
@@ -317,6 +324,7 @@ export abstract class QueryEditorStore {
   showAppInfo = false;
   showDataspaceInfo = false;
   showDataProductInfo = false;
+  showIngestInfo = false;
   enableMinialGraphForDataSpaceLoadingPerformance = true;
 
   constructor(
@@ -331,6 +339,7 @@ export abstract class QueryEditorStore {
       showAppInfo: observable,
       showDataspaceInfo: observable,
       showDataProductInfo: observable,
+      showIngestInfo: observable,
       queryBuilderState: observable,
       enableMinialGraphForDataSpaceLoadingPerformance: observable,
       isPerformingBlockingAction: computed,
@@ -339,6 +348,7 @@ export abstract class QueryEditorStore {
       setShowAppInfo: action,
       setShowDataspaceInfo: action,
       setShowDataProductInfo: action,
+      setShowIngestInfo: action,
       setEnableMinialGraphForDataSpaceLoadingPerformance: action,
       initialize: flow,
       buildGraph: flow,
@@ -453,6 +463,10 @@ export abstract class QueryEditorStore {
     this.showDataProductInfo = val;
   }
 
+  setShowIngestInfo(val: boolean): void {
+    this.showIngestInfo = val;
+  }
+
   setShowRegisterServiceModal(val: boolean): void {
     this.showRegisterServiceModal = val;
   }
@@ -1926,13 +1940,64 @@ export class ExistingQueryEditorStore extends QueryEditorStore {
         queryInfo?.executionContext instanceof
           QueryDataProductModelAccessExecutionContextInfo ||
         queryInfo?.executionContext instanceof
-          QueryDataProductLakehouseExecutionContextInfo
+          QueryDataProductLakehouseExecutionContextInfo ||
+        queryInfo?.executionContext instanceof QueryIngestExecutionContextInfo
       )
     ) {
       yield flowResult(this.buildFullGraph());
+    } else if (
+      queryInfo?.executionContext instanceof QueryIngestExecutionContextInfo
+    ) {
+      // Mirror the creator-store flow: build a graph containing only the
+      // referenced ingest definition. The full project entity fetch is
+      // skipped, but we still list every ingest in the project so the source
+      // dropdown can show them and `swapIngestDefinition` can build a
+      // different one on demand.
+      yield* this.buildIngestGraph(
+        queryInfo.groupId,
+        queryInfo.artifactId,
+        queryInfo.versionId,
+        queryInfo.executionContext.ingestDefinitionPath,
+      );
     }
   }
 
+  /**
+   * Cached ingest entities for the existing-query flow. Populated by
+   * {@link buildIngestGraph} (called from {@link buildGraph}) and consumed by
+   * {@link initQueryBuildStateFromQuery} to back the source dropdown +
+   * {@link IngestLegendQueryBuilderState.swapIngest} callback.
+   */
+  private _ingestEntitiesByPath: Map<string, Entity> = new Map();
+
+  private *buildIngestGraph(
+    groupId: string,
+    artifactId: string,
+    versionId: string,
+    ingestDefinitionPath: string,
+  ): GeneratorFn<void> {
+    yield this.graphManagerState.initializeSystem();
+
+    this._ingestEntitiesByPath = (yield fetchIngestEntitiesByClassifier(
+      this.depotServerClient,
+      groupId,
+      artifactId,
+      versionId,
+    )) as Map<string, Entity>;
+
+    const target = this._ingestEntitiesByPath.get(ingestDefinitionPath);
+    if (!target) {
+      throw new Error(
+        `Can't find ingest definition '${ingestDefinitionPath}' in project ${groupId}:${artifactId}:${versionId}`,
+      );
+    }
+    yield swapIngestInGraph(this.graphManagerState, target, {
+      groupId,
+      artifactId,
+      versionId,
+    });
+  }
+
   override async setUpEditorState(): Promise<void> {
     const queryInfo = await this.graphManagerState.graphManager.getQueryInfo(
       this.queryId,
@@ -2191,6 +2256,71 @@ export class ExistingQueryEditorStore extends QueryEditorStore {
         },
       );
       return queryBuilderState;
+    } else if (exec instanceof QueryIngestExecutionContextInfo) {
+      // The graph was already populated by `buildIngestGraph` (above) with
+      // just the referenced ingest definition. Resolve it, build the adhoc
+      // lakehouse runtime, and stand up the ingest query builder state.
+      const ingestDefinition = guaranteeType(
+        this.graphManagerState.graph.getElement(exec.ingestDefinitionPath),
+        IngestDefinition,
+        `Can't find ingest definition '${exec.ingestDefinitionPath}'`,
+      );
+      const projectInfo = this.getProjectInfo();
+      const adhocRuntime = await this.createLakehousePackageableRuntime(
+        exec.ingestDefinitionPath,
+        projectInfo,
+      );
+      this.graphManagerState.graph.addElement(adhocRuntime, '_internal_');
+
+      // Tracked separately from the closure on `ingestQueryBuilderState` so
+      // the swap callback can capture it directly without forming a self-
+      // reference (which trips the `no-unsafe-assignment` rule).
+      let currentIngestPath = exec.ingestDefinitionPath;
+      const swapIngest = async (path: string): Promise<IngestDefinition> => {
+        // Mirror `IngestQueryCreatorStore.swapIngestDefinition`: rebuild
+        // the requested ingest in place of the current one. The entity
+        // cache was populated up front by `buildIngestGraph`.
+        const entity = this._ingestEntitiesByPath.get(path);
+        if (!entity) {
+          throw new Error(
+            `Can't find ingest definition '${path}' in project ${projectInfo.groupId}:${projectInfo.artifactId}:${projectInfo.versionId}`,
+          );
+        }
+        const next = await swapIngestInGraph(this.graphManagerState, entity, {
+          currentPath: currentIngestPath,
+          groupId: projectInfo.groupId,
+          artifactId: projectInfo.artifactId,
+          versionId: projectInfo.versionId,
+        });
+        currentIngestPath = next.path;
+        return next;
+      };
+
+      const ingestQueryBuilderState = new IngestLegendQueryBuilderState(
+        this.applicationStore,
+        undefined,
+        this.graphManagerState,
+        QueryBuilderDataBrowserWorkflow.INSTANCE,
+        new QueryBuilderActionConfig_QueryApplication(this),
+        ingestDefinition,
+        Array.from(this._ingestEntitiesByPath.keys()),
+        swapIngest,
+        adhocRuntime,
+        projectInfo,
+        this.applicationStore.config.options.queryBuilderConfig,
+        {
+          groupId: projectInfo.groupId,
+          artifactId: projectInfo.artifactId,
+          versionId: projectInfo.versionId,
+          queryId: queryInfo.id,
+        },
+      );
+      await ingestQueryBuilderState.changeAccessorOwner(ingestDefinition);
+      await ingestQueryBuilderState.changeAccessor({
+        tableName: exec.dataSet,
+      });
+      ingestQueryBuilderState.changeSelectedRuntime(adhocRuntime);
+      return ingestQueryBuilderState;
     }
     throw new UnsupportedOperationError(`Unsupported query execution context`);
   }