@finos/legend-application-marketplace 0.2.11 → 0.2.13

package/src/pages/Lakehouse/searchResults/LegendMarketplaceFieldSearchResults.tsx

@@ -20,20 +20,22 @@ import { flowResult } from 'mobx';
 import { useCallback, useEffect, useRef } from 'react';
 import { useSyncStateAndSearchParam } from '@finos/legend-application';
 import { useSearchParams } from '@finos/legend-application/browser';
-import { isNonEmptyString } from '@finos/legend-shared';
+import { isNonEmptyString, LogEvent } from '@finos/legend-shared';
 import {
   CubesLoadingIndicator,
   CubesLoadingIndicatorIcon,
 } from '@finos/legend-art';
-import { DATAPRODUCT_TYPE } from '@finos/legend-extension-dsl-data-product';
 import {
   LEGEND_MARKETPLACE_FIELD_SEARCH_RESULTS_QUERY_PARAM_TOKEN,
   generateLakehouseSearchResultsRoute,
+  EXTERNAL_APPLICATION_NAVIGATION__generateDataSpaceQueryEditorUrl,
 } from '../../../__lib__/LegendMarketplaceNavigation.js';
 import {
   LEGEND_MARKETPLACE_PAGE,
   LegendMarketplaceTelemetryHelper,
 } from '../../../__lib__/LegendMarketplaceTelemetryHelper.js';
+
+import { LEGEND_MARKETPLACE_APP_EVENT } from '../../../__lib__/LegendMarketplaceAppEvent.js';
 import {
   useLegendMarketplaceFieldSearchResultsStore,
   withLegendMarketplaceFieldSearchResultsStore,
@@ -54,6 +56,9 @@ const FieldSearchResultsContent = observer(
     handleItemsPerPageChange: (itemsPerPage: number) => void;
     handleToggleExpandRow: (rowId: string) => void;
     handleOpenDataProduct: (dataProduct: FieldSearchDataProductEntry) => void;
+    handleOpenDatasetInQuery: (
+      dataProduct: FieldSearchDataProductEntry,
+    ) => void;
   }) => {
     const {
       fieldSearchResultsStore,
@@ -61,6 +66,7 @@ const FieldSearchResultsContent = observer(
       handleItemsPerPageChange,
       handleToggleExpandRow,
       handleOpenDataProduct,
+      handleOpenDatasetInQuery,
     } = props;
 
     if (fieldSearchResultsStore.isLoading) {
@@ -160,6 +166,9 @@ const FieldSearchResultsContent = observer(
             <Typography className="marketplace-lakehouse-field-search-results__list-header-cell">
               Data Products
             </Typography>
+            <Typography className="marketplace-lakehouse-field-search-results__list-header-cell">
+              Datasets
+            </Typography>
           </div>
           <div className="marketplace-lakehouse-field-search-results__list-body">
             {fieldSearchResultsStore.tableRows.map((row) => (
@@ -169,6 +178,7 @@ const FieldSearchResultsContent = observer(
                 expanded={fieldSearchResultsStore.isRowExpanded(row.id)}
                 onToggleExpanded={handleToggleExpandRow}
                 onOpenDataProduct={handleOpenDataProduct}
+                onOpenDatasetInQuery={handleOpenDatasetInQuery}
               />
             ))}
           </div>
@@ -283,29 +293,45 @@ const LegendMarketplaceFieldSearchResultsPage = observer(() => {
     [applicationStore, fieldSearchResultsStore],
   );
 
-  const handleOpenDataProduct = useCallback(
-    (dataProduct: FieldSearchDataProductEntry) => {
-      LegendMarketplaceTelemetryHelper.logEvent_ClickingDataProductCard(
-        applicationStore.telemetryService,
-        {
-          origin:
-            dataProduct.productType === DataProductTypeFilter.LEGACY
-              ? {
-                  type: DATAPRODUCT_TYPE.SDLC,
-                  groupId: dataProduct.groupId,
-                  artifactId: dataProduct.artifactId,
-                  versionId: dataProduct.versionId,
-                  path: dataProduct.entityPath,
-                }
-              : {
-                  type: DATAPRODUCT_TYPE.ADHOC,
-                },
-          dataProductId: dataProduct.dataProductId,
-          name: dataProduct.name,
-          deploymentId: dataProduct.deploymentId,
-        },
-        LEGEND_MARKETPLACE_PAGE.SEARCH_RESULTS_PAGE,
-      );
+  const openInLegendQuery = useCallback(
+    (
+      dataProduct: FieldSearchDataProductEntry,
+      modelPath: string | undefined,
+    ) => {
+      const isLegacyDataProduct =
+        dataProduct.productType === DataProductTypeFilter.LEGACY;
+      const canOpenLegacyQuery =
+        dataProduct.groupId &&
+        dataProduct.artifactId &&
+        dataProduct.versionId &&
+        dataProduct.executionContextKey;
+
+      if (isLegacyDataProduct && canOpenLegacyQuery) {
+        applicationStore.navigationService.navigator.visitAddress(
+          EXTERNAL_APPLICATION_NAVIGATION__generateDataSpaceQueryEditorUrl(
+            applicationStore.config.queryApplicationUrl,
+            dataProduct.groupId,
+            dataProduct.artifactId,
+            dataProduct.versionId,
+            dataProduct.entityPath,
+            dataProduct.executionContextKey,
+            undefined,
+            modelPath,
+          ),
+        );
+        return;
+      }
+
+      if (isLegacyDataProduct && !canOpenLegacyQuery) {
+        applicationStore.logService.warn(
+          LogEvent.create(
+            LEGEND_MARKETPLACE_APP_EVENT.CLICK_QUERY_DATA_PRODUCT,
+          ),
+          `Falling back to marketplace navigation for legacy data product ${dataProduct.entityPath} due to missing query context`,
+        );
+      }
+
+      // Fallback to marketplace data product page for adhoc/lakehouse products
       applicationStore.navigationService.navigator.visitAddress(
         applicationStore.navigationService.navigator.generateAddress(
           dataProduct.path,
@@ -315,6 +341,20 @@ const LegendMarketplaceFieldSearchResultsPage = observer(() => {
     [applicationStore],
   );
 
+  const handleOpenDataProduct = useCallback(
+    (dataProduct: FieldSearchDataProductEntry) => {
+      openInLegendQuery(dataProduct, undefined);
+    },
+    [openInLegendQuery],
+  );
+
+  const handleOpenDatasetInQuery = useCallback(
+    (dataProduct: FieldSearchDataProductEntry) => {
+      openInLegendQuery(dataProduct, dataProduct.modelPath);
+    },
+    [openInLegendQuery],
+  );
+
   const handleToggleExpandRow = useCallback(
     (rowId: string) => {
       fieldSearchResultsStore.toggleExpandRow(rowId);
@@ -322,6 +362,15 @@ const LegendMarketplaceFieldSearchResultsPage = observer(() => {
     [fieldSearchResultsStore],
   );
 
+  const handleOpenDataProductsTab = useCallback(() => {
+    applicationStore.navigationService.navigator.goToLocation(
+      generateLakehouseSearchResultsRoute(
+        fieldSearchResultsStore.searchQuery,
+        false,
+      ),
+    );
+  }, [applicationStore, fieldSearchResultsStore.searchQuery]);
+
   return (
     <LegendMarketplacePage className="marketplace-lakehouse-search-results marketplace-lakehouse-field-search-results">
       <div ref={pageRef} />
@@ -345,6 +394,31 @@ const LegendMarketplaceFieldSearchResultsPage = observer(() => {
           >
             {fieldSearchResultsStore.totalFieldMatches} Fields
           </Typography>
+          <div
+            className="legend-marketplace-search-results__search-type-tabs"
+            role="tablist"
+            aria-label="Search result type"
+          >
+            <button
+              type="button"
+              role="tab"
+              aria-selected={false}
+              tabIndex={-1}
+              className="legend-marketplace-search-results__search-type-tab"
+              onClick={handleOpenDataProductsTab}
+            >
+              Data Products
+            </button>
+            <button
+              type="button"
+              role="tab"
+              aria-selected={true}
+              tabIndex={0}
+              className="legend-marketplace-search-results__search-type-tab legend-marketplace-search-results__search-type-tab--active"
+            >
+              Data Fields
+            </button>
+          </div>
         </div>
       </div>
       <Container
@@ -366,6 +440,7 @@ const LegendMarketplaceFieldSearchResultsPage = observer(() => {
               handleItemsPerPageChange={handleItemsPerPageChange}
               handleToggleExpandRow={handleToggleExpandRow}
               handleOpenDataProduct={handleOpenDataProduct}
+              handleOpenDatasetInQuery={handleOpenDatasetInQuery}
             />
           </div>
         </div>

package/src/pages/Lakehouse/searchResults/LegendMarketplaceSearchResults.tsx

@@ -382,6 +382,43 @@ export const LegendMarketplaceSearchResults =
                   ? `${searchResultsStore.filterSortProducts?.length ?? 0} Products`
                   : `${searchResultsStore.totalItems} Products`}
               </Typography>
+              <div
+                className="legend-marketplace-search-results__search-type-tabs"
+                role="tablist"
+                aria-label="Search result type"
+              >
+                <button
+                  type="button"
+                  role="tab"
+                  aria-selected={true}
+                  tabIndex={0}
+                  className="legend-marketplace-search-results__search-type-tab legend-marketplace-search-results__search-type-tab--active"
+                >
+                  Data Products
+                </button>
+                <button
+                  type="button"
+                  role="tab"
+                  aria-selected={false}
+                  tabIndex={-1}
+                  className="legend-marketplace-search-results__search-type-tab"
+                  onClick={() => {
+                    if (isNonEmptyString(searchResultsStore.searchQuery)) {
+                      applicationStore.navigationService.navigator.goToLocation(
+                        generateFieldSearchResultsRoute(
+                          searchResultsStore.searchQuery,
+                        ),
+                      );
+                    } else {
+                      applicationStore.navigationService.navigator.goToLocation(
+                        generateFieldSearchResultsRoute(undefined),
+                      );
+                    }
+                  }}
+                >
+                  Data Fields
+                </button>
+              </div>
               <div className="legend-marketplace-search-results__sort-bar__controls">
                 <div className="legend-marketplace-search-results__view-toggle">
                   <div

package/src/stores/lakehouse/entitlements/EntitlementsDashboardState.ts

@@ -18,27 +18,40 @@ import {
   ActionState,
   assertErrorThrown,
   guaranteeNonNullable,
+  guaranteeType,
   isNonNullable,
   type GeneratorFn,
   type PlainObject,
 } from '@finos/legend-shared';
 import { deserialize } from 'serializr';
 import {
-  type V1_DataContract,
+  type PureProtocolProcessorPlugin,
+  type V1_DataProduct,
   type V1_EnrichedUserApprovalStatus,
+  type V1_EntitlementsDataProductDetails,
   type V1_LiteDataContract,
   type V1_LiteDataContractWithUserStatus,
   type V1_PendingTasksResponse,
   type V1_TaskStatus,
   type V1_ContractUserEventRecord,
   type V1_TaskStatusChangeResponse,
-  V1_dataContractsResponseModelSchema,
+  RawLambda,
+  V1_DataProductAccessor,
+  V1_deserializeDataContractResponse,
   V1_entitlementsDataProductDetailsResponseToDataProductDetails,
+  V1_IngestDefinitionAccessor,
+  V1_LakehouseAccessPoint,
   V1_liteDataContractWithUserStatusModelSchema,
   V1_pendingTasksResponseModelSchema,
+  V1_PureGraphManager,
+  V1_resolveAccessorsFromRawLambda,
+  V1_ResourceType,
+  V1_SdlcDeploymentDataProductOrigin,
   V1_TaskStatusChangeResponseModelSchema,
   V1_transformDataContractToLiteDatacontract,
 } from '@finos/legend-graph';
+import { DEFAULT_TAB_SIZE } from '@finos/legend-application';
+import { type IngestDeploymentServerConfig } from '@finos/legend-server-lakehouse';
 import {
   makeObservable,
   flow,
@@ -51,6 +64,82 @@ import {
   TEST_USER,
   type LakehouseEntitlementsStore,
 } from './LakehouseEntitlementsStore.js';
+import { getDataProductFromDetails } from '../../../utils/LakehouseUtils.js';
+
+const collectIngestSpecPathsFromOriginDp = (
+  rootDataProduct: V1_DataProduct,
+  accessPointGroupId: string,
+  graphManager: V1_PureGraphManager,
+  plugins: PureProtocolProcessorPlugin[],
+): Set<string> => {
+  const dpPath = `${rootDataProduct.package}::${rootDataProduct.name}`;
+  const targetApg = rootDataProduct.accessPointGroups.find(
+    (apg) => apg.id === accessPointGroupId,
+  );
+  if (!targetApg) {
+    throw new Error(
+      `Access point group '${accessPointGroupId}' not found in data product '${dpPath}'`,
+    );
+  }
+  const specs = new Set<string>();
+  const visited = new Set<string>();
+  const worklist: string[] = [accessPointGroupId];
+
+  const collectFromApg = (apgId: string): void => {
+    const apg = rootDataProduct.accessPointGroups.find((g) => g.id === apgId);
+    if (!apg) {
+      return;
+    }
+    for (const accessPoint of apg.accessPoints) {
+      if (!(accessPoint instanceof V1_LakehouseAccessPoint)) {
+        continue;
+      }
+      const visitKey = `${dpPath}::${accessPoint.id}`;
+      if (visited.has(visitKey)) {
+        continue;
+      }
+      visited.add(visitKey);
+
+      const rawLambda = new RawLambda(
+        accessPoint.func.parameters,
+        accessPoint.func.body,
+      );
+      const accessors =
+        V1_resolveAccessorsFromRawLambda(rawLambda, graphManager, plugins) ??
+        [];
+      for (const accessor of accessors) {
+        if (accessor instanceof V1_IngestDefinitionAccessor) {
+          const specPath = accessor.path[0];
+          if (specPath) {
+            specs.add(specPath);
+          }
+        } else if (accessor instanceof V1_DataProductAccessor) {
+          const refDpPath = accessor.path[0];
+          const refApId = accessor.path[1];
+          if (!refDpPath || !refApId) {
+            continue;
+          }
+          if (refDpPath !== dpPath) {
+            continue;
+          }
+          const refApg = rootDataProduct.accessPointGroups.find((g) =>
+            g.accessPoints.some((ap) => ap.id === refApId),
+          );
+          if (refApg && !worklist.includes(refApg.id)) {
+            worklist.push(refApg.id);
+          }
+        }
+      }
+    }
+  };
+
+  while (worklist.length > 0) {
+    const apgId = guaranteeNonNullable(worklist.shift());
+    collectFromApg(apgId);
+  }
+
+  return specs;
+};
 
 export class ContractCreatedByUserDetails {
   readonly contractResultLite: V1_LiteDataContract;
@@ -125,6 +214,7 @@ export class EntitlementsDashboardState {
       fetchContractsForUser: flow,
       fetchContractsCreatedByUser: flow,
       fetchContractDeploymentEnvironments: flow,
+      getUnverifiedIngestDefinitions: flow,
       updateContract: flow,
     });
   }
@@ -246,30 +336,35 @@ export class EntitlementsDashboardState {
     const pendingTaskContractIds = Array.from(
       new Set(pendingTasks.map((t) => t.dataContractId)),
     );
-    const pendingTaskContracts = (
-      (yield Promise.all(
-        pendingTaskContractIds.map(async (contractId) => {
-          const rawContractResponse =
-            await this.lakehouseEntitlementsStore.lakehouseContractServerClient.getDataContract(
-              contractId,
-              false,
-              token,
-            );
-          const contractResponse = deserialize(
-            V1_dataContractsResponseModelSchema(
-              this.lakehouseEntitlementsStore.applicationStore.pluginManager.getPureProtocolProcessorPlugins(),
-            ),
-            rawContractResponse,
+    const contractClient =
+      this.lakehouseEntitlementsStore.lakehouseContractServerClient;
+    const plugins =
+      this.lakehouseEntitlementsStore.applicationStore.pluginManager.getPureProtocolProcessorPlugins();
+    const pendingTaskContracts = (yield Promise.all(
+      pendingTaskContractIds.map(async (contractId) => {
+        try {
+          const rawContractResponse = await contractClient.getDataContract(
+            contractId,
+            false,
+            token,
           );
-          return contractResponse.dataContracts?.[0]?.dataContract;
-        }),
-      )) as (V1_DataContract | undefined)[]
-    )
-      .filter(isNonNullable)
-      .map(V1_transformDataContractToLiteDatacontract);
+          const dataContract = V1_deserializeDataContractResponse(
+            rawContractResponse,
+            plugins,
+          )[0]?.dataContract;
+          if (!dataContract) {
+            return undefined;
+          }
+          return V1_transformDataContractToLiteDatacontract(dataContract);
+        } catch (error) {
+          assertErrorThrown(error);
+          return undefined;
+        }
+      }),
+    )) as (V1_LiteDataContract | undefined)[];
     const resultMap = new Map<string, V1_LiteDataContract>();
-    pendingTaskContractIds.forEach((contractId) => {
-      const contract = pendingTaskContracts.find((c) => c.guid === contractId);
+    pendingTaskContractIds.forEach((contractId, idx) => {
+      const contract = pendingTaskContracts[idx];
       if (contract) {
         resultMap.set(contractId, contract);
       }
@@ -356,29 +451,26 @@ export class EntitlementsDashboardState {
     }
 
     const didToEnvType = new Map<number, string>();
+    const contractClient =
+      this.lakehouseEntitlementsStore.lakehouseContractServerClient;
     yield Promise.all(
       Array.from(uniqueDIDToDataProduct.entries()).map(
         async ([deploymentId, resourceId]) => {
           try {
-            const raw =
-              await this.lakehouseEntitlementsStore.lakehouseContractServerClient.getDataProductByIdAndDID(
-                resourceId,
-                deploymentId,
-                token,
-              );
-            const details =
+            const raw = await contractClient.getDataProductByIdAndDID(
+              resourceId,
+              deploymentId,
+              token,
+            );
+            const env =
               V1_entitlementsDataProductDetailsResponseToDataProductDetails(
                 raw,
-              );
-            const env = details[0]?.lakehouseEnvironment?.type;
+              )[0]?.lakehouseEnvironment?.type;
             if (env) {
               didToEnvType.set(deploymentId, env);
             }
           } catch (error) {
             assertErrorThrown(error);
-            this.lakehouseEntitlementsStore.applicationStore.notificationService.notifyError(
-              `Error fetching deployment environment for deployment ${deploymentId}: ${error.message}`,
-            );
           }
         },
       ),
@@ -386,6 +478,154 @@ export class EntitlementsDashboardState {
     return didToEnvType;
   }
 
+  *getUnverifiedIngestDefinitions(
+    contractId: string,
+    token: string | undefined,
+  ): GeneratorFn<string[] | undefined> {
+    const entitlementsStore = this.lakehouseEntitlementsStore;
+    const baseStore = entitlementsStore.marketplaceBaseStore;
+    const applicationStore = entitlementsStore.applicationStore;
+    const plugins =
+      applicationStore.pluginManager.getPureProtocolProcessorPlugins();
+    const contractClient = entitlementsStore.lakehouseContractServerClient;
+
+    const PROD_ENV = 'prod';
+    const SDLC_DEPLOYMENT = 'alloy-git';
+
+    try {
+      const liteContract = (yield (async () => {
+        try {
+          const rawContractResponse = await contractClient.getDataContract(
+            contractId,
+            false,
+            token,
+          );
+          const dataContract = V1_deserializeDataContractResponse(
+            rawContractResponse,
+            plugins,
+          )[0]?.dataContract;
+          if (!dataContract) {
+            return undefined;
+          }
+          return V1_transformDataContractToLiteDatacontract(dataContract);
+        } catch (error) {
+          assertErrorThrown(error);
+          return undefined;
+        }
+      })()) as V1_LiteDataContract | undefined;
+      if (!liteContract) {
+        return [];
+      }
+
+      const accessPointGroupId =
+        liteContract.resourceType === V1_ResourceType.ACCESS_POINT_GROUP
+          ? (liteContract.accessPointGroup ?? undefined)
+          : undefined;
+      if (!accessPointGroupId) {
+        return [];
+      }
+
+      const dpDetails = (yield (async () => {
+        try {
+          const raw = await contractClient.getDataProductByIdAndDID(
+            liteContract.resourceId,
+            liteContract.deploymentId,
+            token,
+          );
+          return V1_entitlementsDataProductDetailsResponseToDataProductDetails(
+            raw,
+          )[0];
+        } catch (error) {
+          assertErrorThrown(error);
+          return undefined;
+        }
+      })()) as V1_EntitlementsDataProductDetails | undefined;
+      if (!dpDetails) {
+        return [];
+      }
+
+      if (!(dpDetails.origin instanceof V1_SdlcDeploymentDataProductOrigin)) {
+        return [];
+      }
+
+      const graphManager = new V1_PureGraphManager(
+        applicationStore.pluginManager,
+        applicationStore.logService,
+        baseStore.remoteEngine,
+      );
+      yield graphManager.initialize(
+        {
+          env: applicationStore.config.env,
+          tabSize: DEFAULT_TAB_SIZE,
+          clientConfig: {
+            baseUrl: applicationStore.config.engineServerUrl,
+          },
+        },
+        { engine: baseStore.remoteEngine },
+      );
+
+      const v1DataProduct = (yield getDataProductFromDetails(
+        dpDetails,
+        graphManager,
+        baseStore,
+      )) as V1_DataProduct | undefined;
+      if (!v1DataProduct) {
+        return [];
+      }
+
+      const specs = collectIngestSpecPathsFromOriginDp(
+        v1DataProduct,
+        accessPointGroupId,
+        graphManager,
+        plugins,
+      );
+      if (specs.size === 0) {
+        return [];
+      }
+
+      const ingestEnvironment =
+        (yield baseStore.lakehouseDataProductService.getOrFetchEnvironmentForDID(
+          liteContract.deploymentId,
+          token,
+        )) as IngestDeploymentServerConfig | undefined;
+      const ingestServerUrl = ingestEnvironment?.ingestServerUrl;
+      if (ingestServerUrl === undefined) {
+        return [];
+      }
+
+      const sdlcOrigin = guaranteeType(
+        dpDetails.origin,
+        V1_SdlcDeploymentDataProductOrigin,
+      );
+      const gav = `${sdlcOrigin.group}~${sdlcOrigin.artifact}`;
+      const specsToVerify = Array.from(specs, (specPath) => ({
+        specPath,
+        urn: `urn:lakehouse:${PROD_ENV}:ingest:definition:${SDLC_DEPLOYMENT}:${gav}~${specPath}`,
+      }));
+
+      const ingestClient = baseStore.lakehouseIngestServerClient;
+      const settled = (yield Promise.all(
+        specsToVerify.map(async (entry) => {
+          try {
+            await ingestClient.getIngestDefinitionDetail(
+              entry.urn,
+              ingestServerUrl,
+              token,
+            );
+            return undefined;
+          } catch (error) {
+            assertErrorThrown(error);
+            return entry.specPath;
+          }
+        }),
+      )) as (string | undefined)[];
+      return settled.filter(isNonNullable);
+    } catch (error) {
+      assertErrorThrown(error);
+      return undefined;
+    }
+  }
+
   private filterByUserEnvironment(
     pendingData: {
       tasks: V1_ContractUserEventRecord[];