@finos/legend-application-marketplace 0.2.12 → 0.2.14

package/src/stores/lakehouse/entitlements/EntitlementsDashboardState.ts

@@ -18,13 +18,17 @@ import {
   ActionState,
   assertErrorThrown,
   guaranteeNonNullable,
+  guaranteeType,
+  HttpStatus,
   isNonNullable,
+  NetworkClientError,
   type GeneratorFn,
   type PlainObject,
 } from '@finos/legend-shared';
 import { deserialize } from 'serializr';
 import {
-  type V1_DataContract,
+  type PureProtocolProcessorPlugin,
+  type V1_DataProduct,
   type V1_EnrichedUserApprovalStatus,
   type V1_LiteDataContract,
   type V1_LiteDataContractWithUserStatus,
@@ -32,13 +36,23 @@ import {
   type V1_TaskStatus,
   type V1_ContractUserEventRecord,
   type V1_TaskStatusChangeResponse,
-  V1_dataContractsResponseModelSchema,
+  RawLambda,
+  V1_DataProductAccessor,
+  V1_deserializeDataContractResponse,
   V1_entitlementsDataProductDetailsResponseToDataProductDetails,
+  V1_IngestDefinitionAccessor,
+  V1_LakehouseAccessPoint,
   V1_liteDataContractWithUserStatusModelSchema,
   V1_pendingTasksResponseModelSchema,
+  V1_PureGraphManager,
+  V1_resolveAccessorsFromRawLambda,
+  V1_ResourceType,
+  V1_SdlcDeploymentDataProductOrigin,
   V1_TaskStatusChangeResponseModelSchema,
   V1_transformDataContractToLiteDatacontract,
 } from '@finos/legend-graph';
+import { DEFAULT_TAB_SIZE } from '@finos/legend-application';
+import type { ContractErrorLayer } from '@finos/legend-extension-dsl-data-product';
 import {
   makeObservable,
   flow,
@@ -51,6 +65,94 @@ import {
   TEST_USER,
   type LakehouseEntitlementsStore,
 } from './LakehouseEntitlementsStore.js';
+import { getDataProductFromDetails } from '../../../utils/LakehouseUtils.js';
+
+export enum ContractSyncStatus {
+  NEVER_SYNCED = 'NEVER_SYNCED',
+  NOT_FULLY_SYNCED = 'NOT_FULLY_SYNCED',
+}
+
+export type LakehouseContractSyncStatusResponse = {
+  status: string;
+  unsyncedUsers?: { username: string }[];
+  unsyncedAccessPoints?: { accessPointName: string }[];
+  unsyncedTargetAccounts?: string[];
+};
+
+const collectIngestSpecPathsFromOriginDp = (
+  rootDataProduct: V1_DataProduct,
+  accessPointGroupId: string,
+  graphManager: V1_PureGraphManager,
+  plugins: PureProtocolProcessorPlugin[],
+): Set<string> => {
+  const dpPath = `${rootDataProduct.package}::${rootDataProduct.name}`;
+  const targetApg = rootDataProduct.accessPointGroups.find(
+    (apg) => apg.id === accessPointGroupId,
+  );
+  if (!targetApg) {
+    throw new Error(
+      `Access point group '${accessPointGroupId}' not found in data product '${dpPath}'`,
+    );
+  }
+  const specs = new Set<string>();
+  const visited = new Set<string>();
+  const worklist: string[] = [accessPointGroupId];
+
+  const collectFromApg = (apgId: string): void => {
+    const apg = rootDataProduct.accessPointGroups.find((g) => g.id === apgId);
+    if (!apg) {
+      return;
+    }
+    for (const accessPoint of apg.accessPoints) {
+      if (!(accessPoint instanceof V1_LakehouseAccessPoint)) {
+        continue;
+      }
+      const visitKey = `${dpPath}::${accessPoint.id}`;
+      if (visited.has(visitKey)) {
+        continue;
+      }
+      visited.add(visitKey);
+
+      const rawLambda = new RawLambda(
+        accessPoint.func.parameters,
+        accessPoint.func.body,
+      );
+      const accessors =
+        V1_resolveAccessorsFromRawLambda(rawLambda, graphManager, plugins) ??
+        [];
+      for (const accessor of accessors) {
+        if (accessor instanceof V1_IngestDefinitionAccessor) {
+          const specPath = accessor.path[0];
+          if (specPath) {
+            specs.add(specPath);
+          }
+        } else if (accessor instanceof V1_DataProductAccessor) {
+          const refDpPath = accessor.path[0];
+          const refApId = accessor.path[1];
+          if (!refDpPath || !refApId) {
+            continue;
+          }
+          if (refDpPath !== dpPath) {
+            continue;
+          }
+          const refApg = rootDataProduct.accessPointGroups.find((g) =>
+            g.accessPoints.some((ap) => ap.id === refApId),
+          );
+          if (refApg && !worklist.includes(refApg.id)) {
+            worklist.push(refApg.id);
+          }
+        }
+      }
+    }
+  };
+
+  while (worklist.length > 0) {
+    const apgId = guaranteeNonNullable(worklist.shift());
+    collectFromApg(apgId);
+  }
+
+  return specs;
+};
 
 export class ContractCreatedByUserDetails {
   readonly contractResultLite: V1_LiteDataContract;
@@ -246,30 +348,35 @@ export class EntitlementsDashboardState {
     const pendingTaskContractIds = Array.from(
       new Set(pendingTasks.map((t) => t.dataContractId)),
     );
-    const pendingTaskContracts = (
-      (yield Promise.all(
-        pendingTaskContractIds.map(async (contractId) => {
-          const rawContractResponse =
-            await this.lakehouseEntitlementsStore.lakehouseContractServerClient.getDataContract(
-              contractId,
-              false,
-              token,
-            );
-          const contractResponse = deserialize(
-            V1_dataContractsResponseModelSchema(
-              this.lakehouseEntitlementsStore.applicationStore.pluginManager.getPureProtocolProcessorPlugins(),
-            ),
-            rawContractResponse,
+    const contractClient =
+      this.lakehouseEntitlementsStore.lakehouseContractServerClient;
+    const plugins =
+      this.lakehouseEntitlementsStore.applicationStore.pluginManager.getPureProtocolProcessorPlugins();
+    const pendingTaskContracts = (yield Promise.all(
+      pendingTaskContractIds.map(async (contractId) => {
+        try {
+          const rawContractResponse = await contractClient.getDataContract(
+            contractId,
+            false,
+            token,
           );
-          return contractResponse.dataContracts?.[0]?.dataContract;
-        }),
-      )) as (V1_DataContract | undefined)[]
-    )
-      .filter(isNonNullable)
-      .map(V1_transformDataContractToLiteDatacontract);
+          const dataContract = V1_deserializeDataContractResponse(
+            rawContractResponse,
+            plugins,
+          )[0]?.dataContract;
+          if (!dataContract) {
+            return undefined;
+          }
+          return V1_transformDataContractToLiteDatacontract(dataContract);
+        } catch (error) {
+          assertErrorThrown(error);
+          return undefined;
+        }
+      }),
+    )) as (V1_LiteDataContract | undefined)[];
     const resultMap = new Map<string, V1_LiteDataContract>();
-    pendingTaskContractIds.forEach((contractId) => {
-      const contract = pendingTaskContracts.find((c) => c.guid === contractId);
+    pendingTaskContractIds.forEach((contractId, idx) => {
+      const contract = pendingTaskContracts[idx];
       if (contract) {
         resultMap.set(contractId, contract);
       }
@@ -356,29 +463,26 @@ export class EntitlementsDashboardState {
     }
 
     const didToEnvType = new Map<number, string>();
+    const contractClient =
+      this.lakehouseEntitlementsStore.lakehouseContractServerClient;
     yield Promise.all(
       Array.from(uniqueDIDToDataProduct.entries()).map(
         async ([deploymentId, resourceId]) => {
           try {
-            const raw =
-              await this.lakehouseEntitlementsStore.lakehouseContractServerClient.getDataProductByIdAndDID(
-                resourceId,
-                deploymentId,
-                token,
-              );
-            const details =
+            const raw = await contractClient.getDataProductByIdAndDID(
+              resourceId,
+              deploymentId,
+              token,
+            );
+            const env =
               V1_entitlementsDataProductDetailsResponseToDataProductDetails(
                 raw,
-              );
-            const env = details[0]?.lakehouseEnvironment?.type;
+              )[0]?.lakehouseEnvironment?.type;
             if (env) {
               didToEnvType.set(deploymentId, env);
             }
           } catch (error) {
             assertErrorThrown(error);
-            this.lakehouseEntitlementsStore.applicationStore.notificationService.notifyError(
-              `Error fetching deployment environment for deployment ${deploymentId}: ${error.message}`,
-            );
           }
         },
       ),
@@ -386,7 +490,237 @@ export class EntitlementsDashboardState {
     return didToEnvType;
   }
 
-  private filterByUserEnvironment(
+  async getUnverifiedIngestDefinitions(
+    contractId: string,
+    token: string | undefined,
+  ): Promise<string[]> {
+    const entitlementsStore = this.lakehouseEntitlementsStore;
+    const baseStore = entitlementsStore.marketplaceBaseStore;
+    const applicationStore = entitlementsStore.applicationStore;
+    const plugins =
+      applicationStore.pluginManager.getPureProtocolProcessorPlugins();
+    const contractClient = entitlementsStore.lakehouseContractServerClient;
+
+    const PROD_ENV = 'prod';
+    const SDLC_DEPLOYMENT = 'alloy-git';
+
+    try {
+      const liteContract = await (async () => {
+        try {
+          const rawContractResponse = await contractClient.getDataContract(
+            contractId,
+            false,
+            token,
+          );
+          const dataContract = V1_deserializeDataContractResponse(
+            rawContractResponse,
+            plugins,
+          )[0]?.dataContract;
+          if (!dataContract) {
+            return undefined;
+          }
+          return V1_transformDataContractToLiteDatacontract(dataContract);
+        } catch (error) {
+          assertErrorThrown(error);
+          return undefined;
+        }
+      })();
+      if (!liteContract) {
+        return [];
+      }
+
+      const accessPointGroupId =
+        liteContract.resourceType === V1_ResourceType.ACCESS_POINT_GROUP
+          ? (liteContract.accessPointGroup ?? undefined)
+          : undefined;
+      if (!accessPointGroupId) {
+        return [];
+      }
+
+      const dpDetails = await (async () => {
+        try {
+          const raw = await contractClient.getDataProductByIdAndDID(
+            liteContract.resourceId,
+            liteContract.deploymentId,
+            token,
+          );
+          return V1_entitlementsDataProductDetailsResponseToDataProductDetails(
+            raw,
+          )[0];
+        } catch (error) {
+          assertErrorThrown(error);
+          return undefined;
+        }
+      })();
+      if (!dpDetails) {
+        return [];
+      }
+
+      if (!(dpDetails.origin instanceof V1_SdlcDeploymentDataProductOrigin)) {
+        return [];
+      }
+
+      const graphManager = new V1_PureGraphManager(
+        applicationStore.pluginManager,
+        applicationStore.logService,
+        baseStore.remoteEngine,
+      );
+      await graphManager.initialize(
+        {
+          env: applicationStore.config.env,
+          tabSize: DEFAULT_TAB_SIZE,
+          clientConfig: {
+            baseUrl: applicationStore.config.engineServerUrl,
+          },
+        },
+        { engine: baseStore.remoteEngine },
+      );
+
+      const v1DataProduct = await getDataProductFromDetails(
+        dpDetails,
+        graphManager,
+        baseStore,
+      );
+      if (!v1DataProduct) {
+        return [];
+      }
+
+      const specs = collectIngestSpecPathsFromOriginDp(
+        v1DataProduct,
+        accessPointGroupId,
+        graphManager,
+        plugins,
+      );
+      if (specs.size === 0) {
+        return [];
+      }
+
+      const ingestEnvironment =
+        await baseStore.lakehouseDataProductService.getOrFetchEnvironmentForDID(
+          liteContract.deploymentId,
+          token,
+        );
+      const ingestServerUrl = ingestEnvironment?.ingestServerUrl;
+      if (ingestServerUrl === undefined) {
+        return [];
+      }
+
+      const sdlcOrigin = guaranteeType(
+        dpDetails.origin,
+        V1_SdlcDeploymentDataProductOrigin,
+      );
+      const gav = `${sdlcOrigin.group}~${sdlcOrigin.artifact}`;
+      const specsToVerify = Array.from(specs, (specPath) => ({
+        specPath,
+        urn: `urn:lakehouse:${PROD_ENV}:ingest:definition:${SDLC_DEPLOYMENT}:${gav}~${specPath}`,
+      }));
+
+      const ingestClient = baseStore.lakehouseIngestServerClient;
+      const settled = await Promise.all(
+        specsToVerify.map(async (entry) => {
+          try {
+            await ingestClient.getIngestDefinitionDetail(
+              entry.urn,
+              ingestServerUrl,
+              token,
+            );
+            return undefined;
+          } catch (error) {
+            assertErrorThrown(error);
+            if (
+              error instanceof NetworkClientError &&
+              error.response.status === HttpStatus.NOT_FOUND
+            ) {
+              return entry.specPath;
+            }
+            return undefined;
+          }
+        }),
+      );
+      return settled.filter(isNonNullable);
+    } catch (error) {
+      assertErrorThrown(error);
+      return [];
+    }
+  }
+
+  async getContractSyncErrors(
+    contractId: string,
+    token: string | undefined,
+  ): Promise<ContractErrorLayer | undefined> {
+    try {
+      const response =
+        (await this.lakehouseEntitlementsStore.lakehouseContractServerClient.getContractSyncStatus(
+          contractId,
+          token,
+        )) as LakehouseContractSyncStatusResponse;
+
+      const status = response.status.toUpperCase();
+
+      if (status === ContractSyncStatus.NEVER_SYNCED) {
+        return { title: 'Sync Error: Contract Never Synced' };
+      }
+
+      if (status === ContractSyncStatus.NOT_FULLY_SYNCED) {
+        const unsyncedUsers =
+          response.unsyncedUsers?.map((user) => user.username) ?? [];
+        const unsyncedAccessPoints =
+          response.unsyncedAccessPoints?.map(
+            (accessPoint) => accessPoint.accessPointName,
+          ) ?? [];
+        const unsyncedTargetAccounts = response.unsyncedTargetAccounts ?? [];
+
+        const syncGroupingLayers: ContractErrorLayer[] = [
+          { title: 'Users', errorItems: unsyncedUsers },
+          { title: 'Target Accounts', errorItems: unsyncedTargetAccounts },
+          { title: 'Access Points', errorItems: unsyncedAccessPoints },
+        ].filter((layer) => layer.errorItems.length > 0);
+
+        if (syncGroupingLayers.length === 0) {
+          return undefined;
+        }
+
+        return {
+          title: 'Unsynced Entities',
+          childLayers: syncGroupingLayers,
+        };
+      }
+
+      return undefined;
+    } catch (error) {
+      assertErrorThrown(error);
+      return undefined;
+    }
+  }
+
+  async getContractErrors(
+    contractId: string,
+    token: string | undefined,
+    checkSyncStatus = false,
+  ): Promise<ContractErrorLayer | undefined> {
+    const [unverifiedIngestDefinitions, syncErrorsLayer] = await Promise.all([
+      this.getUnverifiedIngestDefinitions(contractId, token),
+      checkSyncStatus
+        ? this.getContractSyncErrors(contractId, token)
+        : Promise.resolve(undefined),
+    ]);
+
+    const childLayers: ContractErrorLayer[] = [
+      unverifiedIngestDefinitions.length > 0
+        ? {
+            title: `Ingest${unverifiedIngestDefinitions.length === 1 ? '' : 's'} Not Found`,
+            errorItems: unverifiedIngestDefinitions,
+          }
+        : undefined,
+      syncErrorsLayer,
+    ].filter(isNonNullable);
+
+    return childLayers.length > 0
+      ? { title: 'Contract Errors', childLayers }
+      : undefined;
+  }
+
+  filterByUserEnvironment(
     pendingData: {
       tasks: V1_ContractUserEventRecord[];
       taskContractMap: Map<string, V1_LiteDataContract>;

package/src/stores/lakehouse/fieldSearch/FieldSearchResultState.ts

@@ -19,6 +19,7 @@ import {
   type GroupedFieldSearchDataProduct,
   type GroupedFieldSearchResultEntry,
 } from '@finos/legend-server-marketplace';
+import { hashArray } from '@finos/legend-shared';
 import { DataProductTypeFilter } from '../LegendMarketplaceSearchResultsStore.js';
 import { generateGAVCoordinates } from '@finos/legend-storage';
 import {
@@ -26,6 +27,15 @@ import {
   generateLegacyDataProductPath,
 } from '../../../__lib__/LegendMarketplaceNavigation.js';
 
+enum FieldSearchResultStateDefaultValue {
+  UNKNOWN_FIELD_TYPE = 'Unknown',
+  EMPTY_FIELD_DESCRIPTION = '-',
+}
+
+enum FieldSearchDataProductKey {
+  DISTINCT_SEPARATOR = '|',
+}
+
 const PRODUCT_TYPE_FILTER_MAP: Record<
   DataProductSearchResultDetailsType,
   DataProductTypeFilter
@@ -39,6 +49,28 @@ const PRODUCT_TYPE_FILTER_MAP: Record<
 const getDataProductName = (path: string): string =>
   path.split('::').at(-1) ?? path;
 
+const generateFieldSearchResultId = (
+  fieldName: string,
+  fieldType: string,
+  fieldDescription: string,
+): string => `${hashArray([fieldName, fieldType, fieldDescription])}`;
+
+const getDistinctDataProducts = (
+  dataProducts: FieldSearchDataProductEntry[],
+): FieldSearchDataProductEntry[] => {
+  const seen = new Set<string>();
+  return dataProducts.filter((dp) => {
+    // Dedupe primarily by owning data product path. Fallback to a stable
+    // composite key only when path is unavailable.
+    const dedupeKey = dp.path || dp.distinctKey;
+    if (seen.has(dedupeKey)) {
+      return false;
+    }
+    seen.add(dedupeKey);
+    return true;
+  });
+};
+
 const getOwningDataProductPath = (
   dataProduct: GroupedFieldSearchDataProduct,
 ): string => {
@@ -71,6 +103,8 @@ const getOwningDataProductPath = (
 export class FieldSearchDataProductEntry {
   readonly name: string;
   readonly datasetName: string | undefined;
+  readonly datasetDescription: string | undefined;
+  readonly executionContextKey: string | undefined;
   readonly modelPath: string | undefined;
   readonly path: string;
   readonly entityPath: string;
@@ -80,6 +114,7 @@ export class FieldSearchDataProductEntry {
   readonly artifactId: string | undefined;
   readonly versionId: string | undefined;
   readonly productType: DataProductTypeFilter | undefined;
+  readonly distinctKey: string;
 
   constructor(dataProduct: GroupedFieldSearchDataProduct) {
     const productType = PRODUCT_TYPE_FILTER_MAP[dataProduct.productType];
@@ -87,6 +122,8 @@ export class FieldSearchDataProductEntry {
 
     this.name = dataProductName;
     this.datasetName = dataProduct.datasetName;
+    this.datasetDescription = dataProduct.datasetDescription;
+    this.executionContextKey = dataProduct.defaultExecutionContext;
     this.modelPath = dataProduct.modelPath;
     this.path = getOwningDataProductPath(dataProduct);
     this.entityPath = dataProduct.path;
@@ -96,6 +133,12 @@ export class FieldSearchDataProductEntry {
     this.artifactId = dataProduct.artifactId;
     this.versionId = dataProduct.versionId;
     this.productType = productType;
+    this.distinctKey = [
+      this.path,
+      this.entityPath,
+      this.dataProductId,
+      this.name,
+    ].join(FieldSearchDataProductKey.DISTINCT_SEPARATOR);
   }
 }
 
@@ -105,18 +148,24 @@ export class FieldSearchResultState {
   readonly fieldType: string;
   readonly fieldDescription: string;
   readonly dataProducts: FieldSearchDataProductEntry[];
+  readonly distinctDataProducts: FieldSearchDataProductEntry[];
 
   constructor(result: GroupedFieldSearchResultEntry) {
     this.fieldName = result.fieldName;
-    this.fieldType = result.fieldType ?? 'Unknown';
-    this.fieldDescription = result.fieldDescription ?? '-';
-    this.id = JSON.stringify([
+    this.fieldType =
+      result.fieldType ?? FieldSearchResultStateDefaultValue.UNKNOWN_FIELD_TYPE;
+    this.fieldDescription =
+      result.fieldDescription ??
+      FieldSearchResultStateDefaultValue.EMPTY_FIELD_DESCRIPTION;
+    this.id = generateFieldSearchResultId(
       this.fieldName,
       this.fieldType,
       this.fieldDescription,
-    ]);
+    );
     this.dataProducts = result.dataProducts.map(
       (dp) => new FieldSearchDataProductEntry(dp),
     );
+    // Precompute once since dataProducts are immutable after construction.
+    this.distinctDataProducts = getDistinctDataProducts(this.dataProducts);
   }
 }