@finos/legend-application-marketplace 0.2.24 → 0.2.26

package/src/stores/lakehouse/entitlements/EntitlementsDashboardState.ts

@@ -15,20 +15,15 @@
  */
 
 import {
+  type GeneratorFn,
+  type PlainObject,
   ActionState,
   assertErrorThrown,
   guaranteeNonNullable,
-  guaranteeType,
-  HttpStatus,
   isNonNullable,
-  NetworkClientError,
-  type GeneratorFn,
-  type PlainObject,
 } from '@finos/legend-shared';
 import { deserialize } from 'serializr';
 import {
-  type PureProtocolProcessorPlugin,
-  type V1_DataProduct,
   type V1_EnrichedUserApprovalStatus,
   type V1_LiteDataContract,
   type V1_LiteDataContractWithUserStatus,
@@ -36,26 +31,24 @@ import {
   type V1_TaskStatus,
   type V1_ContractUserEventRecord,
   type V1_TaskStatusChangeResponse,
-  RawLambda,
-  V1_DataProductAccessor,
-  V1_deserializeDataContractResponse,
   type V1_DataRequestWithWorkflow,
+  type V1_DataRequestsWithWorkflowResponse,
+  V1_deserializeDataContractResponse,
   V1_entitlementsDataProductDetailsResponseToDataProductDetails,
-  V1_IngestDefinitionAccessor,
-  V1_LakehouseAccessPoint,
   V1_liteDataContractWithUserStatusModelSchema,
   V1_pendingTasksResponseModelSchema,
-  V1_PureGraphManager,
-  V1_resolveAccessorsFromRawLambda,
-  V1_ResourceType,
-  V1_SdlcDeploymentDataProductOrigin,
   V1_TaskStatusChangeResponseModelSchema,
   V1_transformDataContractToLiteDatacontract,
   V1_deserializeDataRequestsWithWorkflowResponse,
-  type V1_DataRequestsWithWorkflowResponse,
 } from '@finos/legend-graph';
-import { DEFAULT_TAB_SIZE } from '@finos/legend-application';
-import type { ContractErrorLayer } from '@finos/legend-extension-dsl-data-product';
+import {
+  type ContractErrorLayer,
+  type LakehouseContractSyncStatusResponse,
+  buildSyncErrorLayer,
+  buildContractErrorsRoot,
+  getContractAPGCoordinates,
+  getUnverifiedIngestDefinitionsForAPG,
+} from '@finos/legend-extension-dsl-data-product';
 import {
   makeObservable,
   flow,
@@ -65,97 +58,9 @@ import {
   computed,
 } from 'mobx';
 import {
-  TEST_USER,
   type LakehouseEntitlementsStore,
+  TEST_USER,
 } from './LakehouseEntitlementsStore.js';
-import { getDataProductFromDetails } from '../../../utils/LakehouseUtils.js';
-
-export enum ContractSyncStatus {
-  NEVER_SYNCED = 'NEVER_SYNCED',
-  NOT_FULLY_SYNCED = 'NOT_FULLY_SYNCED',
-}
-
-export type LakehouseContractSyncStatusResponse = {
-  status: string;
-  unsyncedUsers?: { username: string }[];
-  unsyncedAccessPoints?: { accessPointName: string }[];
-  unsyncedTargetAccounts?: string[];
-};
-
-const collectIngestSpecPathsFromOriginDp = (
-  rootDataProduct: V1_DataProduct,
-  accessPointGroupId: string,
-  graphManager: V1_PureGraphManager,
-  plugins: PureProtocolProcessorPlugin[],
-): Set<string> => {
-  const dpPath = `${rootDataProduct.package}::${rootDataProduct.name}`;
-  const targetApg = rootDataProduct.accessPointGroups.find(
-    (apg) => apg.id === accessPointGroupId,
-  );
-  if (!targetApg) {
-    throw new Error(
-      `Access point group '${accessPointGroupId}' not found in data product '${dpPath}'`,
-    );
-  }
-  const specs = new Set<string>();
-  const visited = new Set<string>();
-  const worklist: string[] = [accessPointGroupId];
-
-  const collectFromApg = (apgId: string): void => {
-    const apg = rootDataProduct.accessPointGroups.find((g) => g.id === apgId);
-    if (!apg) {
-      return;
-    }
-    for (const accessPoint of apg.accessPoints) {
-      if (!(accessPoint instanceof V1_LakehouseAccessPoint)) {
-        continue;
-      }
-      const visitKey = `${dpPath}::${accessPoint.id}`;
-      if (visited.has(visitKey)) {
-        continue;
-      }
-      visited.add(visitKey);
-
-      const rawLambda = new RawLambda(
-        accessPoint.func.parameters,
-        accessPoint.func.body,
-      );
-      const accessors =
-        V1_resolveAccessorsFromRawLambda(rawLambda, graphManager, plugins) ??
-        [];
-      for (const accessor of accessors) {
-        if (accessor instanceof V1_IngestDefinitionAccessor) {
-          const specPath = accessor.path[0];
-          if (specPath) {
-            specs.add(specPath);
-          }
-        } else if (accessor instanceof V1_DataProductAccessor) {
-          const refDpPath = accessor.path[0];
-          const refApId = accessor.path[1];
-          if (!refDpPath || !refApId) {
-            continue;
-          }
-          if (refDpPath !== dpPath) {
-            continue;
-          }
-          const refApg = rootDataProduct.accessPointGroups.find((g) =>
-            g.accessPoints.some((ap) => ap.id === refApId),
-          );
-          if (refApg && !worklist.includes(refApg.id)) {
-            worklist.push(refApg.id);
-          }
-        }
-      }
-    }
-  };
-
-  while (worklist.length > 0) {
-    const apgId = guaranteeNonNullable(worklist.shift());
-    collectFromApg(apgId);
-  }
-
-  return specs;
-};
 
 export class ContractCreatedByUserDetails {
   readonly contractResultLite: V1_LiteDataContract;
@@ -531,158 +436,37 @@ export class EntitlementsDashboardState {
     return didToEnvType;
   }
 
-  async getUnverifiedIngestDefinitions(
+  async getContractIngestErrors(
     contractId: string,
     token: string | undefined,
-  ): Promise<string[]> {
-    const entitlementsStore = this.lakehouseEntitlementsStore;
-    const baseStore = entitlementsStore.marketplaceBaseStore;
-    const applicationStore = entitlementsStore.applicationStore;
+  ): Promise<ContractErrorLayer | undefined> {
+    const baseStore = this.lakehouseEntitlementsStore.marketplaceBaseStore;
     const plugins =
-      applicationStore.pluginManager.getPureProtocolProcessorPlugins();
-    const contractClient = entitlementsStore.lakehouseContractServerClient;
-
-    const PROD_ENV = 'prod';
-    const SDLC_DEPLOYMENT = 'alloy-git';
-
-    try {
-      const liteContract = await (async () => {
-        try {
-          const rawContractResponse = await contractClient.getDataContract(
-            contractId,
-            false,
-            token,
-          );
-          const dataContract = V1_deserializeDataContractResponse(
-            rawContractResponse,
-            plugins,
-          )[0]?.dataContract;
-          if (!dataContract) {
-            return undefined;
-          }
-          return V1_transformDataContractToLiteDatacontract(dataContract);
-        } catch (error) {
-          assertErrorThrown(error);
-          return undefined;
-        }
-      })();
-      if (!liteContract) {
-        return [];
-      }
-
-      const accessPointGroupId =
-        liteContract.resourceType === V1_ResourceType.ACCESS_POINT_GROUP
-          ? (liteContract.accessPointGroup ?? undefined)
-          : undefined;
-      if (!accessPointGroupId) {
-        return [];
-      }
-
-      const dpDetails = await (async () => {
-        try {
-          const raw = await contractClient.getDataProductByIdAndDID(
-            liteContract.resourceId,
-            liteContract.deploymentId,
-            token,
-          );
-          return V1_entitlementsDataProductDetailsResponseToDataProductDetails(
-            raw,
-          )[0];
-        } catch (error) {
-          assertErrorThrown(error);
-          return undefined;
-        }
-      })();
-      if (!dpDetails) {
-        return [];
-      }
-
-      if (!(dpDetails.origin instanceof V1_SdlcDeploymentDataProductOrigin)) {
-        return [];
-      }
-
-      const graphManager = new V1_PureGraphManager(
-        applicationStore.pluginManager,
-        applicationStore.logService,
-        baseStore.remoteEngine,
-      );
-      await graphManager.initialize(
-        {
-          env: applicationStore.config.env,
-          tabSize: DEFAULT_TAB_SIZE,
-          clientConfig: {
-            baseUrl: applicationStore.config.engineServerUrl,
-          },
-        },
-        { engine: baseStore.remoteEngine },
-      );
-
-      const v1DataProduct = await getDataProductFromDetails(
-        dpDetails,
-        graphManager,
+      this.lakehouseEntitlementsStore.applicationStore.pluginManager.getPureProtocolProcessorPlugins();
+    const apg = await getContractAPGCoordinates(
+      contractId,
+      baseStore.lakehouseContractServerClient,
+      plugins,
+      token,
+    );
+    if (!apg) {
+      return undefined;
+    }
+    const unverifiedIngestDefinitions =
+      await getUnverifiedIngestDefinitionsForAPG(
+        apg,
         baseStore,
-      );
-      if (!v1DataProduct) {
-        return [];
-      }
-
-      const specs = collectIngestSpecPathsFromOriginDp(
-        v1DataProduct,
-        accessPointGroupId,
-        graphManager,
         plugins,
+        () => baseStore.createInitializedGraphManager(),
+        token,
       );
-      if (specs.size === 0) {
-        return [];
-      }
-
-      const ingestEnvironment =
-        await baseStore.lakehouseDataProductService.getOrFetchEnvironmentForDID(
-          liteContract.deploymentId,
-          token,
-        );
-      const ingestServerUrl = ingestEnvironment?.ingestServerUrl;
-      if (ingestServerUrl === undefined) {
-        return [];
-      }
-
-      const sdlcOrigin = guaranteeType(
-        dpDetails.origin,
-        V1_SdlcDeploymentDataProductOrigin,
-      );
-      const gav = `${sdlcOrigin.group}~${sdlcOrigin.artifact}`;
-      const specsToVerify = Array.from(specs, (specPath) => ({
-        specPath,
-        urn: `urn:lakehouse:${PROD_ENV}:ingest:definition:${SDLC_DEPLOYMENT}:${gav}~${specPath}`,
-      }));
-
-      const ingestClient = baseStore.lakehouseIngestServerClient;
-      const settled = await Promise.all(
-        specsToVerify.map(async (entry) => {
-          try {
-            await ingestClient.getIngestDefinitionDetail(
-              entry.urn,
-              ingestServerUrl,
-              token,
-            );
-            return undefined;
-          } catch (error) {
-            assertErrorThrown(error);
-            if (
-              error instanceof NetworkClientError &&
-              error.response.status === HttpStatus.NOT_FOUND
-            ) {
-              return entry.specPath;
-            }
-            return undefined;
-          }
-        }),
-      );
-      return settled.filter(isNonNullable);
-    } catch (error) {
-      assertErrorThrown(error);
-      return [];
+    if (unverifiedIngestDefinitions.length === 0) {
+      return undefined;
     }
+    return {
+      title: `Ingest${unverifiedIngestDefinitions.length === 1 ? '' : 's'} Not Found:`,
+      errorItems: unverifiedIngestDefinitions,
+    };
   }
 
   async getContractSyncErrors(
@@ -695,39 +479,7 @@ export class EntitlementsDashboardState {
           contractId,
           token,
         )) as LakehouseContractSyncStatusResponse;
-
-      const status = response.status.toUpperCase();
-
-      if (status === ContractSyncStatus.NEVER_SYNCED) {
-        return { title: 'Sync Error: Contract Never Synced' };
-      }
-
-      if (status === ContractSyncStatus.NOT_FULLY_SYNCED) {
-        const unsyncedUsers =
-          response.unsyncedUsers?.map((user) => user.username) ?? [];
-        const unsyncedAccessPoints =
-          response.unsyncedAccessPoints?.map(
-            (accessPoint) => accessPoint.accessPointName,
-          ) ?? [];
-        const unsyncedTargetAccounts = response.unsyncedTargetAccounts ?? [];
-
-        const syncGroupingLayers: ContractErrorLayer[] = [
-          { title: 'Users', errorItems: unsyncedUsers },
-          { title: 'Target Accounts', errorItems: unsyncedTargetAccounts },
-          { title: 'Access Points', errorItems: unsyncedAccessPoints },
-        ].filter((layer) => layer.errorItems.length > 0);
-
-        if (syncGroupingLayers.length === 0) {
-          return undefined;
-        }
-
-        return {
-          title: 'Unsynced Entities',
-          childLayers: syncGroupingLayers,
-        };
-      }
-
-      return undefined;
+      return buildSyncErrorLayer(response);
     } catch (error) {
       assertErrorThrown(error);
       return undefined;
@@ -739,26 +491,13 @@ export class EntitlementsDashboardState {
     token: string | undefined,
     checkSyncStatus = false,
   ): Promise<ContractErrorLayer | undefined> {
-    const [unverifiedIngestDefinitions, syncErrorsLayer] = await Promise.all([
-      this.getUnverifiedIngestDefinitions(contractId, token),
+    const [ingestErrorsLayer, syncErrorsLayer] = await Promise.all([
+      this.getContractIngestErrors(contractId, token),
       checkSyncStatus
         ? this.getContractSyncErrors(contractId, token)
         : Promise.resolve(undefined),
     ]);
-
-    const childLayers: ContractErrorLayer[] = [
-      unverifiedIngestDefinitions.length > 0
-        ? {
-            title: `Ingest${unverifiedIngestDefinitions.length === 1 ? '' : 's'} Not Found`,
-            errorItems: unverifiedIngestDefinitions,
-          }
-        : undefined,
-      syncErrorsLayer,
-    ].filter(isNonNullable);
-
-    return childLayers.length > 0
-      ? { title: 'Contract Errors', childLayers }
-      : undefined;
+    return buildContractErrorsRoot([ingestErrorsLayer, syncErrorsLayer]);
   }
 
   filterByUserEnvironment(

package/src/utils/LakehouseUtils.tsx

@@ -18,23 +18,15 @@ import {
   type GraphManagerState,
   type V1_EntitlementsDataProductDetails,
   type V1_PureGraphManager,
-  type V1_DataProduct,
   V1_AdHocDeploymentDataProductOrigin,
-  V1_dataProductModelSchema,
   V1_SdlcDeploymentDataProductOrigin,
-  CORE_PURE_PATH,
 } from '@finos/legend-graph';
-import {
-  ActionState,
-  guaranteeNonNullable,
-  type PlainObject,
-} from '@finos/legend-shared';
+import { ActionState, type PlainObject } from '@finos/legend-shared';
 import {
   type ProjectVersionEntities,
   resolveVersion,
 } from '@finos/legend-server-depot';
 import type { Entity } from '@finos/legend-storage';
-import { deserialize } from 'serializr';
 import type { LegendMarketplaceBaseStore } from '../stores/LegendMarketplaceBaseStore.js';
 
 export const buildGraphForDataProduct = async (
@@ -77,74 +69,3 @@ export const buildGraphForDataProduct = async (
     );
   }
 };
-
-export const getDataProductFromDetails = async (
-  details: V1_EntitlementsDataProductDetails,
-  graphManager: V1_PureGraphManager,
-  marketplaceBaseStore: LegendMarketplaceBaseStore,
-): Promise<V1_DataProduct | undefined> => {
-  if (details.origin instanceof V1_SdlcDeploymentDataProductOrigin) {
-    const rawEntities =
-      (await marketplaceBaseStore.depotServerClient.getVersionEntities(
-        details.origin.group,
-        details.origin.artifact,
-        resolveVersion(details.origin.version),
-        CORE_PURE_PATH.DATA_PRODUCT,
-      )) as {
-        artifactId: string;
-        entity: Entity;
-        groupId: string;
-        versionId: string;
-        versionedEntity: boolean;
-      }[];
-    const entities = rawEntities.map((entity) =>
-      deserialize(
-        V1_dataProductModelSchema(
-          graphManager.pluginManager.getPureProtocolProcessorPlugins(),
-        ),
-        entity.entity.content,
-      ),
-    );
-    const matchingEntities = entities.filter(
-      (entity) => entity.name.toLowerCase() === details.id.toLowerCase(),
-    );
-    if (matchingEntities.length === 0) {
-      throw new Error(
-        `No data product found with name ${details.id} in project`,
-      );
-    } else if (matchingEntities.length > 1) {
-      throw new Error(
-        `Multiple data products found with name ${details.id} in project`,
-      );
-    }
-    return matchingEntities[0];
-  } else if (details.origin instanceof V1_AdHocDeploymentDataProductOrigin) {
-    const entities: Entity[] = await graphManager.pureCodeToEntities(
-      details.origin.definition,
-    );
-    const elements = entities
-      .filter((e) => e.classifierPath === CORE_PURE_PATH.DATA_PRODUCT)
-      .map((entity) =>
-        deserialize(
-          V1_dataProductModelSchema(
-            graphManager.pluginManager.getPureProtocolProcessorPlugins(),
-          ),
-          entity.content,
-        ),
-      );
-    const matchingEntities = elements.filter(
-      (element) => element.name.toLowerCase() === details.id.toLowerCase(),
-    );
-    if (matchingEntities.length > 1) {
-      throw new Error(
-        `Multiple data products found with name ${details.id} in deployed definition`,
-      );
-    }
-    return guaranteeNonNullable(
-      matchingEntities[0],
-      `No data product found with name ${details.id} in deployed definition`,
-    );
-  } else {
-    return undefined;
-  }
-};

package/tsconfig.json

@@ -63,6 +63,7 @@
     "./src/application/__test-utils__/LegendMarketplaceApplicationTestUtils.ts",
     "./src/application/extensions/Core_LegendMarketplace_LegendApplicationPlugin.ts",
     "./src/components/ProviderCard/orderProfileUtils.ts",
+    "./src/components/__test-utils__/TEST_DATA__ContractErrors.ts",
     "./src/components/__test-utils__/TEST_DATA__DataAPIs.ts",
     "./src/components/__test-utils__/TEST_DATA__LakehouseData.ts",
     "./src/components/__test-utils__/TEST_DATA__LakehouseSearchResultData.ts",