@finos/legend-graph 32.3.35 → 32.3.37

package/src/graph-manager/protocol/pure/v1/model/packageableElements/ingest/V1_IngestDefinition.ts

@@ -15,6 +15,7 @@
  */
 
 import type { V1_AppDirNode } from '../../../lakehouse/entitlements/V1_CoreEntitlements.js';
+import type { V1_RelationTypeColumn } from '../type/V1_RelationType.js';
 import { V1_INTERNAL__UnknownPackageableElement } from '../V1_INTERNAL__UnknownPackageableElement.js';
 import type { V1_PackageableElementVisitor } from '../V1_PackageableElement.js';
 
@@ -29,3 +30,18 @@ export class V1_IngestDefinition extends V1_INTERNAL__UnknownPackageableElement
     return visitor.visit_IngestDefinition(this);
   }
 }
+export interface V1_IngestDataset {
+  name: string;
+  primaryKey: string[];
+  source: {
+    _type: string;
+    schema: {
+      _type: string;
+      columns: V1_RelationTypeColumn[];
+    };
+  };
+}
+
+export interface V1_IngestDefinitionContent {
+  datasets?: V1_IngestDataset[];
+}

package/src/graph-manager/protocol/pure/v1/transformation/pureProtocol/V1_PureProtocolSerialization.ts

@@ -29,6 +29,7 @@ import {
   usingConstantValueSchema,
   UnsupportedOperationError,
   assertErrorThrown,
+  guaranteeNonNullable,
   usingModelSchema,
 } from '@finos/legend-shared';
 import { V1_PureModelContextData } from '../../model/context/V1_PureModelContextData.js';
@@ -101,29 +102,26 @@ export const V1_entitiesToPureModelContextData = async (
         TEMPORARY__entityPathIndex?.set(element.path, entity.path);
         return element;
       };
-      graph.elements = await Promise.all<V1_PackageableElement>(
-        entities.map(
-          (e) =>
-            new Promise((resolve, reject) =>
-              setTimeout(() => {
-                try {
-                  resolve(
-                    // NOTE: here we skip the check for classifier path, so there could be cases
-                    // where the classifier path is different from the actua element protocol path
-                    // we might need to do validation here. This can happen when the classifier
-                    // path is changed in the backend. If we are to check for this, we might consider
-                    // not throwing error but quitely print out warnings about elements that would not
-                    // be built.
-                    entityToElement(e),
-                  );
-                } catch (error) {
-                  assertErrorThrown(error);
-                  reject(error);
-                }
-              }, 0),
-            ),
-        ),
-      );
+      // Process entities in batches to avoid per-element setTimeout overhead.
+      // Each batch yields to the event loop to keep the UI responsive.
+      const DESERIALIZATION_BATCH_SIZE = 100;
+      const results: V1_PackageableElement[] = [];
+      for (let i = 0; i < entities.length; i += DESERIALIZATION_BATCH_SIZE) {
+        if (i > 0) {
+          await new Promise<void>((resolve) => setTimeout(resolve, 0));
+        }
+        const end = Math.min(i + DESERIALIZATION_BATCH_SIZE, entities.length);
+        for (let j = i; j < end; j++) {
+          // NOTE: here we skip the check for classifier path, so there could be cases
+          // where the classifier path is different from the actua element protocol path
+          // we might need to do validation here. This can happen when the classifier
+          // path is changed in the backend. If we are to check for this, we might consider
+          // not throwing error but quitely print out warnings about elements that would not
+          // be built.
+          results.push(entityToElement(guaranteeNonNullable(entities[j])));
+        }
+      }
+      graph.elements = results;
     }
   } catch (error) {
     assertErrorThrown(error);

package/src/graph-manager/protocol/pure/v1/transformation/pureProtocol/serializationHelpers/V1_EntitlementSerializationHelper.ts

@@ -55,8 +55,8 @@ import {
   list,
   optional,
   primitive,
+  raw,
   serialize,
-  SKIP,
 } from 'serializr';
 import {
   type V1_OrganizationalScope,
@@ -139,11 +139,18 @@ export const V1_EntitlementsDataProductModelSchema = createModelSchema(
 export const V1_AccessPointGroupReferenceModelSchema = createModelSchema(
   V1_AccessPointGroupReference,
   {
+    _type: usingConstantValueSchema(
+      V1_AccessPointGroupReferenceType.AccessPointGroupReference,
+    ),
     dataProduct: usingModelSchema(V1_EntitlementsDataProductModelSchema),
     accessPointGroup: primitive(),
   },
 );
 
+export const V1_DataBundleModelSchema = createModelSchema(V1_DataBundle, {
+  content: raw(),
+});
+
 export const V1_AdhocTeamModelSchema = createModelSchema(V1_AdhocTeam, {
   _type: usingConstantValueSchema(V1_OrganizationalScopeType.AdHocTeam),
   users: customListWithSchema(V1_UserModelSchema),
@@ -207,10 +214,29 @@ const V1_serializeOrganizationalScope = (
       return result;
     }
   }
-  throw new UnsupportedOperationError();
+  throw new UnsupportedOperationError(
+    `Can't serialize unsupported organizational scope type: ${organizationalScope.constructor.name}`,
+  );
 };
 
-const V1_deseralizeV1_ConsumerEntitlementResource = (
+const V1_seralizeConsumerEntitlementResource = (
+  consumerEntitlementResource: V1_ConsumerEntitlementResource,
+): PlainObject<V1_ConsumerEntitlementResource> => {
+  if (consumerEntitlementResource instanceof V1_AccessPointGroupReference) {
+    return serialize(
+      V1_AccessPointGroupReferenceModelSchema,
+      consumerEntitlementResource,
+    );
+  } else if (consumerEntitlementResource instanceof V1_DataBundle) {
+    return serialize(V1_DataBundleModelSchema, consumerEntitlementResource);
+  } else {
+    throw new UnsupportedOperationError(
+      `Can't serialize unsupported consumer entitlement resource type: ${consumerEntitlementResource.constructor.name}`,
+    );
+  }
+};
+
+const V1_deseralizeConsumerEntitlementResource = (
   json: PlainObject<V1_ConsumerEntitlementResource>,
 ): V1_ConsumerEntitlementResource => {
   switch (json._type) {
@@ -240,7 +266,10 @@ export const V1_dataContractModelSchema = (
     guid: primitive(),
     version: primitive(),
     state: primitive(),
-    resource: custom(() => SKIP, V1_deseralizeV1_ConsumerEntitlementResource),
+    resource: custom(
+      V1_seralizeConsumerEntitlementResource,
+      V1_deseralizeConsumerEntitlementResource,
+    ),
     members: optional(
       list(usingModelSchema(V1_contractUserMembershipModelSchema)),
     ),
@@ -585,7 +614,9 @@ const V1_serializeDataProductOrigin = (
   if (origin instanceof V1_SdlcDeploymentDataProductOrigin) {
     return serialize(V1_SdlcDeploymentDataProductOriginModelSchema, origin);
   }
-  throw new UnsupportedOperationError();
+  throw new UnsupportedOperationError(
+    `Can't serialize unsupported data product origin type: ${origin?.constructor.name}`,
+  );
 };
 
 export const V1_EntitlementsLakehouseEnvironmentModelSchema = createModelSchema(

package/src/index.ts

@@ -93,7 +93,11 @@ export { SnowflakeAppDeploymentConfiguration } from './graph/metamodel/pure/func
 export { SnowflakeM2MUdf } from './graph/metamodel/pure/packageableElements/function/SnowflakeM2MUdf.js';
 export { SnowflakeM2MUdfDeploymentConfiguration } from './graph/metamodel/pure/functionActivator/SnowflakeM2MUdfDeploymentConfiguration.js';
 export { INTERNAL__UnknownElement } from './graph/metamodel/pure/packageableElements/INTERNAL__UnknownElement.js';
-export { V1_IngestDefinition } from './graph-manager/protocol/pure/v1/model/packageableElements/ingest/V1_IngestDefinition.js';
+export {
+  V1_IngestDefinition,
+  type V1_IngestDataset,
+  type V1_IngestDefinitionContent,
+} from './graph-manager/protocol/pure/v1/model/packageableElements/ingest/V1_IngestDefinition.js';
 export {
   Ownership,
   DeploymentOwner,
@@ -374,8 +378,8 @@ export {
   V1_EntitlementsAccessPointModelSchema,
   V1_EntitlementsDataProductDetailsModelSchema,
   V1_EntitlementsDataProductDetailsResponseModelSchema,
-  V1_EntitlementsDataProductLiteModelSchema,
   V1_entitlementsDataProductDetailsResponseToDataProductDetails,
+  V1_EntitlementsDataProductLiteModelSchema,
   V1_EntitlementsDataProductModelSchema,
   V1_EntitlementsLakehouseEnvironmentModelSchema,
   V1_liteDataContractModelSchema,
@@ -384,6 +388,7 @@ export {
   V1_OrganizationalScopeType,
   V1_pendingTasksResponseModelSchema,
   V1_SdlcDeploymentDataProductOriginModelSchema,
+  V1_taskResponseModelSchema,
   V1_TaskStatusChangeResponseModelSchema,
   V1_terminalProvisionPayloadModelSchema,
 } from './graph-manager/protocol/pure/v1/transformation/pureProtocol/serializationHelpers/V1_EntitlementSerializationHelper.js';
@@ -402,6 +407,7 @@ export {
   V1_NativeModelExecutionContextInfo,
   V1_AccessPointImplementation,
   V1_AccessPointGroupInfo,
+  V1_DataProductInfo,
   V1_ModelAccessPointGroupInfo,
   V1_AccessPointGroupInfoType,
   V1_SampleQuery,