@finos/legend-graph 32.4.3 → 32.5.1

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/to/helpers/V1_ValueSpecificationBuilderHelper.ts

@@ -116,9 +116,21 @@ import {
 } from '../../../../helpers/V1_DomainHelper.js';
 import {
   ColSpec,
+  ColSpecArray,
+  ColSpecArrayInstance,
   ColSpecInstanceValue,
 } from '../../../../../../../../graph/metamodel/pure/valueSpecification/RelationValueSpecification.js';
 import { V1_ColSpec } from '../../../../model/valueSpecification/raw/classInstance/relation/V1_ColSpec.js';
+import { V1_ColSpecArray } from '../../../../model/valueSpecification/raw/classInstance/relation/V1_ColSpecArray.js';
+import {
+  V1_RelationStoreAccessor,
+  V1_IngestDefinitionAccessor,
+  V1_DataProductAccessor,
+} from '../../../../model/valueSpecification/raw/classInstance/relation/V1_RelationStoreAccessor.js';
+import { AccessorInstanceValue } from '../../../../../../../../graph/metamodel/pure/packageableElements/relation/Accessor.js';
+import { V1_createAccessorFromPackageableElement } from '../../../../helpers/V1_AccessorHelper.js';
+import { Database } from '../../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/Database.js';
+import { IngestDefinition } from '../../../../../../../../graph/metamodel/pure/packageableElements/ingest/IngestDefinition.js';
 
 const buildPrimtiveInstanceValue = (
   type: PRIMITIVE_TYPE,
@@ -471,9 +483,125 @@ export class V1_ValueSpecificationBuilder
         const protocol = guaranteeType(valueSpecification.value, V1_ColSpec);
         const value = new ColSpec();
         value.name = protocol.name;
+        value.type = protocol.type;
+        value.function1 = protocol.function1
+          ? protocol.function1.accept_ValueSpecificationVisitor(
+              new V1_ValueSpecificationBuilder(
+                this.context,
+                this.processingContext,
+                this.openVariables,
+              ),
+            )
+          : undefined;
+        value.function2 = protocol.function2
+          ? protocol.function2.accept_ValueSpecificationVisitor(
+              new V1_ValueSpecificationBuilder(
+                this.context,
+                this.processingContext,
+                this.openVariables,
+              ),
+            )
+          : undefined;
         instanceValue.values = [value];
         return instanceValue;
       }
+      case V1_ClassInstanceType.COL_SPEC_ARRAY: {
+        const instanceValue = new ColSpecArrayInstance(
+          Multiplicity.ONE,
+          undefined,
+        );
+        const protocol = guaranteeType(
+          valueSpecification.value,
+          V1_ColSpecArray,
+        );
+        const value = new ColSpecArray();
+        value.colSpecs = protocol.colSpecs.map((colSpec) => {
+          const colSpecInstance = new ColSpec();
+          colSpecInstance.name = colSpec.name;
+          colSpecInstance.type = colSpec.type;
+          colSpecInstance.function1 = colSpec.function1
+            ? colSpec.function1.accept_ValueSpecificationVisitor(
+                new V1_ValueSpecificationBuilder(
+                  this.context,
+                  this.processingContext,
+                  this.openVariables,
+                ),
+              )
+            : undefined;
+          colSpecInstance.function2 = colSpec.function2
+            ? colSpec.function2.accept_ValueSpecificationVisitor(
+                new V1_ValueSpecificationBuilder(
+                  this.context,
+                  this.processingContext,
+                  this.openVariables,
+                ),
+              )
+            : undefined;
+          return colSpecInstance;
+        });
+        instanceValue.values = [value];
+        return instanceValue;
+      }
+      case V1_ClassInstanceType.RELATION_STORE_ACCESSOR: {
+        const protocol = guaranteeType(
+          valueSpecification.value,
+          V1_RelationStoreAccessor,
+        );
+        const dbPath = guaranteeNonNullable(protocol.path[0]);
+        const schemaName = protocol.path[1];
+        const tableName = protocol.path[2];
+        const db = guaranteeType(
+          this.context.resolveElement(dbPath, false).value,
+          Database,
+        );
+        const accessor = guaranteeNonNullable(
+          V1_createAccessorFromPackageableElement(db, this.context, {
+            schemaName,
+            tableName,
+          }),
+          `Can't build accessor for database '${dbPath}'`,
+        );
+        const accessorInstanceValue = new AccessorInstanceValue();
+        accessorInstanceValue.values = [accessor];
+        accessorInstanceValue.genericType = GenericTypeExplicitReference.create(
+          new GenericType(accessor.relationType),
+        );
+        return accessorInstanceValue;
+      }
+      case V1_ClassInstanceType.INGEST_ACCESSOR: {
+        const protocol = guaranteeType(
+          valueSpecification.value,
+          V1_IngestDefinitionAccessor,
+        );
+        const ingestPath = guaranteeNonNullable(protocol.path[0]);
+        const datasetName = protocol.path[1];
+        const ingestDef = guaranteeType(
+          this.context.resolveElement(ingestPath, false).value,
+          IngestDefinition,
+        );
+        const accessor = guaranteeNonNullable(
+          V1_createAccessorFromPackageableElement(ingestDef, this.context, {
+            tableName: datasetName,
+          }),
+          `Can't build accessor for ingest definition '${ingestPath}'`,
+        );
+        const accessorInstanceValue = new AccessorInstanceValue();
+        accessorInstanceValue.values = [accessor];
+        accessorInstanceValue.genericType = GenericTypeExplicitReference.create(
+          new GenericType(accessor.relationType),
+        );
+        return accessorInstanceValue;
+      }
+      case V1_ClassInstanceType.DATA_PRODUCT_ACCESSOR: {
+        const protocol = guaranteeType(
+          valueSpecification.value,
+          V1_DataProductAccessor,
+        );
+        const _path = protocol.path;
+        throw new UnsupportedOperationError(
+          `Can't build accessor for data product with path '${_path.join('.')}': data product accessor building is not yet supported`,
+        );
+      }
       default: {
         const builders = this.context.extensions.plugins.flatMap(
           (plugin) => plugin.V1_getExtraClassInstanceValueBuilders?.() ?? [],
@@ -609,6 +737,7 @@ export function V1_processProperty(
     const _funcExp = new FunctionExpression(col.name);
     _funcExp.func = col;
     _funcExp.parametersValues = processedParameters;
+    _funcExp.genericType = col.genericType;
     return _funcExp;
   }
   throw new UnsupportedOperationError(

package/src/graph-manager/protocol/pure/v1/transformation/pureProtocol/serializationHelpers/V1_IngestSerializationHelper.ts

@@ -20,7 +20,13 @@ import {
   usingModelSchema,
   type PlainObject,
 } from '@finos/legend-shared';
-import { V1_IngestDefinition } from '../../../model/packageableElements/ingest/V1_IngestDefinition.js';
+import {
+  V1_IngestDataset,
+  V1_IngestDatasetSchema,
+  V1_IngestDatasetSource,
+  V1_IngestDefinition,
+  V1_IngestDefinitionContent,
+} from '../../../model/packageableElements/ingest/V1_IngestDefinition.js';
 import type { V1_PackageableElement } from '../../../model/packageableElements/V1_PackageableElement.js';
 import type { V1_AppDirNode } from '../../../lakehouse/entitlements/V1_CoreEntitlements.js';
 import { V1_AppDirNodeModelSchema } from './lakehouse/V1_CoreEntitlementsSerializationHelper.js';
@@ -28,9 +34,12 @@ import {
   createModelSchema,
   custom,
   deserialize,
+  list,
+  optional,
   primitive,
   serialize,
 } from 'serializr';
+import { V1_relationTypeColumnModelSchema } from './V1_TypeSerializationHelper.js';
 import {
   type V1_IngestEnvironment,
   V1_AWSSnowflakeIngestEnvironment,
@@ -167,3 +176,44 @@ export const V1_deserializeProducerEnvironment = (
       throw new Error(`Unknown V1_ProducerEnvironment type: ${json._type}`);
   }
 };
+
+// --------------------------------------------- Ingest Dataset Content ---------------------------------------------
+
+const V1_IngestDatasetSchemaModelSchema = createModelSchema(
+  V1_IngestDatasetSchema,
+  {
+    _type: primitive(),
+    columns: list(usingModelSchema(V1_relationTypeColumnModelSchema)),
+  },
+);
+
+const V1_IngestDatasetSourceModelSchema = createModelSchema(
+  V1_IngestDatasetSource,
+  {
+    _type: primitive(),
+    schema: usingModelSchema(V1_IngestDatasetSchemaModelSchema),
+  },
+);
+
+export const V1_IngestDatasetModelSchema = createModelSchema(V1_IngestDataset, {
+  name: primitive(),
+  primaryKey: list(primitive()),
+  source: usingModelSchema(V1_IngestDatasetSourceModelSchema),
+});
+
+export const V1_IngestDefinitionContentModelSchema = createModelSchema(
+  V1_IngestDefinitionContent,
+  {
+    datasets: optional(list(usingModelSchema(V1_IngestDatasetModelSchema))),
+  },
+);
+
+export const V1_deserializeIngestDefinitionContent = (
+  json: PlainObject<V1_IngestDefinitionContent>,
+): V1_IngestDefinitionContent =>
+  deserialize(V1_IngestDefinitionContentModelSchema, json);
+
+export const V1_serializeIngestDefinitionContent = (
+  content: V1_IngestDefinitionContent,
+): PlainObject<V1_IngestDefinitionContent> =>
+  serialize(V1_IngestDefinitionContentModelSchema, content);

package/src/graph-manager/protocol/pure/v1/transformation/pureProtocol/serializationHelpers/V1_ServiceSerializationHelper.ts

@@ -355,6 +355,7 @@ export const V1_serviceModelSchema = (
       (val) => V1_serializeServiceExecution(val),
       (val) => V1_deserializeServiceExecution(val),
     ),
+    mcpServer: optional(primitive()),
     name: primitive(),
     owners: list(primitive()),
     ownership: optionalCustom(
@@ -363,6 +364,9 @@ export const V1_serviceModelSchema = (
     ),
     package: primitive(),
     pattern: primitive(),
+    postValidations: customListWithSchema(V1_servicePostValidationModelSchema, {
+      INTERNAL__forceReturnEmptyInTest: true,
+    }),
     stereotypes: customListWithSchema(V1_stereotypePtrModelSchema, {
       INTERNAL__forceReturnEmptyInTest: true,
     }),
@@ -380,8 +384,5 @@ export const V1_serviceModelSchema = (
         INTERNAL__forceReturnEmptyInTest: true,
       },
     ),
-    postValidations: customListWithSchema(V1_servicePostValidationModelSchema, {
-      INTERNAL__forceReturnEmptyInTest: true,
-    }),
-    mcpServer: optional(primitive()),
+    title: optional(primitive()),
   });

package/src/graph-manager/protocol/pure/v1/transformation/pureProtocol/serializationHelpers/V1_TypeSerializationHelper.ts

@@ -91,7 +91,7 @@ const packageableTypeModelSchema = createModelSchema(V1_PackageableType, {
   fullPath: primitive(),
 });
 
-const V1_relationTypeColumnModelSchema = createModelSchema(
+export const V1_relationTypeColumnModelSchema = createModelSchema(
   V1_RelationTypeColumn,
   {
     genericType: custom(

package/src/graph-manager/protocol/pure/v1/transformation/pureProtocol/serializationHelpers/V1_ValueSpecificationSerializer.ts

@@ -69,6 +69,7 @@ import { V1_ColSpecArray } from '../../../model/valueSpecification/raw/classInst
 import { V1_ColSpec } from '../../../model/valueSpecification/raw/classInstance/relation/V1_ColSpec.js';
 import {
   V1_DataProductAccessor,
+  V1_IngestDefinitionAccessor,
   V1_RelationStoreAccessor,
   V1_SQLAccessor,
 } from '../../../model/valueSpecification/raw/classInstance/relation/V1_RelationStoreAccessor.js';
@@ -773,6 +774,14 @@ const relationStoreAccessorModelSchema = createModelSchema(
   },
 );
 
+const ingestAccessorModelSchema = createModelSchema(
+  V1_IngestDefinitionAccessor,
+  {
+    path: list(primitive()),
+    metadata: optional(primitive()),
+  },
+);
+
 const dataProductAccessorModelSchema = createModelSchema(
   V1_DataProductAccessor,
   {
@@ -789,8 +798,22 @@ const colSpecModelSchema = (
   plugins: PureProtocolProcessorPlugin[],
 ): ModelSchema<V1_ColSpec> =>
   createModelSchema(V1_ColSpec, {
-    function1: optional(usingModelSchema(V1_lambdaModelSchema(plugins))),
-    function2: optional(usingModelSchema(V1_lambdaModelSchema(plugins))),
+    function1: optional(
+      custom(
+        (val) => {
+          return val ? V1_serializeValueSpecification(val, plugins) : SKIP;
+        },
+        (val) => V1_deserializeValueSpecification(val, plugins),
+      ),
+    ),
+    function2: optional(
+      custom(
+        (val) => {
+          return val ? V1_serializeValueSpecification(val, plugins) : SKIP;
+        },
+        (val) => V1_deserializeValueSpecification(val, plugins),
+      ),
+    ),
     name: primitive(),
     type: optional(primitive()),
   });
@@ -856,8 +879,9 @@ export function V1_deserializeClassInstanceValue(
     case V1_ClassInstanceType.COL_SPEC_ARRAY:
       return deserialize(colSpecArrayModelSchema(plugins), json);
     case V1_ClassInstanceType.RELATION_STORE_ACCESSOR:
-    case V1_ClassInstanceType.INGEST_ACCESSOR:
       return deserialize(relationStoreAccessorModelSchema, json);
+    case V1_ClassInstanceType.INGEST_ACCESSOR:
+      return deserialize(ingestAccessorModelSchema, json);
     case V1_ClassInstanceType.DATA_PRODUCT_ACCESSOR:
       return deserialize(dataProductAccessorModelSchema, json);
     case V1_ClassInstanceType.SQL_ACCESSOR:
@@ -920,6 +944,8 @@ export function V1_serializeClassInstanceValue(
     return serialize(dataProductAccessorModelSchema, protocol);
   } else if (protocol instanceof V1_SQLAccessor) {
     return serialize(sqlAccessorModelSchema, protocol);
+  } else if (protocol instanceof V1_IngestDefinitionAccessor) {
+    return serialize(ingestAccessorModelSchema, protocol);
   }
   const serializers = plugins.flatMap(
     (plugin) =>

package/src/index.ts

@@ -97,8 +97,10 @@ export { SnowflakeM2MUdfDeploymentConfiguration } from './graph/metamodel/pure/f
 export { INTERNAL__UnknownElement } from './graph/metamodel/pure/packageableElements/INTERNAL__UnknownElement.js';
 export {
   V1_IngestDefinition,
-  type V1_IngestDataset,
-  type V1_IngestDefinitionContent,
+  V1_IngestDatasetSchema,
+  V1_IngestDatasetSource,
+  V1_IngestDataset,
+  V1_IngestDefinitionContent,
 } from './graph-manager/protocol/pure/v1/model/packageableElements/ingest/V1_IngestDefinition.js';
 export {
   Ownership,
@@ -376,6 +378,10 @@ export { V1_transformDataContractToLiteDatacontract } from './graph-manager/prot
 export {
   V1_deserializeIngestEnvironment,
   V1_deserializeProducerEnvironment,
+  V1_IngestDatasetModelSchema,
+  V1_IngestDefinitionContentModelSchema,
+  V1_deserializeIngestDefinitionContent,
+  V1_serializeIngestDefinitionContent,
 } from './graph-manager/protocol/pure/v1/transformation/pureProtocol/serializationHelpers/V1_IngestSerializationHelper.js';
 export * from './graph-manager/protocol/pure/v1/model/lineage/V1_Lineage.js';
 
@@ -847,7 +853,17 @@ export {
   RelationType,
   RelationColumn,
 } from './graph/metamodel/pure/packageableElements/relation/RelationType.js';
+export {
+  Accessor,
+  DataProductAccessor,
+  IngestionAccessor,
+  RelationalStoreAccessor,
+  AccessorInstanceValue,
+  type AccessorOwner,
+} from './graph/metamodel/pure/packageableElements/relation/Accessor.js';
+export { V1_createAccessorFromPackageableElement as createAccessorFromPackageableElement } from './graph-manager/protocol/pure/v1/helpers/V1_AccessorHelper.js';
 export { Store } from './graph/metamodel/pure/packageableElements/store/Store.js';
+export { Database } from './graph/metamodel/pure/packageableElements/store/relational/model/Database.js';
 export { Mapping } from './graph/metamodel/pure/packageableElements/mapping/Mapping.js';
 export {
   Runtime,

package/tsconfig.json

@@ -249,6 +249,7 @@
     "./src/graph/metamodel/pure/packageableElements/mapping/xStore/XStoreAssociationImplementation.ts",
     "./src/graph/metamodel/pure/packageableElements/mapping/xStore/XStorePropertyFetchDetails.ts",
     "./src/graph/metamodel/pure/packageableElements/mapping/xStore/XStorePropertyMapping.ts",
+    "./src/graph/metamodel/pure/packageableElements/relation/Accessor.ts",
     "./src/graph/metamodel/pure/packageableElements/relation/Relation.ts",
     "./src/graph/metamodel/pure/packageableElements/relation/RelationType.ts",
     "./src/graph/metamodel/pure/packageableElements/runtime/PackageableRuntime.ts",
@@ -511,6 +512,7 @@
     "./src/graph-manager/protocol/pure/v1/engine/test/V1_DebugTestsResult.ts",
     "./src/graph-manager/protocol/pure/v1/engine/test/V1_RunTestsInput.ts",
     "./src/graph-manager/protocol/pure/v1/engine/test/V1_RunTestsResult.ts",
+    "./src/graph-manager/protocol/pure/v1/helpers/V1_AccessorHelper.ts",
     "./src/graph-manager/protocol/pure/v1/helpers/V1_DataProductHelper.ts",
     "./src/graph-manager/protocol/pure/v1/helpers/V1_DomainHelper.ts",
     "./src/graph-manager/protocol/pure/v1/helpers/V1_DomainObserverHelper.ts",