@finos/legend-graph 31.1.2 → 31.1.4

package/src/graph-manager/AbstractPureGraphManager.ts

@@ -98,9 +98,11 @@ import type {
   SubtypeInfo,
 } from './action/protocol/ProtocolInfo.js';
 import type { FunctionActivatorConfiguration } from './action/functionActivator/FunctionActivatorConfiguration.js';
+import type { RelationalDatabaseTypeConfiguration } from './action/relational/RelationalDatabaseTypeConfiguration.js';
 import type { FunctionActivator } from '../graph/metamodel/pure/packageableElements/function/FunctionActivator.js';
 import type { RelationalDatabaseConnection } from '../STO_Relational_Exports.js';
 import type { ArtifactGenerationExtensionResult } from './action/generation/ArtifactGenerationExtensionResult.js';
+import type { TestDataGenerationResult } from '../graph/metamodel/pure/packageableElements/service/TestGenerationResult.js';
 
 export interface TEMPORARY__EngineSetupConfig {
   env: string;
@@ -412,6 +414,15 @@ export abstract class AbstractPureGraphManager {
     graph: PureModel,
   ): Promise<Entity[]>;
 
+  // --------------------------------------------- Test Data Generation ---------------------------------------------
+
+  abstract generateTestData(
+    query: RawLambda,
+    mapping: string,
+    runtime: string,
+    graph: PureModel,
+  ): Promise<TestDataGenerationResult>;
+
   // ------------------------------------------- External Format ----------------------------------
 
   abstract getAvailableExternalFormatsDescriptions(): Promise<
@@ -581,6 +592,10 @@ export abstract class AbstractPureGraphManager {
     graph: PureModel,
   ): Promise<Entity[]>;
 
+  abstract getAvailableRelationalDatabaseTypeConfigurations(): Promise<
+    RelationalDatabaseTypeConfiguration[] | undefined
+  >;
+
   // ------------------------------------------- Service -------------------------------------------
   /**
    * @modularize

package/src/graph-manager/action/execution/ExecutionResult.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { uuid } from '@finos/legend-shared';
+import { guaranteeNonNullable, uuid } from '@finos/legend-shared';
 
 // Core
 export enum BuilderType {
@@ -138,3 +138,19 @@ export class ClassExecutionResult extends ExecutionResult {
   override builder = new ClassBuilder(BuilderType.CLASS_BUILDER);
   objects!: object;
 }
+
+export const getTDSRowRankByColumnInAsc = (
+  a: TDSRow,
+  b: TDSRow,
+  colIndex: number,
+): number => {
+  const a1 =
+    a.values[colIndex] === null || a.values[colIndex] === undefined
+      ? -Infinity
+      : a.values[colIndex];
+  const b1 =
+    b.values[colIndex] === null || b.values[colIndex] === undefined
+      ? -Infinity
+      : b.values[colIndex];
+  return Number(guaranteeNonNullable(a1)) - Number(guaranteeNonNullable(b1));
+};

package/src/graph-manager/action/relational/RelationalDatabaseTypeConfiguration.ts

@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export class RelationalDatabaseTypeConfiguration {
+  type!: string;
+  compatibleAuthStrategies: string[] = [];
+  compatibleDataSources: string[] = [];
+}

package/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts

@@ -36,6 +36,8 @@ import {
   isNonNullable,
   filterByType,
   isString,
+  assertNonEmptyString,
+  uniq,
 } from '@finos/legend-shared';
 import type { TEMPORARY__AbstractEngineConfig } from '../../../../graph-manager/action/TEMPORARY__AbstractEngineConfig.js';
 import {
@@ -304,6 +306,10 @@ import {
   V1_buildArtifactsByExtensionElement,
 } from './engine/generation/V1_ArtifactGenerationExtensionApi.js';
 import type { V1_RawValueSpecification } from './model/rawValueSpecification/V1_RawValueSpecification.js';
+import { V1_TestDataGenerationInput } from './engine/service/V1_TestDataGenerationInput.js';
+import type { TestDataGenerationResult } from '../../../../graph/metamodel/pure/packageableElements/service/TestGenerationResult.js';
+import { V1_buildTestDataGenerationResult } from './engine/service/V1_TestDataGenerationResult.js';
+import { RelationalDatabaseTypeConfiguration } from '../../../action/relational/RelationalDatabaseTypeConfiguration.js';
 
 class V1_PureModelContextDataIndex {
   elements: V1_PackageableElement[] = [];
@@ -2060,6 +2066,37 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     return this.pureModelContextDataToEntities(generatedModel);
   }
 
+  // --------------------------------------------- Test Data Generation ---------------------------------------------
+
+  async generateTestData(
+    query: RawLambda,
+    mapping: string,
+    runtime: string,
+    graph: PureModel,
+  ): Promise<TestDataGenerationResult> {
+    const testDataGenerationInput = new V1_TestDataGenerationInput();
+    testDataGenerationInput.query = V1_transformRawLambda(
+      query,
+      new V1_GraphTransformerContextBuilder(
+        this.pluginManager.getPureProtocolProcessorPlugins(),
+      ).build(),
+    );
+    testDataGenerationInput.mapping = mapping;
+    testDataGenerationInput.runtime = runtime;
+    const graphData = this.getFullGraphModelContext(
+      graph,
+      V1_PureGraphManager.DEV_PROTOCOL_VERSION,
+    );
+    testDataGenerationInput.model = graphData;
+    return V1_buildTestDataGenerationResult(
+      await this.engine.generateTestData(
+        testDataGenerationInput,
+        this.pluginManager.getPureProtocolProcessorPlugins(),
+      ),
+      this.getBuilderContext(graph, graph, new V1_Mapping()), // use a dummy V1_PackageableElement to generate V1_GraphBuilderContext which is used in V1_buildEmbeddedData()
+    );
+  }
+
   // ------------------------------------------- Test  -------------------------------------------
 
   async runTests(
@@ -3021,6 +3058,57 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     return this.pureModelContextDataToEntities(generatedModel);
   }
 
+  async getAvailableRelationalDatabaseTypeConfigurations(): Promise<
+    RelationalDatabaseTypeConfiguration[] | undefined
+  > {
+    try {
+      const configs =
+        await this.engine.getAvailableRelationalDatabaseTypeConfigurations();
+      return uniq(configs.map((e) => e.dbType)).map((type) => {
+        assertNonEmptyString(
+          type,
+          'Property dbType missing in database authentication flow',
+        );
+        const compatibles = configs.filter((f) => f.dbType === type);
+        const config = new RelationalDatabaseTypeConfiguration();
+        config.type = type;
+        config.compatibleAuthStrategies = uniq(
+          compatibles
+            .map((aFlow) => aFlow.authStrategy)
+            .flat()
+            .map((e) => {
+              assertNonEmptyString(
+                e,
+                'Property authStrategy missing in database authentication flow',
+              );
+              return e;
+            }),
+        );
+        config.compatibleDataSources = uniq(
+          compatibles
+            .map((aFlow) => aFlow.dataSource)
+            .flat()
+            .map((dataSource) => {
+              assertNonEmptyString(
+                dataSource,
+                'Property dataSource missing in database authentication flow',
+              );
+              return dataSource;
+            }),
+        );
+        return config;
+      });
+    } catch (error) {
+      assertErrorThrown(error);
+      assertErrorThrown(error);
+      this.logService.error(
+        LogEvent.create(GRAPH_MANAGER_EVENT.RELATIONAL_CONNECTION),
+        error,
+      );
+      return undefined;
+    }
+  }
+
   // --------------------------------------------- Service ---------------------------------------------
 
   async registerService(

package/src/graph-manager/protocol/pure/v1/engine/V1_Engine.ts

@@ -136,6 +136,12 @@ import {
   V1_ArtifactGenerationExtensionInput,
 } from './generation/V1_ArtifactGenerationExtensionApi.js';
 import { V1_DatabaseToModelGenerationInput } from './relational/V1_DatabaseToModelGenerationInput.js';
+import { V1_TestDataGenerationInput } from './service/V1_TestDataGenerationInput.js';
+import {
+  type V1_TestDataGenerationResult,
+  V1_testDataGenerationResultModelSchema,
+} from './service/V1_TestDataGenerationResult.js';
+import { V1_RelationalConnectionBuilder } from './relational/V1_RelationalConnectionBuilder.js';
 
 class V1_EngineConfig extends TEMPORARY__AbstractEngineConfig {
   private engine: V1_Engine;
@@ -719,6 +725,20 @@ export class V1_Engine {
     );
   }
 
+  // --------------------------------------------- Test Data Generation ---------------------------------------------
+
+  async generateTestData(
+    input: V1_TestDataGenerationInput,
+    plugins: PureProtocolProcessorPlugin[],
+  ): Promise<V1_TestDataGenerationResult> {
+    return deserialize(
+      V1_testDataGenerationResultModelSchema(plugins),
+      await this.engineServerClient.generateTestData(
+        V1_TestDataGenerationInput.serialization.toJson(input),
+      ),
+    );
+  }
+
   // ------------------------------------------- File Generation -------------------------------------------
 
   async getAvailableGenerationConfigurationDescriptions(): Promise<
@@ -1041,4 +1061,20 @@ export class V1_Engine {
       throw error;
     }
   }
+
+  async getAvailableRelationalDatabaseTypeConfigurations(): Promise<
+    V1_RelationalConnectionBuilder[]
+  > {
+    try {
+      return (
+        await this.engineServerClient.getAvailableRelationalDatabaseTypeConfigurations()
+      ).map((dbTypeToDataSourceAndAuthMap) =>
+        V1_RelationalConnectionBuilder.serialization.fromJson(
+          dbTypeToDataSourceAndAuthMap,
+        ),
+      );
+    } catch (error) {
+      return [];
+    }
+  }
 }

package/src/graph-manager/protocol/pure/v1/engine/V1_EngineServerClient.ts

@@ -78,12 +78,16 @@ import type {
   V1_ArtifactGenerationExtensionOutput,
 } from './generation/V1_ArtifactGenerationExtensionApi.js';
 import type { V1_DatabaseToModelGenerationInput } from './relational/V1_DatabaseToModelGenerationInput.js';
+import type { V1_TestDataGenerationInput } from './service/V1_TestDataGenerationInput.js';
+import type { V1_TestDataGenerationResult } from './service/V1_TestDataGenerationResult.js';
+import type { V1_RelationalConnectionBuilder } from './relational/V1_RelationalConnectionBuilder.js';
 
 enum CORE_ENGINE_ACTIVITY_TRACE {
   GRAMMAR_TO_JSON = 'transform Pure code to protocol',
   JSON_TO_GRAMMAR = 'transform protocol to Pure code',
 
   DATABASE_TO_MODELS = 'generate models from database',
+  TEST_DATA_GENERATION = 'generate test data',
 
   EXTERNAL_FORMAT_TO_PROTOCOL = 'transform external format code to protocol',
   GENERATE_FILE = 'generate file',
@@ -502,6 +506,20 @@ export class V1_EngineServerClient extends AbstractServerClient {
     PlainObject<V1_GenerationConfigurationDescription>[]
   > => this.get(`${this._pure()}/schemaGeneration/availableGenerations`);
 
+  // --------------------------------------------- Test Data Generation ---------------------------------------------
+
+  _testDataGeneration = (): string => `${this._pure()}/testData/generation`;
+
+  generateTestData(
+    input: PlainObject<V1_TestDataGenerationInput>,
+  ): Promise<PlainObject<V1_TestDataGenerationResult>> {
+    return this.postWithTracing(
+      this.getTraceData(CORE_ENGINE_ACTIVITY_TRACE.TEST_DATA_GENERATION),
+      `${this._testDataGeneration()}/DONOTUSE_generateTestData`,
+      this.debugPayload(input, CORE_ENGINE_ACTIVITY_TRACE.TEST_DATA_GENERATION),
+    );
+  }
+
   // ------------------------------------------- Compile -------------------------------------------
 
   compile = (
@@ -791,6 +809,13 @@ export class V1_EngineServerClient extends AbstractServerClient {
     );
   }
 
+  getAvailableRelationalDatabaseTypeConfigurations = (): Promise<
+    PlainObject<V1_RelationalConnectionBuilder>[]
+  > =>
+    this.get(
+      `${this._relationalElement()}/connection/supportedDbAuthenticationFlows`,
+    );
+
   // ------------------------------------------- Service -------------------------------------------
 
   _service = (serviceServerUrl?: string): string =>

package/src/graph-manager/protocol/pure/v1/engine/relational/V1_RelationalConnectionBuilder.ts

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { primitive, createModelSchema } from 'serializr';
+import { SerializationFactory } from '@finos/legend-shared';
+
+export class V1_RelationalConnectionBuilder {
+  dbType!: string;
+  authStrategy!: string;
+  dataSource!: string;
+
+  static readonly serialization = new SerializationFactory(
+    createModelSchema(V1_RelationalConnectionBuilder, {
+      dbType: primitive(),
+      authStrategy: primitive(),
+      dataSource: primitive(),
+    }),
+  );
+}

package/src/graph-manager/protocol/pure/v1/engine/service/V1_TestDataGenerationInput.ts

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { SerializationFactory, usingModelSchema } from '@finos/legend-shared';
+import { createModelSchema, optional, primitive } from 'serializr';
+import type { V1_PureModelContext } from '../../model/context/V1_PureModelContext.js';
+import type { V1_RawLambda } from '../../model/rawValueSpecification/V1_RawLambda.js';
+import { V1_pureModelContextPropSchema } from '../../transformation/pureProtocol/V1_PureProtocolSerialization.js';
+import { V1_rawLambdaModelSchema } from '../../transformation/pureProtocol/serializationHelpers/V1_RawValueSpecificationSerializationHelper.js';
+
+export class V1_TestDataGenerationInput {
+  clientVersion: string | undefined;
+  /**
+   * Studio does not process value specification, they are left in raw JSON form
+   *
+   * Should be optional. For now, it's mandatory.
+   * @discrepancy model
+   */
+  query!: V1_RawLambda;
+  mapping!: string;
+  runtime!: string;
+  model!: V1_PureModelContext;
+
+  static readonly serialization = new SerializationFactory(
+    createModelSchema(V1_TestDataGenerationInput, {
+      clientVersion: optional(primitive()),
+      query: usingModelSchema(V1_rawLambdaModelSchema),
+      mapping: primitive(),
+      model: V1_pureModelContextPropSchema,
+      runtime: primitive(),
+    }),
+  );
+}

package/src/graph-manager/protocol/pure/v1/engine/service/V1_TestDataGenerationResult.ts

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createModelSchema, list, custom, type ModelSchema } from 'serializr';
+import type { V1_EmbeddedData } from '../../model/data/V1_EmbeddedData.js';
+import {
+  V1_deserializeEmbeddedDataType,
+  V1_serializeEmbeddedDataType,
+} from '../../transformation/pureProtocol/serializationHelpers/V1_DataElementSerializationHelper.js';
+import type { PureProtocolProcessorPlugin } from '../../../PureProtocolProcessorPlugin.js';
+import type { V1_GraphBuilderContext } from '../../transformation/pureGraph/to/V1_GraphBuilderContext.js';
+import { TestDataGenerationResult } from '../../../../../../graph/metamodel/pure/packageableElements/service/TestGenerationResult.js';
+import { V1_buildEmbeddedData } from '../../transformation/pureGraph/to/helpers/V1_DataElementBuilderHelper.js';
+
+export class V1_TestDataGenerationResult {
+  data: V1_EmbeddedData[] = [];
+}
+
+export const V1_testDataGenerationResultModelSchema = (
+  plugins: PureProtocolProcessorPlugin[],
+): ModelSchema<V1_TestDataGenerationResult> =>
+  createModelSchema(V1_TestDataGenerationResult, {
+    data: list(
+      custom(
+        (val) => V1_serializeEmbeddedDataType(val, plugins),
+        (val) => V1_deserializeEmbeddedDataType(val, plugins),
+      ),
+    ),
+  });
+
+export const V1_buildTestDataGenerationResult = (
+  protocol: V1_TestDataGenerationResult,
+  context: V1_GraphBuilderContext,
+): TestDataGenerationResult =>
+  new TestDataGenerationResult(
+    protocol.data.map((p) => V1_buildEmbeddedData(p, context)),
+  );

package/src/index.ts

@@ -70,6 +70,7 @@ export {
   TagExplicitReference,
 } from './graph/metamodel/pure/packageableElements/domain/TagReference.js';
 export * from './graph/metamodel/pure/packageableElements/service/ExecutionEnvironmentInstance.js';
+export { TestDataGenerationResult } from './graph/metamodel/pure/packageableElements/service/TestGenerationResult.js';
 export { FunctionActivator } from './graph/metamodel/pure/packageableElements/function/FunctionActivator.js';
 export { INTERNAL__UnknownFunctionActivator } from './graph/metamodel/pure/packageableElements/function/INTERNAL__UnknownFunctionActivator.js';
 
@@ -220,6 +221,7 @@ export {
   TDSExecutionResult as TDSExecutionResult,
   RawExecutionResult,
   EXECUTION_SERIALIZATION_FORMAT,
+  getTDSRowRankByColumnInAsc,
 } from './graph-manager/action/execution/ExecutionResult.js';
 export { ExternalFormatDescription } from './graph-manager/action/externalFormat/ExternalFormatDescription.js';
 export * from './graph-manager/action/generation/ArtifactGenerationExtensionResult.js';
@@ -247,6 +249,7 @@ export { SourceInformation } from './graph-manager/action/SourceInformation.js';
 export * from './graph-manager/protocol/pure/PureProtocolProcessorPlugin.js';
 export * from './graph-manager/protocol/pure/extensions/DSL_ExternalFormat_PureProtocolProcessorPlugin.js';
 export * from './graph-manager/action/functionActivator/FunctionActivatorConfiguration.js';
+export * from './graph-manager/action/relational/RelationalDatabaseTypeConfiguration.js';
 
 // --------------------------------------------- TRANSFORMATION --------------------------------------------------
 

package/tsconfig.json

@@ -196,6 +196,7 @@
     "./src/graph/metamodel/pure/packageableElements/service/ServiceTest.ts",
     "./src/graph/metamodel/pure/packageableElements/service/ServiceTestData.ts",
     "./src/graph/metamodel/pure/packageableElements/service/ServiceTestSuite.ts",
+    "./src/graph/metamodel/pure/packageableElements/service/TestGenerationResult.ts",
     "./src/graph/metamodel/pure/packageableElements/store/INTERNAL__UnknownStore.ts",
     "./src/graph/metamodel/pure/packageableElements/store/Store.ts",
     "./src/graph/metamodel/pure/packageableElements/store/flatData/connection/FlatDataConnection.ts",
@@ -326,6 +327,7 @@
     "./src/graph-manager/action/protocol/ProtocolInfo.ts",
     "./src/graph-manager/action/query/Query.ts",
     "./src/graph-manager/action/query/QuerySearchSpecification.ts",
+    "./src/graph-manager/action/relational/RelationalDatabaseTypeConfiguration.ts",
     "./src/graph-manager/action/service/DEPRECATED__ServiceTestResult.ts",
     "./src/graph-manager/action/service/ServiceExecutionMode.ts",
     "./src/graph-manager/action/service/ServiceRegistrationResult.ts",
@@ -389,10 +391,13 @@
     "./src/graph-manager/protocol/pure/v1/engine/query/V1_Query.ts",
     "./src/graph-manager/protocol/pure/v1/engine/query/V1_QuerySearchSpecification.ts",
     "./src/graph-manager/protocol/pure/v1/engine/relational/V1_DatabaseToModelGenerationInput.ts",
+    "./src/graph-manager/protocol/pure/v1/engine/relational/V1_RelationalConnectionBuilder.ts",
     "./src/graph-manager/protocol/pure/v1/engine/service/V1_DEPRECATED__ServiceTestResult.ts",
     "./src/graph-manager/protocol/pure/v1/engine/service/V1_ServiceConfiguration.ts",
     "./src/graph-manager/protocol/pure/v1/engine/service/V1_ServiceRegistrationResult.ts",
     "./src/graph-manager/protocol/pure/v1/engine/service/V1_ServiceStorage.ts",
+    "./src/graph-manager/protocol/pure/v1/engine/service/V1_TestDataGenerationInput.ts",
+    "./src/graph-manager/protocol/pure/v1/engine/service/V1_TestDataGenerationResult.ts",
     "./src/graph-manager/protocol/pure/v1/engine/test/V1_RunTestsInput.ts",
     "./src/graph-manager/protocol/pure/v1/engine/test/V1_RunTestsResult.ts",
     "./src/graph-manager/protocol/pure/v1/helpers/V1_DomainHelper.ts",