@finos/legend-graph 22.3.0 → 23.0.1

package/src/graphManager/AbstractPureGraphManager.ts

@@ -52,11 +52,15 @@ import {
   type TracerService,
 } from '@finos/legend-shared';
 import type { LightQuery, Query, QueryInfo } from './action/query/Query.js';
-import type { Entity, LegendSDLC } from '@finos/legend-storage';
+import type {
+  EntitiesWithOrigin,
+  Entity,
+  GraphDataOrigin,
+} from '@finos/legend-storage';
 import type { QuerySearchSpecification } from './action/query/QuerySearchSpecification.js';
 import type { ExternalFormatDescription } from './action/externalFormat/ExternalFormatDescription.js';
 import type { ConfigurationProperty } from '../graph/metamodel/pure/packageableElements/fileGeneration/ConfigurationProperty.js';
-import type { GraphBuilderReport } from './GraphBuilderReport.js';
+import type { GraphManagerOperationReport } from './GraphManagerMetrics.js';
 import type { RunTestsTestableInput } from '../graph/metamodel/pure/test/result/RunTestsTestableInput.js';
 import type { TestResult } from '../graph/metamodel/pure/test/result/TestResult.js';
 import type { GraphManagerPluginManager } from './GraphManagerPluginManager.js';
@@ -101,7 +105,7 @@ export interface GraphBuilderOptions {
   /**
    * This ties a graph to an sdlc pointer. Meaning the graph is immutable and tied to a specific `versioned` SDLC.
    */
-  sdlc?: LegendSDLC | undefined;
+  origin?: GraphDataOrigin | undefined;
 }
 
 export interface ExecutionOptions {
@@ -187,7 +191,8 @@ export abstract class AbstractPureGraphManager {
     systemModel: SystemModel,
     buildState: ActionState,
     options?: GraphBuilderOptions,
-  ): Promise<GraphBuilderReport>;
+    report?: GraphManagerOperationReport,
+  ): Promise<void>;
 
   /**
    * Build immutable models which holds dependencies.
@@ -201,10 +206,11 @@ export abstract class AbstractPureGraphManager {
     coreModel: CoreModel,
     systemModel: SystemModel,
     dependencyManager: DependencyManager,
-    dependencyEntitiesIndex: Map<string, Entity[]>,
+    dependencyEntitiesIndex: Map<string, EntitiesWithOrigin>,
     buildState: ActionState,
     options?: GraphBuilderOptions,
-  ): Promise<GraphBuilderReport>;
+    report?: GraphManagerOperationReport,
+  ): Promise<void>;
 
   /**
    * Process entities and build the main graph.
@@ -214,7 +220,8 @@ export abstract class AbstractPureGraphManager {
     entities: Entity[],
     buildState: ActionState,
     options?: GraphBuilderOptions,
-  ): Promise<GraphBuilderReport>;
+    report?: GraphManagerOperationReport,
+  ): Promise<void>;
 
   /**
    * Process entities and build the light graph.
@@ -224,14 +231,16 @@ export abstract class AbstractPureGraphManager {
     entities: Entity[],
     buildState: ActionState,
     options?: GraphBuilderOptions,
-  ): Promise<GraphBuilderReport>;
+    report?: GraphManagerOperationReport,
+  ): Promise<void>;
 
   abstract buildGenerations(
     graph: PureModel,
     generationEntities: Map<string, Entity[]>,
     buildState: ActionState,
     options?: GraphBuilderOptions,
-  ): Promise<GraphBuilderReport>;
+    report?: GraphManagerOperationReport,
+  ): Promise<void>;
 
   // ------------------------------------------- Grammar -------------------------------------------
 
@@ -278,11 +287,13 @@ export abstract class AbstractPureGraphManager {
       onError?: () => void;
       keepSourceInformation?: boolean;
     },
+    report?: GraphManagerOperationReport,
   ): Promise<CompilationResult>;
   abstract compileText(
     graphGrammar: string,
     graph: PureModel,
     options?: { onError?: () => void },
+    report?: GraphManagerOperationReport,
   ): Promise<TextCompilationResult>;
   abstract getLambdaReturnType(
     lambda: RawLambda,
@@ -290,7 +301,7 @@ export abstract class AbstractPureGraphManager {
     options?: { keepSourceInformation?: boolean },
   ): Promise<string>;
 
-  // ------------------------------------------- Test  -------------------------------------------
+  // ------------------------------------------- Test -------------------------------------------
 
   abstract runTests(
     inputs: RunTestsTestableInput[],
@@ -304,7 +315,7 @@ export abstract class AbstractPureGraphManager {
     graph: PureModel,
   ): Promise<AssertFail>;
 
-  // ------------------------------------------- Value Specification  -------------------------------------------
+  // ------------------------------------------- Value Specification -------------------------------------------
 
   abstract buildValueSpecification(
     json: PlainObject,
@@ -351,14 +362,12 @@ export abstract class AbstractPureGraphManager {
   abstract getAvailableExternalFormatsDescriptions(): Promise<
     ExternalFormatDescription[]
   >;
-
   abstract generateModelFromExternalFormat(
     schemaSet: SchemaSet,
     targetBinding: string | undefined,
     configs: ConfigurationProperty[],
     graph: PureModel,
   ): Promise<string>;
-
   abstract generateSchemaFromExternalFormatConfig(
     modelUnit: ModelUnit,
     targetBinding: string | undefined,
@@ -367,6 +376,7 @@ export abstract class AbstractPureGraphManager {
   ): Promise<SchemaSet[]>;
 
   // ------------------------------------------- Import -------------------------------------------
+
   abstract getExamplePureProtocolText(): string;
   abstract getExampleExternalFormatImportText(): string;
   abstract entitiesToPureProtocolText(entities: Entity[]): Promise<string>;
@@ -380,6 +390,7 @@ export abstract class AbstractPureGraphManager {
     runtime: Runtime | undefined,
     graph: PureModel,
     options?: ExecutionOptions,
+    report?: GraphManagerOperationReport,
   ): Promise<ExecutionResult>;
 
   abstract generateExecutionPlan(
@@ -387,6 +398,7 @@ export abstract class AbstractPureGraphManager {
     mapping: Mapping | undefined,
     runtime: Runtime | undefined,
     graph: PureModel,
+    report?: GraphManagerOperationReport,
   ): Promise<RawExecutionPlan>;
 
   abstract debugExecutionPlanGeneration(
@@ -394,19 +406,9 @@ export abstract class AbstractPureGraphManager {
     mapping: Mapping | undefined,
     runtime: Runtime | undefined,
     graph: PureModel,
+    report?: GraphManagerOperationReport,
   ): Promise<{ plan: RawExecutionPlan; debug: string }>;
 
-  abstract buildExecutionPlan(
-    executionPlanJson: RawExecutionPlan,
-    graph: PureModel,
-  ): ExecutionPlan;
-
-  abstract serializeExecutionPlan(
-    executionPlan: ExecutionPlan,
-  ): RawExecutionPlan;
-
-  abstract serializeExecutionNode(executionNode: ExecutionNode): object;
-
   abstract generateExecuteTestData(
     lambda: RawLambda,
     parameters: (string | number | boolean)[],
@@ -417,8 +419,20 @@ export abstract class AbstractPureGraphManager {
       // Anonymizes data by hashing any string values in the generated data
       anonymizeGeneratedData?: boolean;
     },
+    report?: GraphManagerOperationReport,
   ): Promise<string>;
 
+  abstract buildExecutionPlan(
+    executionPlanJson: RawExecutionPlan,
+    graph: PureModel,
+  ): ExecutionPlan;
+
+  abstract serializeExecutionPlan(
+    executionPlan: ExecutionPlan,
+  ): RawExecutionPlan;
+
+  abstract serializeExecutionNode(executionNode: ExecutionNode): object;
+
   // ------------------------------------------- Service -------------------------------------------
   /**
    * @modularize

package/src/graphManager/GraphManagerEvent.ts

@@ -15,29 +15,35 @@
  */
 
 export enum GRAPH_MANAGER_EVENT {
-  GRAPH_BUILDER_ELEMENTS_DESERIALIZED = 'graph-manager.graph-builder.deserialization.success',
-  GRAPH_BUILDER_ELEMENTS_INDEXED = 'graph-manager.graph-builder.indexing.success',
-  GRAPH_BUILDER_SECTION_INDICES_BUILT = 'graph-manager.graph-builder.build-section-indices.success',
-  GRAPH_BUILDER_DOMAIN_MODELS_BUILT = 'graph-manager.graph-builder.build-domain-models.success',
-  GRAPH_BUILDER_STORES_BUILT = 'graph-manager.graph-builder.build-store.success',
-  GRAPH_BUILDER_MAPPINGS_BUILT = 'graph-manager.graph-builder.build-mappings.success',
-  GRAPH_BUILDER_CONNECTIONS_AND_RUNTIMES_BUILT = 'graph-manager.graph-builder.build-connections-and-runtimes.success',
-  GRAPH_BUILDER_SERVICES_BUILT = 'graph-manager.graph-builder.build-services.success',
-  GRAPH_BUILDER_DATA_ELEMENTS_BUILT = 'graph-manager.graph-builder.build-data.success',
-  GRAPH_BUILDER_OTHER_ELEMENTS_BUILT = 'graph-manager.graph-builder.build-other-elements.success',
-  GRAPH_BUILDER_COMPLETED = 'graph-manager.graph-builder.success',
+  GRAPH_BUILDER_DESERIALIZE_ELEMENTS__SUCCESS = 'graph-manager.graph-builder.deserialization.success',
+  GRAPH_BUILDER_INDEX_ELEMENTS__SUCCESS = 'graph-manager.graph-builder.indexing.success',
+  GRAPH_BUILDER_BUILD_SECTION_INDICES__SUCCESS = 'graph-manager.graph-builder.build-section-indices.success',
+  GRAPH_BUILDER_BUILD_DOMAIN_MODELS__SUCCESS = 'graph-manager.graph-builder.build-domain-models.success',
+  GRAPH_BUILDER_BUILD_STORES__SUCCESS = 'graph-manager.graph-builder.build-store.success',
+  GRAPH_BUILDER_BUILD_MAPPINGS__SUCCESS = 'graph-manager.graph-builder.build-mappings.success',
+  GRAPH_BUILDER_BUILD_CONNECTIONS_AND_RUNTIMES__SUCCESS = 'graph-manager.graph-builder.build-connections-and-runtimes.success',
+  GRAPH_BUILDER_BUILD_SERVICES__SUCCESS = 'graph-manager.graph-builder.build-services.success',
+  GRAPH_BUILDER_BUILD_DATA_ELEMENTS__SUCCESS = 'graph-manager.graph-builder.build-data.success',
+  GRAPH_BUILDER_BUILD_OTHER_ELEMENTS__SUCCESS = 'graph-manager.graph-builder.build-other-elements.success',
+  GRAPH_BUILDER_BUILD_GRAPH__SUCCESS = 'graph-manager.graph-builder.success',
 
-  GRAPH_SYSTEM_INITIALIZED = 'graph-manager.system-initialization.success',
-  GRAPH_INITIALIZED = 'graph-manager.graph-initialization.success',
-  GRAPH_DEPENDENCIES_FETCHED = 'graph-manager.fetch-dependencies.success',
-  GRAPH_ENTITIES_FETCHED = 'graph-manager.entities-dependencies.success',
-  GRAPH_UPDATED_AND_REBUILT = 'graph-manager.rebuild.success',
-  GRAPH_MODEL_TO_GRAMMAR_TRANSFORMED = 'graph-manager.grammar.composing.success',
-  GRAPH_GRAMMAR_TO_MODEL_TRANSFORMED = 'graph-manager.grammar.parsing.success',
-  GRAPH_META_MODEL_TO_PROTOCOL_TRANSFORMED = 'graph-manager.transformation.success',
-  GRAPH_COMPILE_CONTEXT_COLLECTED = 'graph-manager.build-graph-compile-context.success',
-  GRAPH_PROTOCOL_SERIALIZED = 'graph-manager.graph-serialization.success',
+  INITIALIZE_GRAPH__SUCCESS = 'graph-manager.graph-initialization.success',
+  INITIALIZE_GRAPH_SYSTEM__SUCCESS = 'graph-manager.system-initialization.success',
+  FETCH_GRAPH_DEPENDENCIES__SUCCESS = 'graph-manager.fetch-dependencies.success',
+  FETCH_GRAPH_ENTITIES__SUCCESS = 'graph-manager.entities-dependencies.success',
+  TRANSFORM_GRAPH_META_MODEL_TO_PROTOCOL__SUCCESS = 'graph-manager.transformation.success',
+  COLLECT_GRAPH_COMPILE_CONTEXT__SUCCESS = 'graph-manager.build-graph-compile-context.success',
+  SERIALIZE_GRAPH_PROTOCOL__SUCCESS = 'graph-manager.graph-serialization.success',
+  SERIALIZE_GRAPH_CONTEXT_PROTOCOL__SUCCESS = 'graph-manager.graph-context-serialization.success',
+  UPDATE_AND_REBUILD_GRAPH__SUCCESS = 'graph-manager.rebuild.success',
+  TRANSFORM_GRAPH_META_MODEL_TO_GRAMMAR__SUCCESS = 'graph-manager.grammar.composing.success',
+  TRANSOFMR_GRAPH_GRAMMAR_TO_META_MODEL__SUCCESS = 'graph-manager.grammar.parsing.success',
 
+  V1_ENGINE_OPERATION_SERVER_CALL__SUCCESS = 'graph-manager.v1.engine-operation.server-call.success',
+  V1_ENGINE_OPERATION_INPUT__SUCCESS = 'graph-manager.v1.engine-operation.graph.collect-input.success',
+
+  // FAILURE
+  // TODO: consider to spliting all of these generic errors into more specific events
   GRAPH_BUILDER_FAILURE = 'graph-manager.graph-builder.failure',
   EXECUTION_FAILURE = 'graph-manager.execution.failure',
   COMPILATION_FAILURE = 'graph-manager.compilation.failure',

package/src/graphManager/GraphManagerMetrics.ts

@@ -0,0 +1,93 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { isNonNullable } from '@finos/legend-shared';
+import type { PureModel } from '../graph/PureModel.js';
+import { GRAPH_MANAGER_EVENT } from './GraphManagerEvent.js';
+
+export type GraphManagerOperationReport = {
+  timings: {
+    total: number;
+    [key: string]: number;
+  };
+  elementCount: {
+    total: number | undefined;
+    [key: string]: number | undefined;
+  };
+};
+
+export const createGraphManagerOperationReport =
+  (): GraphManagerOperationReport => ({
+    timings: { total: 0 },
+    elementCount: { total: undefined },
+  });
+
+export const createGraphBuilderReport = (): GraphManagerOperationReport => ({
+  timings: {
+    total: 0,
+    [GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_GRAPH__SUCCESS]: 0,
+  },
+  elementCount: { total: undefined },
+});
+
+export const reportGraphAnalytics = (
+  graph: PureModel,
+): GraphManagerOperationReport & {
+  dependenciesCount: number;
+} => {
+  const report = createGraphManagerOperationReport();
+
+  report.elementCount.sectionIndex =
+    (report.elementCount.sectionIndex ?? 0) + graph.sectionIndices.length;
+
+  report.elementCount.association =
+    (report.elementCount.association ?? 0) + graph.associations.length;
+  report.elementCount.class =
+    (report.elementCount.class ?? 0) + graph.classes.length;
+  report.elementCount.enumeration =
+    (report.elementCount.enumeration ?? 0) + graph.enumerations.length;
+  report.elementCount.function =
+    (report.elementCount.function ?? 0) + graph.functions.length;
+  report.elementCount.profile =
+    (report.elementCount.profile ?? 0) + graph.profiles.length;
+  report.elementCount.measure =
+    (report.elementCount.measure ?? 0) + graph.measures.length;
+
+  report.elementCount.store =
+    (report.elementCount.store ?? 0) + graph.stores.length;
+  report.elementCount.mapping =
+    (report.elementCount.mapping ?? 0) + graph.mappings.length;
+  report.elementCount.connection =
+    (report.elementCount.connection ?? 0) + graph.connections.length;
+  report.elementCount.runtime =
+    (report.elementCount.runtime ?? 0) + graph.runtimes.length;
+
+  report.elementCount.service =
+    (report.elementCount.service ?? 0) + graph.services.length;
+
+  const counted = Object.values(report.elementCount)
+    .filter(isNonNullable)
+    .reduce((a, b) => a + b, 0);
+
+  report.elementCount.total =
+    (report.elementCount.total ?? 0) + graph.allElements.length;
+  report.elementCount.other = report.elementCount.total - counted;
+
+  return {
+    ...report,
+    dependenciesCount: graph.dependencyManager.numberOfDependencies,
+  };
+};

package/src/graphManager/GraphManagerTelemetry.ts

@@ -15,21 +15,26 @@
  */
 
 import type { TelemetryService } from '@finos/legend-shared';
-import type { GraphBuilderReport } from './GraphBuilderReport.js';
+import type { GraphManagerOperationReport } from './GraphManagerMetrics.js';
 import { GRAPH_MANAGER_EVENT } from './GraphManagerEvent.js';
 
-type GraphBuilt_TelemetryData = {
+type GraphInitialized_TelemetryData = {
   timings: Record<string, number>;
-  dependencies: GraphBuilderReport;
-  graph: GraphBuilderReport;
-  generations?: GraphBuilderReport;
+  dependencies: GraphManagerOperationReport;
+  dependenciesCount: number;
+  graph: GraphManagerOperationReport;
+  generations?: GraphManagerOperationReport;
+  generationCount?: number;
 };
 
 export class GraphManagerTelemetry {
-  static logEvent_GraphInitialized(
+  static logEvent_GraphInitializationSucceeded(
     telemetryService: TelemetryService,
-    data: GraphBuilt_TelemetryData,
+    data: GraphInitialized_TelemetryData,
   ): void {
-    telemetryService.logEvent(GRAPH_MANAGER_EVENT.GRAPH_INITIALIZED, data);
+    telemetryService.logEvent(
+      GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS,
+      data,
+    );
   }
 }

package/src/graphManager/action/changeDetection/GraphObserverHelper.ts

@@ -99,7 +99,10 @@ export const observe_DependencyManager = skipObserved(
       roots: observable,
       projectDependencyModelsIndex: observable,
 
+      numberOfDependencies: computed,
+      hasDependencies: computed,
       dependencyGraphs: computed,
+
       sectionIndices: computed,
       profiles: computed,
       enumerations: computed,