@finos/legend-graph 32.5.1 → 32.5.3

package/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts

@@ -51,8 +51,9 @@ import {
   type ExecutionOptions,
   type ServiceRegistrationOptions,
 } from '../../../../graph-manager/AbstractPureGraphManager.js';
-import type { Mapping } from '../../../../graph/metamodel/pure/packageableElements/mapping/Mapping.js';
+import { Mapping } from '../../../../graph/metamodel/pure/packageableElements/mapping/Mapping.js';
 import type { Runtime } from '../../../../graph/metamodel/pure/packageableElements/runtime/Runtime.js';
+import { PackageableRuntime } from '../../../../graph/metamodel/pure/packageableElements/runtime/PackageableRuntime.js';
 import type { PackageableElement } from '../../../../graph/metamodel/pure/packageableElements/PackageableElement.js';
 import {
   type SystemModel,
@@ -348,6 +349,7 @@ import { V1_UserListOwnership } from './model/packageableElements/service/V1_Ser
 import { V1_PureSingleExecution } from './model/packageableElements/service/V1_ServiceExecution.js';
 import {
   type V1_Runtime,
+  V1_EngineRuntime,
   V1_RuntimePointer,
 } from './model/packageableElements/runtime/V1_Runtime.js';
 import type { TestDebug } from '../../../../graph/metamodel/pure/test/result/DebugTestsResult.js';
@@ -379,10 +381,14 @@ import {
   DataProductAnalysis,
 } from '../../../action/analytics/data-product/DataProductAnalysis.js';
 import {
+  AccessPointGroup,
   DataProductAccessType,
+  DataProductElementScope,
+  LakehouseAccessPoint,
   ModelAccessPointGroup,
   NativeModelAccess,
   NativeModelExecutionContext,
+  type DataProductElement,
 } from '../../../../graph/metamodel/pure/dataProduct/DataProduct.js';
 import { V1_MemSQLFunction } from './model/packageableElements/function/V1_MemSQLFunction.js';
 import { LineageModel } from '../../../../graph/metamodel/pure/lineage/LineageModel.js';
@@ -403,6 +409,12 @@ import { IngestDefinition } from '../../../../graph/metamodel/pure/packageableEl
 import { Database } from '../../../../graph/metamodel/pure/packageableElements/store/relational/model/Database.js';
 import { V1_createAccessorFromPackageableElement } from './helpers/V1_AccessorHelper.js';
 
+/**
+ * Number of elements to process synchronously before yielding to the event loop.
+ * This avoids per-element setTimeout overhead while keeping the UI responsive.
+ */
+const GRAPH_BUILDER_BATCH_SIZE = 100;
+
 class V1_PureModelContextDataIndex {
   elements: V1_PackageableElement[] = [];
   nativeElements: V1_PackageableElement[] = [];
@@ -1297,46 +1309,57 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
       graph.allElements.map((el) => el.path),
     );
 
-    await Promise.all(
-      inputs.flatMap(async (input) => {
-        // create the package cache
-        const packageCache = new Map<string, Package>();
-        await Promise.all(
-          input.data.nativeElements.map((element) => {
-            return this.visitWithGraphBuilderErrorHandling(
-              element,
-              new V1_ElementFirstPassBuilder(
-                this.getBuilderContext(graph, input.model, element, options),
-                packageCache,
-                elementPathCache,
-              ),
-            );
-          }),
-        );
-        await Promise.all(
-          this.graphBuilderExtensions.sortedExtraElementBuilders.flatMap(
-            (builder) =>
-              (input.data.otherElementsByBuilder.get(builder) ?? []).map(
-                (element) => {
-                  return this.visitWithGraphBuilderErrorHandling(
-                    element,
-                    new V1_ElementFirstPassBuilder(
-                      this.getBuilderContext(
-                        graph,
-                        input.model,
-                        element,
-                        options,
-                      ),
-                      packageCache,
-                      elementPathCache,
-                    ),
-                  );
-                },
-              ),
-          ),
+    // NOTE: We must index ALL native elements across ALL inputs before indexing
+    // plugin-contributed ("other") elements. Plugin elements
+    // may resolve references to native elements (e.g. Mapping, Runtime) from
+    // other inputs during their first pass. If we process each input fully
+    // (native + other) before moving to the next, a plugin element in input[0]
+    // could fail to find a native element that only exists in input[1].
+    const packageCaches = new Map<
+      V1_PureGraphBuilderInput,
+      Map<string, Package>
+    >();
+
+    const createFirstPassBuilder = (
+      input: V1_PureGraphBuilderInput,
+      packageCache: Map<string, Package>,
+      element: V1_PackageableElement,
+    ): V1_ElementFirstPassBuilder =>
+      new V1_ElementFirstPassBuilder(
+        this.getBuilderContext(graph, input.model, element, options),
+        packageCache,
+        elementPathCache,
+      );
+
+    for (const input of inputs) {
+      const packageCache = new Map<string, Package>();
+      packageCaches.set(input, packageCache);
+
+      // Phase 1: index native elements for this input
+      await this.runBatchedLoop(input.data.nativeElements, (element) =>
+        this.visitWithGraphBuilderErrorHandling(
+          element,
+          createFirstPassBuilder(input, packageCache, element),
+        ),
+      );
+    }
+
+    // Phase 2: index other (plugin-contributed) elements across all inputs,
+    // now that all native elements have been indexed
+    for (const input of inputs) {
+      const packageCache = guaranteeNonNullable(packageCaches.get(input));
+
+      const otherElements =
+        this.graphBuilderExtensions.sortedExtraElementBuilders.flatMap(
+          (builder) => input.data.otherElementsByBuilder.get(builder) ?? [],
         );
-      }),
-    );
+      await this.runBatchedLoop(otherElements, (element) =>
+        this.visitWithGraphBuilderErrorHandling(
+          element,
+          createFirstPassBuilder(input, packageCache, element),
+        ),
+      );
+    }
   }
 
   private async buildTypes(
@@ -1345,128 +1368,78 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     options?: GraphBuilderOptions,
   ): Promise<void> {
     // Second pass
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.profiles.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.profiles,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.classes.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.classes,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.enumerations.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.enumerations,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.measures.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.measures,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.functions.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.functions,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
     // Third pass
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.classes.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementThirdPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.classes,
+      (ctx) => new V1_ElementThirdPassBuilder(ctx),
+      options,
     );
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.associations.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementThirdPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.associations,
+      (ctx) => new V1_ElementThirdPassBuilder(ctx),
+      options,
     );
     // Fourth Pass
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.classes.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementFourthPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.classes,
+      (ctx) => new V1_ElementFourthPassBuilder(ctx),
+      options,
     );
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.associations.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementFourthPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.associations,
+      (ctx) => new V1_ElementFourthPassBuilder(ctx),
+      options,
     );
     // Fifth pass
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.classes.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementFifthPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.classes,
+      (ctx) => new V1_ElementFifthPassBuilder(ctx),
+      options,
     );
   }
 
@@ -1475,17 +1448,12 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     inputs: V1_PureGraphBuilderInput[],
     options?: GraphBuilderOptions,
   ): Promise<void> {
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.functionActivators.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.functionActivators,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
   }
 
@@ -1494,53 +1462,33 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     inputs: V1_PureGraphBuilderInput[],
     options?: GraphBuilderOptions,
   ): Promise<void> {
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.stores.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.stores,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.stores.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementThirdPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.stores,
+      (ctx) => new V1_ElementThirdPassBuilder(ctx),
+      options,
     );
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.stores.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementFourthPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.stores,
+      (ctx) => new V1_ElementFourthPassBuilder(ctx),
+      options,
     );
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.stores.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementFifthPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.stores,
+      (ctx) => new V1_ElementFifthPassBuilder(ctx),
+      options,
     );
   }
 
@@ -1549,41 +1497,26 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     inputs: V1_PureGraphBuilderInput[],
     options?: GraphBuilderOptions,
   ): Promise<void> {
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.mappings.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.mappings,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.mappings.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementThirdPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.mappings,
+      (ctx) => new V1_ElementThirdPassBuilder(ctx),
+      options,
     );
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.mappings.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementFourthPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.mappings,
+      (ctx) => new V1_ElementFourthPassBuilder(ctx),
+      options,
     );
   }
 
@@ -1593,29 +1526,19 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     options?: GraphBuilderOptions,
   ): Promise<void> {
     // NOTE: connections must be built before runtimes
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.connections.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.connections,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.runtimes.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.runtimes,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
   }
 
@@ -1624,29 +1547,19 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     inputs: V1_PureGraphBuilderInput[],
     options?: GraphBuilderOptions,
   ): Promise<void> {
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.services.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.services,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.executionEnvironments.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.executionEnvironments,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
   }
 
@@ -1655,17 +1568,12 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     inputs: V1_PureGraphBuilderInput[],
     options?: GraphBuilderOptions,
   ): Promise<void> {
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.dataElements.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.dataElements,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
   }
 
@@ -1674,17 +1582,12 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     inputs: V1_PureGraphBuilderInput[],
     options?: GraphBuilderOptions,
   ): Promise<void> {
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.products.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.products,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
   }
 
@@ -1693,17 +1596,12 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     inputs: V1_PureGraphBuilderInput[],
     options?: GraphBuilderOptions,
   ): Promise<void> {
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.fileGenerations.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.fileGenerations,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
   }
 
@@ -1712,17 +1610,12 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     inputs: V1_PureGraphBuilderInput[],
     options?: GraphBuilderOptions,
   ): Promise<void> {
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.generationSpecifications.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.generationSpecifications,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
   }
 
@@ -1731,17 +1624,12 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     inputs: V1_PureGraphBuilderInput[],
     options?: GraphBuilderOptions,
   ): Promise<void> {
-    await Promise.all(
-      inputs.flatMap((input) =>
-        input.data.sectionIndices.map((element) =>
-          this.visitWithGraphBuilderErrorHandling(
-            element,
-            new V1_ElementSecondPassBuilder(
-              this.getBuilderContext(graph, input.model, element, options),
-            ),
-          ),
-        ),
-      ),
+    await this.processElementsInBatches(
+      graph,
+      inputs,
+      (data) => data.sectionIndices,
+      (ctx) => new V1_ElementSecondPassBuilder(ctx),
+      options,
     );
   }
 
@@ -1750,82 +1638,90 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     inputs: V1_PureGraphBuilderInput[],
     options?: GraphBuilderOptions,
   ): Promise<void> {
-    await Promise.all(
-      this.graphBuilderExtensions.sortedExtraElementBuilders.map(
-        async (builder) => {
-          await Promise.all(
-            inputs.flatMap((input) =>
-              (input.data.otherElementsByBuilder.get(builder) ?? []).map(
-                (element) =>
-                  this.visitWithGraphBuilderErrorHandling(
-                    element,
-                    new V1_ElementSecondPassBuilder(
-                      this.getBuilderContext(
-                        graph,
-                        input.model,
-                        element,
-                        options,
-                      ),
-                    ),
-                  ),
-              ),
-            ),
-          );
-          await Promise.all(
-            inputs.flatMap((input) =>
-              (input.data.otherElementsByBuilder.get(builder) ?? []).map(
-                (element) =>
-                  this.visitWithGraphBuilderErrorHandling(
-                    element,
-                    new V1_ElementThirdPassBuilder(
-                      this.getBuilderContext(
-                        graph,
-                        input.model,
-                        element,
-                        options,
-                      ),
-                    ),
-                  ),
-              ),
-            ),
-          );
-          await Promise.all(
-            inputs.flatMap((input) =>
-              (input.data.otherElementsByBuilder.get(builder) ?? []).map(
-                (element) =>
-                  this.visitWithGraphBuilderErrorHandling(
-                    element,
-                    new V1_ElementFourthPassBuilder(
-                      this.getBuilderContext(
-                        graph,
-                        input.model,
-                        element,
-                        options,
-                      ),
-                    ),
-                  ),
-              ),
-            ),
-          );
-          await Promise.all(
-            inputs.flatMap((input) =>
-              (input.data.otherElementsByBuilder.get(builder) ?? []).map(
-                (element) =>
-                  this.visitWithGraphBuilderErrorHandling(
-                    element,
-                    new V1_ElementFifthPassBuilder(
-                      this.getBuilderContext(
-                        graph,
-                        input.model,
-                        element,
-                        options,
-                      ),
-                    ),
-                  ),
-              ),
-            ),
-          );
-        },
+    for (const builder of this.graphBuilderExtensions
+      .sortedExtraElementBuilders) {
+      const getElements = (
+        data: V1_PureModelContextDataIndex,
+      ): V1_PackageableElement[] =>
+        data.otherElementsByBuilder.get(builder) ?? [];
+      await this.processElementsInBatches(
+        graph,
+        inputs,
+        getElements,
+        (ctx) => new V1_ElementSecondPassBuilder(ctx),
+        options,
+      );
+      await this.processElementsInBatches(
+        graph,
+        inputs,
+        getElements,
+        (ctx) => new V1_ElementThirdPassBuilder(ctx),
+        options,
+      );
+      await this.processElementsInBatches(
+        graph,
+        inputs,
+        getElements,
+        (ctx) => new V1_ElementFourthPassBuilder(ctx),
+        options,
+      );
+      await this.processElementsInBatches(
+        graph,
+        inputs,
+        getElements,
+        (ctx) => new V1_ElementFifthPassBuilder(ctx),
+        options,
+      );
+    }
+  }
+
+  /**
+   * Run a callback for each item in the array, processing items in batches
+   * of {@link GRAPH_BUILDER_BATCH_SIZE} and yielding to the event loop between
+   * batches to keep the UI responsive.
+   */
+  private async runBatchedLoop<T>(
+    items: T[],
+    process: (item: T) => void,
+  ): Promise<void> {
+    for (let i = 0; i < items.length; i += GRAPH_BUILDER_BATCH_SIZE) {
+      if (i > 0) {
+        await new Promise<void>((resolve) => setTimeout(resolve, 0));
+      }
+      const end = Math.min(i + GRAPH_BUILDER_BATCH_SIZE, items.length);
+      for (let j = i; j < end; j++) {
+        process(guaranteeNonNullable(items[j]));
+      }
+    }
+  }
+
+  /**
+   * Process elements from inputs in batches, yielding to the event loop
+   * between batches to keep the UI responsive.
+   */
+  private async processElementsInBatches(
+    graph: PureModel,
+    inputs: V1_PureGraphBuilderInput[],
+    getElements: (
+      data: V1_PureModelContextDataIndex,
+    ) => V1_PackageableElement[],
+    createVisitor: (
+      ctx: V1_GraphBuilderContext,
+    ) => V1_PackageableElementVisitor<unknown>,
+    options?: GraphBuilderOptions,
+  ): Promise<void> {
+    const allItems = inputs.flatMap((input) =>
+      getElements(input.data).map((element) => ({
+        element,
+        model: input.model,
+      })),
+    );
+    await this.runBatchedLoop(allItems, (item) =>
+      this.visitWithGraphBuilderErrorHandling(
+        item.element,
+        createVisitor(
+          this.getBuilderContext(graph, item.model, item.element, options),
+        ),
       ),
     );
   }
@@ -1833,9 +1729,9 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
   private visitWithGraphBuilderErrorHandling<T>(
     element: V1_PackageableElement,
     visitor: V1_PackageableElementVisitor<T>,
-  ): Promise<T> {
+  ): T {
     try {
-      return promisify(() => element.accept_PackageableElementVisitor(visitor));
+      return element.accept_PackageableElementVisitor(visitor);
     } catch (err) {
       assertErrorThrown(err);
       const error =
@@ -3840,6 +3736,24 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     );
   }
 
+  private resolveArtifactElements(
+    elementPaths: string[],
+    pureGraph: PureModel,
+  ): DataProductElementScope[] {
+    const scopes: DataProductElementScope[] = [];
+    for (const path of elementPaths) {
+      const el = pureGraph.getNullableElement(path, true);
+      if (el) {
+        const scope = new DataProductElementScope();
+        scope.element = PackageableElementExplicitReference.create(
+          el as DataProductElement,
+        );
+        scopes.push(scope);
+      }
+    }
+    return scopes;
+  }
+
   async buildDataProductAnalysis(
     artifact: V1_DataProductArtifact,
     dataProductPath: string,
@@ -3849,77 +3763,264 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     projectInfo: ProjectGAVCoordinates,
     graphReport?: GraphManagerOperationReport,
   ): Promise<DataProductAnalysisQueryResult> {
-    // Collect all mapping generation infos from model access point groups and native model access
-    const allMappingGenInfos = new Map<string, V1_MappingGenerationInfo>();
+    switch (dataProductAccessType) {
+      case DataProductAccessType.MODEL:
+        return this.buildModelAccessDataProductAnalysis(
+          artifact,
+          dataProductPath,
+          pureGraph,
+          accessPointId,
+          projectInfo,
+          graphReport,
+        );
+      case DataProductAccessType.NATIVE:
+        return this.buildNativeAccessDataProductAnalysis(
+          artifact,
+          dataProductPath,
+          pureGraph,
+          accessPointId,
+          projectInfo,
+          graphReport,
+        );
+      case DataProductAccessType.LAKEHOUSE:
+        return this.buildLakehouseAccessDataProductAnalysis(
+          artifact,
+          dataProductPath,
+          pureGraph,
+          accessPointId,
+          projectInfo,
+          graphReport,
+        );
+      default:
+        throw new UnsupportedOperationError(
+          `Unsupported data product access type: '${dataProductAccessType}'`,
+        );
+    }
+  }
 
+  private async buildModelAccessDataProductAnalysis(
+    artifact: V1_DataProductArtifact,
+    dataProductPath: string,
+    pureGraph: PureModel,
+    accessPointId: string,
+    projectInfo: ProjectGAVCoordinates,
+    graphReport?: GraphManagerOperationReport,
+  ): Promise<DataProductAnalysisQueryResult> {
     const modelAccessPointGroups = artifact.accessPointGroups.filter(
       (group): group is V1_ModelAccessPointGroupInfo =>
         group instanceof V1_ModelAccessPointGroupInfo,
     );
-    for (const group of modelAccessPointGroups) {
-      allMappingGenInfos.set(
-        group.mappingGeneration.path,
-        group.mappingGeneration,
+    const accessGroup = modelAccessPointGroups.find(
+      (g) => g.id === accessPointId,
+    );
+    if (!accessGroup) {
+      throw new Error(
+        `Can't resolve access point '${accessPointId}' (type: ${DataProductAccessType.MODEL}) in data product '${dataProductPath}'`,
       );
     }
+    const mappingPath = accessGroup.mappingGeneration.path;
 
-    if (artifact.nativeModelAccess?.mappingGenerations) {
-      for (const [path, genInfo] of artifact.nativeModelAccess
-        .mappingGenerations) {
-        if (!allMappingGenInfos.has(path)) {
-          allMappingGenInfos.set(path, genInfo);
-        }
-      }
-    }
-
-    // Resolve mapping path from accessPointId and dataProductAccessType
-    let mappingPath: string | undefined;
-    let accessGroup:
-      | V1_ModelAccessPointGroupInfo
-      | V1_NativeModelExecutionContextInfo
-      | undefined = undefined;
-    if (dataProductAccessType === DataProductAccessType.MODEL) {
-      const group = modelAccessPointGroups.find((g) => g.id === accessPointId);
-      if (group) {
-        mappingPath = group.mappingGeneration.path;
-        accessGroup = group;
-      }
-    } else if (dataProductAccessType === DataProductAccessType.NATIVE) {
-      const nativeCtx =
-        artifact.nativeModelAccess?.nativeModelExecutionContexts.find(
-          (ctx) => ctx.key === accessPointId,
-        );
-      if (nativeCtx) {
-        mappingPath = nativeCtx.mapping;
-        accessGroup = nativeCtx;
-      }
-    }
+    return this.buildDataProductAnalysisCore(
+      artifact,
+      dataProductPath,
+      pureGraph,
+      mappingPath,
+      accessGroup,
+      undefined, // no resolved runtime path for MODEL access
+      projectInfo,
+      graphReport,
+    );
+  }
 
-    if (!mappingPath) {
+  private async buildNativeAccessDataProductAnalysis(
+    artifact: V1_DataProductArtifact,
+    dataProductPath: string,
+    pureGraph: PureModel,
+    accessPointId: string,
+    projectInfo: ProjectGAVCoordinates,
+    graphReport?: GraphManagerOperationReport,
+  ): Promise<DataProductAnalysisQueryResult> {
+    const nativeCtx =
+      artifact.nativeModelAccess?.nativeModelExecutionContexts.find(
+        (ctx) => ctx.key === accessPointId,
+      );
+    if (!nativeCtx) {
       throw new Error(
-        `Can't resolve mapping path for access point '${accessPointId}' (type: ${dataProductAccessType}) in data product '${dataProductPath}'`,
+        `Can't resolve access point '${accessPointId}' (type: ${DataProductAccessType.NATIVE}) in data product '${dataProductPath}'`,
       );
     }
-    if (!accessGroup) {
+    const mappingPath = nativeCtx.mapping;
+    if (!mappingPath) {
       throw new Error(
-        `Can't resolve access group for access point '${accessPointId}' (type: ${dataProductAccessType}) in data product '${dataProductPath}'`,
+        `Can't resolve mapping path for access point '${accessPointId}' (type: ${DataProductAccessType.NATIVE}) in data product '${dataProductPath}'`,
       );
     }
-    // Build the minimal graph using ONLY the elements from the resolved mapping
-    const resolvedMappingGenInfo = mappingPath
-      ? allMappingGenInfos.get(mappingPath)
-      : undefined;
 
-    if (!resolvedMappingGenInfo) {
+    // Resolve runtime path for the native context
+    const resolvedRuntimePath = nativeCtx.runtimeGeneration?.path;
+
+    return this.buildDataProductAnalysisCore(
+      artifact,
+      dataProductPath,
+      pureGraph,
+      mappingPath,
+      nativeCtx,
+      resolvedRuntimePath,
+      projectInfo,
+      graphReport,
+    );
+  }
+
+  private async buildLakehouseAccessDataProductAnalysis(
+    artifact: V1_DataProductArtifact,
+    dataProductPath: string,
+    pureGraph: PureModel,
+    accessPointId: string,
+    projectInfo: ProjectGAVCoordinates,
+    graphReport?: GraphManagerOperationReport,
+  ): Promise<DataProductAnalysisQueryResult> {
+    // Create a dummy data product element
+    const dummyDataProduct = new V1_DataProduct();
+    dummyDataProduct.package =
+      extractPackagePathFromPath(dataProductPath) ?? '';
+    dummyDataProduct.name = extractElementNameFromPath(dataProductPath);
+    dummyDataProduct.title = artifact.dataProduct.title;
+    dummyDataProduct.description = artifact.dataProduct.description;
+
+    // Build graph with only the data product
+    const graphEntities = [dummyDataProduct]
+      .filter((el) => !pureGraph.getNullableElement(el.path, false))
+      .map((el) => this.elementProtocolToEntity(el));
+    await this.buildGraph(
+      pureGraph,
+      graphEntities,
+      ActionState.create(),
+      {
+        origin: new LegendSDLC(
+          projectInfo.groupId,
+          projectInfo.artifactId,
+          projectInfo.versionId,
+        ),
+      },
+      graphReport,
+    );
+
+    const data = pureGraph.getDataProduct(dataProductPath);
+
+    // Create access point groups with LakehouseAccessPoints from artifact data
+    data.accessPointGroups = artifact.accessPointGroups
+      .filter(
+        (groupInfo) => !(groupInfo instanceof V1_ModelAccessPointGroupInfo),
+      )
+      .map((groupInfo) => {
+        const apGroup = new AccessPointGroup();
+        apGroup.id = groupInfo.id;
+        apGroup.description = groupInfo.description;
+        apGroup.accessPoints = groupInfo.accessPointImplementations.map(
+          (apImpl) => {
+            const lakehouseAP = new LakehouseAccessPoint(
+              apImpl.id,
+              '', // targetEnvironment is not available in the artifact
+              new RawLambda(undefined, undefined),
+              apGroup,
+            );
+            lakehouseAP.description = apImpl.description;
+            return lakehouseAP;
+          },
+        );
+        return apGroup;
+      });
+
+    // Find the lakehouse access point matching the requested id
+    const lakehouseResult = data.accessPointGroups
+      .flatMap((group) => group.accessPoints)
+      .find(
+        (ap): ap is LakehouseAccessPoint =>
+          ap instanceof LakehouseAccessPoint && ap.id === accessPointId,
+      );
+    if (!lakehouseResult) {
       throw new Error(
-        `Can't find mapping generation info for access point '${accessPointId}' (type: ${dataProductAccessType}) in data product '${dataProductPath}'`,
+        `Can't resolve lakehouse access point '${accessPointId}' in data product '${dataProductPath}'`,
       );
     }
 
-    // Create a dummy mapping for the resolved mapping path
-    const dummyMapping = new V1_Mapping();
-    dummyMapping.package = extractPackagePathFromPath(mappingPath) ?? '';
-    dummyMapping.name = extractElementNameFromPath(mappingPath);
+    const result = new DataProductAnalysis();
+    result.path = dataProductPath;
+    result.title = artifact.dataProduct.title;
+    result.description = artifact.dataProduct.description;
+
+    return new DataProductAnalysisQueryResult(
+      undefined,
+      result,
+      lakehouseResult,
+    );
+  }
+
+  /**
+   * Shared core logic for building data product analysis after the access type
+   * specific resolution has been performed.
+   */
+  private async buildDataProductAnalysisCore(
+    artifact: V1_DataProductArtifact,
+    dataProductPath: string,
+    pureGraph: PureModel,
+    mappingPath: string,
+    accessGroup:
+      | V1_ModelAccessPointGroupInfo
+      | V1_NativeModelExecutionContextInfo,
+    resolvedRuntimePath: string | undefined,
+    projectInfo: ProjectGAVCoordinates,
+    graphReport?: GraphManagerOperationReport,
+  ): Promise<DataProductAnalysisQueryResult> {
+    const modelAccessPointGroups = artifact.accessPointGroups.filter(
+      (group): group is V1_ModelAccessPointGroupInfo =>
+        group instanceof V1_ModelAccessPointGroupInfo,
+    );
+
+    // Collect mapping generation infos from all sources
+    const includedGenInfos = new Map<string, V1_MappingGenerationInfo>();
+
+    if (artifact.nativeModelAccess?.mappingGenerations) {
+      for (const [path, genInfo] of artifact.nativeModelAccess
+        .mappingGenerations) {
+        includedGenInfos.set(path, genInfo);
+      }
+    }
+    for (const group of modelAccessPointGroups) {
+      if (!includedGenInfos.has(group.mappingGeneration.path)) {
+        includedGenInfos.set(
+          group.mappingGeneration.path,
+          group.mappingGeneration,
+        );
+      }
+    }
+
+    const nativeRuntimePaths = new Map<string, string>();
+    if (artifact.nativeModelAccess?.nativeModelExecutionContexts) {
+      for (const ctx of artifact.nativeModelAccess
+        .nativeModelExecutionContexts) {
+        const rtPath = ctx.runtimeGeneration?.path;
+        if (rtPath) {
+          nativeRuntimePaths.set(ctx.key, rtPath);
+        }
+      }
+    }
+
+    // Only add a stub for the resolved mapping to the graph
+    const resolvedMappingStub = new V1_Mapping();
+    resolvedMappingStub.package = extractPackagePathFromPath(mappingPath) ?? '';
+    resolvedMappingStub.name = extractElementNameFromPath(mappingPath);
+
+    // Only add a runtime stub for the resolved native context (if applicable)
+    let resolvedRuntimeStub: V1_PackageableRuntime | undefined;
+    if (resolvedRuntimePath) {
+      resolvedRuntimeStub = new V1_PackageableRuntime();
+      resolvedRuntimeStub.package =
+        extractPackagePathFromPath(resolvedRuntimePath) ?? '';
+      resolvedRuntimeStub.name =
+        extractElementNameFromPath(resolvedRuntimePath);
+      resolvedRuntimeStub.runtimeValue = new V1_EngineRuntime();
+    }
 
     // Create a dummy data product element
     const dummyDataProduct = new V1_DataProduct();
@@ -3928,9 +4029,14 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     dummyDataProduct.name = extractElementNameFromPath(dataProductPath);
     dummyDataProduct.title = artifact.dataProduct.title;
     dummyDataProduct.description = artifact.dataProduct.description;
-    // Build the minimal graph from the resolved mapping's model elements + dummy mapping + dummy data product
-    const graphEntities = resolvedMappingGenInfo.model.elements
-      .concat([dummyMapping])
+
+    // Only load model elements for the resolved mapping
+    const allModelElements: V1_PackageableElement[] =
+      includedGenInfos.get(mappingPath)?.model.elements ?? [];
+
+    const graphEntities = allModelElements
+      .concat(resolvedMappingStub)
+      .concat(resolvedRuntimeStub ? [resolvedRuntimeStub] : [])
       .concat(dummyDataProduct)
       .filter((el) => !pureGraph.getNullableElement(el.path, false))
       .map((el) => this.elementProtocolToEntity(el));
@@ -3949,45 +4055,115 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     );
 
     const data = pureGraph.getDataProduct(dataProductPath);
-    // build access point group
-    let exec: ModelAccessPointGroup | NativeModelExecutionContext;
-    if (accessGroup instanceof V1_ModelAccessPointGroupInfo) {
-      const group = new ModelAccessPointGroup();
-      group.id = accessGroup.id;
-      data.accessPointGroups = [group];
-      group.mapping = PackageableElementExplicitReference.create(
-        pureGraph.getMapping(mappingPath),
+
+    let exec: ModelAccessPointGroup | NativeModelExecutionContext | undefined;
+
+    if (
+      artifact.nativeModelAccess?.defaultExecutionContext &&
+      artifact.nativeModelAccess.nativeModelExecutionContexts.length
+    ) {
+      const na = new NativeModelAccess();
+      na.nativeModelExecutionContexts =
+        artifact.nativeModelAccess.nativeModelExecutionContexts.map(
+          (ctxInfo) => {
+            const ctx = new NativeModelExecutionContext();
+            ctx.key = ctxInfo.key;
+            const ctxMappingPath = ctxInfo.mapping;
+            const ctxMapping =
+              ctxMappingPath === mappingPath
+                ? pureGraph.getMapping(ctxMappingPath)
+                : new Mapping(ctxMappingPath);
+            ctx.mapping =
+              PackageableElementExplicitReference.create(ctxMapping);
+            const rtPath = nativeRuntimePaths.get(ctxInfo.key);
+            if (rtPath) {
+              const ctxRuntime =
+                rtPath === resolvedRuntimePath
+                  ? pureGraph.getRuntime(rtPath)
+                  : new PackageableRuntime(rtPath);
+              ctx.runtime =
+                PackageableElementExplicitReference.create(ctxRuntime);
+            }
+            ctx.__owner = na;
+            return ctx;
+          },
+        );
+      const defaultExecutionContext = na.nativeModelExecutionContexts.find(
+        (ctx) =>
+          ctx.key === artifact.nativeModelAccess?.defaultExecutionContext,
       );
-      exec = group;
-    } else {
-      const nativeAccess = new NativeModelExecutionContext();
-      nativeAccess.key = accessGroup.key;
-      nativeAccess.mapping = PackageableElementExplicitReference.create(
-        pureGraph.getMapping(mappingPath),
+      if (!defaultExecutionContext) {
+        throw new Error(
+          `Can't find execution context matching default context in '${dataProductPath}'`,
+        );
+      }
+      na.defaultExecutionContext = defaultExecutionContext;
+      na.featuredElements = this.resolveArtifactElements(
+        artifact.nativeModelAccess.elements,
+        pureGraph,
       );
-      const na = new NativeModelAccess();
-      na.nativeModelExecutionContexts = [nativeAccess];
       data.nativeModelAccess = na;
-      exec = nativeAccess;
     }
 
-    // Build MappingModelCoverageAnalysisResult for the resolved mapping only
+    if (modelAccessPointGroups.length > 0) {
+      data.accessPointGroups = modelAccessPointGroups.map((groupInfo) => {
+        const group = new ModelAccessPointGroup();
+        group.id = groupInfo.id;
+        const groupMappingPath = groupInfo.mappingGeneration.path;
+        const groupMapping =
+          groupMappingPath === mappingPath
+            ? pureGraph.getMapping(groupMappingPath)
+            : new Mapping(groupMappingPath);
+        group.mapping =
+          PackageableElementExplicitReference.create(groupMapping);
+        group.featuredElements = this.resolveArtifactElements(
+          groupInfo.elements,
+          pureGraph,
+        );
+        return group;
+      });
+    }
+
+    // Resolve the target exec state
+    if (accessGroup instanceof V1_ModelAccessPointGroupInfo) {
+      exec = data.accessPointGroups.find(
+        (g) => g instanceof ModelAccessPointGroup && g.id === accessGroup.id,
+      ) as ModelAccessPointGroup;
+    } else {
+      exec = data.nativeModelAccess?.nativeModelExecutionContexts.find(
+        (ctx) => ctx.key === accessGroup.key,
+      );
+    }
+
+    if (!exec) {
+      throw new Error(
+        `Can't find execution matching access id in '${dataProductPath}'`,
+      );
+    }
+
+    // Build MappingModelCoverageAnalysisResult for all included mappings
     const mappingToMappingCoverageResult = new Map<
       string,
       MappingModelCoverageAnalysisResult
     >();
-    const v1Result = new V1_MappingModelCoverageAnalysisResult();
-    v1Result.mappedEntities = resolvedMappingGenInfo.mappedEntities;
-    v1Result.model = resolvedMappingGenInfo.model;
-    mappingToMappingCoverageResult.set(
-      mappingPath,
-      V1_buildModelCoverageAnalysisResult(
-        v1Result,
-        this,
-        pureGraph.getMapping(mappingPath),
-        resolvedMappingGenInfo.model,
-      ),
-    );
+    for (const [mPath, genInfo] of includedGenInfos) {
+      const v1Result = new V1_MappingModelCoverageAnalysisResult();
+      v1Result.mappedEntities = genInfo.mappedEntities;
+      v1Result.model = genInfo.model;
+      const coverageMapping =
+        mPath === mappingPath
+          ? pureGraph.getMapping(mPath)
+          : new Mapping(mPath);
+      mappingToMappingCoverageResult.set(
+        mPath,
+        V1_buildModelCoverageAnalysisResult(
+          v1Result,
+          this,
+          coverageMapping,
+          genInfo.model,
+        ),
+      );
+    }
 
     // Build the analysis result
     const result = new DataProductAnalysis();
@@ -4495,6 +4671,21 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     );
   }
 
+  async checkServiceRegisteredByPattern(
+    serviceServerUrl: string,
+    servicePattern: string,
+  ): Promise<boolean> {
+    try {
+      await this.engine.getServiceMetadataByPattern(
+        serviceServerUrl,
+        servicePattern,
+      );
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
   async runServicePostValidations(
     service: Service,
     graph: PureModel,