@finos/legend-graph 32.3.2 → 32.3.3

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/from/V1_DataElementTransformer.ts

@@ -23,6 +23,9 @@ import {
   ModelStoreData,
   ModelEmbeddedData,
   ModelInstanceData,
+  RelationElementsData,
+  type RelationElement,
+  type RelationRowTestData,
 } from '../../../../../../../graph/metamodel/pure/data/EmbeddedData.js';
 import {
   type RelationalCSVDataTable,
@@ -38,6 +41,9 @@ import {
   V1_ModelStoreData,
   V1_ModelEmbeddedData,
   V1_ModelInstanceData,
+  V1_RelationElementsData,
+  V1_RelationElement,
+  V1_RelationRowTestData,
 } from '../../../model/data/V1_EmbeddedData.js';
 import {
   V1_RelationalCSVData,
@@ -103,6 +109,39 @@ const V1_transformModelStoreData = (
   return modelStoreDataElement;
 };
 
+const V1_transformRelationRowTestData = (
+  element: RelationRowTestData,
+  context: V1_GraphTransformerContext,
+): V1_RelationRowTestData => {
+  const relationRowTestData = new V1_RelationRowTestData();
+  relationRowTestData.values = element.values;
+  return relationRowTestData;
+};
+
+const V1_transformRelationElement = (
+  element: RelationElement,
+  context: V1_GraphTransformerContext,
+): V1_RelationElement => {
+  const relationElement = new V1_RelationElement();
+  relationElement.paths = element.paths;
+  relationElement.columns = element.columns;
+  relationElement.rows = element.rows.map((row) =>
+    V1_transformRelationRowTestData(row, context),
+  );
+  return relationElement;
+};
+
+const V1_transformRelationElementsData = (
+  element: RelationElementsData,
+  context: V1_GraphTransformerContext,
+): V1_RelationElementsData => {
+  const relationElementsDataElement = new V1_RelationElementsData();
+  relationElementsDataElement.relationElements = element.relationElements.map(
+    (relationElement) => V1_transformRelationElement(relationElement, context),
+  );
+  return relationElementsDataElement;
+};
+
 export const V1_transformExternalFormatData = (
   element: ExternalFormatData,
 ): V1_ExternalFormatData => {
@@ -180,6 +219,8 @@ export function V1_transformEmbeddedData(
     return V1_transformModelStoreData(metamodel, context);
   } else if (metamodel instanceof ExternalFormatData) {
     return V1_transformExternalFormatData(metamodel);
+  } else if (metamodel instanceof RelationElementsData) {
+    return V1_transformRelationElementsData(metamodel, context);
   } else if (metamodel instanceof DataElementReference) {
     return V1_transformDataElementReference(metamodel, context);
   } else if (metamodel instanceof RelationalCSVData) {

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/from/V1_DatabaseTransformer.ts

@@ -15,7 +15,10 @@
  */
 
 import { UnsupportedOperationError } from '@finos/legend-shared';
-import type { Database } from '../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/Database.js';
+import {
+  INTERNAL__LakehouseGeneratedDatabase,
+  type Database,
+} from '../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/Database.js';
 import {
   type RelationalDataType,
   Real,
@@ -61,6 +64,7 @@ import {
 import type { ColumnMapping } from '../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/ColumnMapping.js';
 import type { View } from '../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/View.js';
 import type { Schema } from '../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/Schema.js';
+import { type IncludeStore } from '../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/IncludeStore.js';
 import {
   type V1_RelationalDataType,
   V1_VarChar,
@@ -114,6 +118,7 @@ import {
 import { V1_PackageableElementPointer } from '../../../model/packageableElements/V1_PackageableElement.js';
 import type { TabularFunction } from '../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/TabularFunction.js';
 import { V1_TabularFunction } from '../../../model/packageableElements/store/relational/model/V1_TabularFunction.js';
+import { V1_IncludeStore } from '../../../model/packageableElements/store/relational/model/V1_IncludeStore.js';
 
 const transformRelationalDataType = (
   type: RelationalDataType,
@@ -194,9 +199,17 @@ export const V1_transformTableAliasToTablePointer = (
   },
 ): V1_TablePtr => {
   const tablePtr = new V1_TablePtr();
+  const ownerRef = tableAlias.relation.ownerReference;
   tablePtr.database = options?.TEMPORARY__resolveToFullPath
-    ? tableAlias.relation.ownerReference.value.path
-    : (tableAlias.relation.ownerReference.valueForSerialization ?? '');
+    ? ownerRef.value.path
+    : (ownerRef.valueForSerialization ?? '');
+  // make sure reference is to the owner not the generated database
+  if (
+    ownerRef.value instanceof INTERNAL__LakehouseGeneratedDatabase &&
+    ownerRef.valueForSerialization === ownerRef.value.OWNER.path
+  ) {
+    tablePtr.database = ownerRef.valueForSerialization;
+  }
   // NOTE: Sometimes, we interpret this, so to maintain roundtrip stability, we need to handle this differrently
   // See https://github.com/finos/legend-studio/issues/295
   tablePtr.mainTableDb = tablePtr.database;
@@ -458,6 +471,16 @@ const transformSchema = (
   return schema;
 };
 
+const transformIncludeStore = (element: IncludeStore): V1_IncludeStore => {
+  const v1IncludeStore = new V1_IncludeStore();
+  v1IncludeStore.packageableElementPointer = new V1_PackageableElementPointer(
+    undefined,
+    element.packageableElementPointer.valueForSerialization ?? '',
+  );
+  v1IncludeStore.storeType = element.storeType;
+  return v1IncludeStore;
+};
+
 export const V1_transformDatabase = (
   element: Database,
   context: V1_GraphTransformerContext,
@@ -484,5 +507,9 @@ export const V1_transformDatabase = (
         store.valueForSerialization ?? '',
       ),
   );
+  database.includedStoreSpecifications =
+    element.includedStoreSpecifications.map((includeStore) =>
+      transformIncludeStore(includeStore),
+    );
   return database;
 };

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/from/V1_DomainTransformer.ts

@@ -71,7 +71,7 @@ import {
   V1_FunctionTest,
 } from '../../../model/packageableElements/function/test/V1_FunctionTest.js';
 import { UnsupportedOperationError } from '@finos/legend-shared';
-import { V1_FunctionStoreTestData } from '../../../model/packageableElements/function/test/V1_FunctionStoreTestData.js';
+import { V1_FunctionTestData } from '../../../model/packageableElements/function/test/V1_FunctionTestData.js';
 import { V1_transformEmbeddedData } from './V1_DataElementTransformer.js';
 import { V1_transformTestAssertion } from './V1_TestTransformer.js';
 import { V1_DefaultValue } from '../../../model/packageableElements/domain/V1_DefaultValue.js';
@@ -354,11 +354,11 @@ const V1_transformFunctionSuite = (
   testSuite.doc = element.doc;
   if (element.testData?.length) {
     testSuite.testData = element.testData.map((elementData) => {
-      const pTestData = new V1_FunctionStoreTestData();
+      const pTestData = new V1_FunctionTestData();
       pTestData.doc = elementData.doc;
-      pTestData.store = V1_transformElementReferencePointer(
-        PackageableElementPointerType.STORE,
-        elementData.store,
+      pTestData.packageableElementPointer = V1_transformElementReferencePointer(
+        undefined,
+        elementData.element,
       );
       pTestData.data = V1_transformEmbeddedData(elementData.data, context);
       return pTestData;

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/from/V1_RuntimeTransformer.ts

@@ -98,10 +98,7 @@ const transformEngineRuntime = (
       connection.connectionPointer,
     );
     val.storePointers = connection.storePointers.map((s) =>
-      V1_transformElementReferencePointer(
-        PackageableElementPointerType.STORE,
-        s,
-      ),
+      V1_transformElementReferencePointer(undefined, s),
     );
     return val;
   });

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/to/V1_ClassMappingSecondPassBuilder.ts

@@ -247,7 +247,11 @@ export class V1_ClassMappingSecondPassBuilder
     }
     let mainTableAlias: TableAlias | undefined;
     if (classMapping.mainTable) {
-      const relation = this.context.resolveRelation(classMapping.mainTable);
+      const relation = this.context.resolveRelation(
+        classMapping.mainTable,
+        undefined,
+        true,
+      );
       mainTableAlias = new TableAlias();
       mainTableAlias.relation = relation;
       mainTableAlias.name = relation.value.name;

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/to/V1_ElementSecondPassBuilder.ts

@@ -24,6 +24,7 @@ import {
   LogEvent,
   guaranteeType,
   guaranteeNonNullable,
+  returnUndefOnError,
 } from '@finos/legend-shared';
 import { Stereotype } from '../../../../../../../graph/metamodel/pure/packageableElements/domain/Stereotype.js';
 import { Tag } from '../../../../../../../graph/metamodel/pure/packageableElements/domain/Tag.js';
@@ -63,7 +64,10 @@ import {
   V1_buildMappingInclude,
 } from './helpers/V1_MappingBuilderHelper.js';
 import { V1_buildFlatDataSection } from './helpers/V1_FlatDataStoreBuilderHelper.js';
-import { V1_buildSchema } from './helpers/V1_DatabaseBuilderHelper.js';
+import {
+  V1_buildSchema,
+  V1_initInternalLakehouseGeneratedDatabase,
+} from './helpers/V1_DatabaseBuilderHelper.js';
 import {
   V1_buildConfigurationProperty,
   V1_buildScopeElement,
@@ -94,7 +98,10 @@ import type { V1_ExecutionEnvironmentInstance } from '../../../model/packageable
 import { V1_buildExecutionParameters } from './V1_ExecutionEnvironmentBuilderHelper.js';
 import type { V1_INTERNAL__UnknownPackageableElement } from '../../../model/packageableElements/V1_INTERNAL__UnknownPackageableElement.js';
 import type { V1_INTERNAL__UnknownFunctionActivator } from '../../../model/packageableElements/function/V1_INTERNAL__UnknownFunctionActivator.js';
-import { PackageableElementExplicitReference } from '../../../../../../../graph/metamodel/pure/packageableElements/PackageableElementReference.js';
+import {
+  PackageableElementExplicitReference,
+  PackageableElementImplicitReference,
+} from '../../../../../../../graph/metamodel/pure/packageableElements/PackageableElementReference.js';
 import { generateFunctionPrettyName } from '../../../../../../../graph/helpers/PureLanguageHelper.js';
 import type { DSL_Mapping_PureProtocolProcessorPlugin_Extension } from '../../../../extensions/DSL_Mapping_PureProtocolProcessorPlugin_Extension.js';
 import {
@@ -136,6 +143,7 @@ import {
   V1_buildDataProductLink,
 } from './helpers/V1_DataProductBuilder.js';
 import type { V1_IngestDefinition } from '../../../model/packageableElements/ingest/V1_IngestDefinition.js';
+import { IncludeStore } from '../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/IncludeStore.js';
 
 export class V1_ElementSecondPassBuilder
   implements V1_PackageableElementVisitor<void>
@@ -507,6 +515,25 @@ export class V1_ElementSecondPassBuilder
     database.includes = element.includedStores.map((includedStore) =>
       this.context.resolveDatabase(includedStore.path),
     );
+    database.includedStoreSpecifications =
+      element.includedStoreSpecifications.map((v1IncludeStore) => {
+        const storePath = v1IncludeStore.packageableElementPointer.path;
+        const storeElement = guaranteeNonNullable(
+          returnUndefOnError(() =>
+            this.context.graph.getIngestDefinition(storePath),
+          ) ??
+            returnUndefOnError(() =>
+              this.context.graph.getDataProduct(storePath),
+            ),
+          `Ingest/Data Product ${storePath} not found`,
+        );
+        const includedStore = new IncludeStore(
+          PackageableElementImplicitReference.create(storeElement, storePath),
+          v1IncludeStore.storeType,
+        );
+        V1_initInternalLakehouseGeneratedDatabase(includedStore, database);
+        return includedStore;
+      });
     database.schemas = element.schemas.map((schema) =>
       V1_buildSchema(schema, database, this.context),
     );

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/to/V1_GraphBuilderContext.ts

@@ -25,6 +25,8 @@ import {
   guaranteeNonNullable,
   assertNonEmptyString,
   guaranteeType,
+  returnUndefOnError,
+  AssertionError,
 } from '@finos/legend-shared';
 import { GenericType } from '../../../../../../../graph/metamodel/pure/packageableElements/domain/GenericType.js';
 import type { PackageableElement } from '../../../../../../../graph/metamodel/pure/packageableElements/PackageableElement.js';
@@ -49,7 +51,10 @@ import type { ConcreteFunctionDefinition } from '../../../../../../../graph/meta
 import type { Store } from '../../../../../../../graph/metamodel/pure/packageableElements/store/Store.js';
 import type { Service } from '../../../../../../../graph/metamodel/pure/packageableElements/service/Service.js';
 import type { FlatData } from '../../../../../../../graph/metamodel/pure/packageableElements/store/flatData/model/FlatData.js';
-import type { Database } from '../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/Database.js';
+import type {
+  Database,
+  INTERNAL__LakehouseGeneratedDatabase,
+} from '../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/Database.js';
 import type { PackageableConnection } from '../../../../../../../graph/metamodel/pure/packageableElements/connection/PackageableConnection.js';
 import type { PackageableRuntime } from '../../../../../../../graph/metamodel/pure/packageableElements/runtime/PackageableRuntime.js';
 import type { FileGenerationSpecification } from '../../../../../../../graph/metamodel/pure/packageableElements/fileGeneration/FileGenerationSpecification.js';
@@ -430,6 +435,10 @@ export class V1_GraphBuilderContext {
 
   resolveRelation = (
     tablePtr: V1_TablePtr,
+    contextDatabse?:
+      | Map<string, INTERNAL__LakehouseGeneratedDatabase>
+      | undefined,
+    allowImplicitToGeneratedDatabse?: boolean | undefined,
   ): ViewImplicitReference | TableImplicitReference => {
     assertNonEmptyString(
       tablePtr.database,
@@ -443,13 +452,47 @@ export class V1_GraphBuilderContext {
       tablePtr.table,
       `Table pointer 'table' field is missing or empty`,
     );
-    const ownerReference = this.resolveDatabase(tablePtr.database);
-    const value = V1_getRelation(
-      ownerReference.value,
-      tablePtr.schema,
-      tablePtr.table,
+    const ownerReference = this.resolveDatabase(
+      tablePtr.database,
+      contextDatabse,
     );
-    return createImplicitRelationReference(ownerReference, value);
+
+    try {
+      const value = V1_getRelation(
+        ownerReference.value,
+        tablePtr.schema,
+        tablePtr.table,
+      );
+      return createImplicitRelationReference(ownerReference, value);
+    } catch (error) {
+      if (
+        allowImplicitToGeneratedDatabse &&
+        ownerReference.value.includedStoreSpecifications.length > 0
+      ) {
+        const generatedDatabase =
+          ownerReference.value.includedStoreSpecifications.find((_s) =>
+            _s.generatedDatabase.schemas.find(
+              (_schema) => _schema.name === tablePtr.schema,
+            ),
+          )?.generatedDatabase ??
+          guaranteeNonNullable(
+            ownerReference.value.includedStoreSpecifications[0]
+              ?.generatedDatabase,
+          );
+        const value = V1_getRelation(
+          generatedDatabase,
+          tablePtr.schema,
+          tablePtr.table,
+        );
+        // we want the reference to be of the table pointer database
+        const ref = PackageableElementImplicitReference.create(
+          generatedDatabase,
+          tablePtr.database,
+        );
+        return createImplicitRelationReference(ref, value);
+      }
+      throw error;
+    }
   };
 
   resolveJoin = (joinPtr: V1_JoinPointer): JoinImplicitReference => {
@@ -559,11 +602,28 @@ export class V1_GraphBuilderContext {
     );
   resolveDatabase = (
     path: string,
-  ): PackageableElementImplicitReference<Database> =>
-    this.createImplicitPackageableElementReference(
-      path,
-      this.graph.getDatabase,
+    generatedDatabasesInScope?:
+      | Map<string, INTERNAL__LakehouseGeneratedDatabase>
+      | undefined,
+  ): PackageableElementImplicitReference<Database> => {
+    const element = returnUndefOnError(() =>
+      this.createImplicitPackageableElementReference(
+        path,
+        this.graph.getDatabase,
+      ),
     );
+    if (element) {
+      return element;
+    }
+    if (generatedDatabasesInScope?.has(path)) {
+      const generatedDb = generatedDatabasesInScope.get(path);
+      if (generatedDb) {
+        return PackageableElementImplicitReference.create(generatedDb, path);
+      }
+    }
+    throw new AssertionError(`Can't find database ${path}`);
+  };
+
   resolveMapping = (
     path: string,
   ): PackageableElementImplicitReference<Mapping> =>

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/to/V1_PropertyMappingBuilder.ts

@@ -695,6 +695,8 @@ export class V1_PropertyMappingBuilder
               this.context,
               this.tableAliasIndex,
               [],
+              undefined,
+              true,
             ),
             new V1_GraphTransformerContextBuilder(
               this.context.extensions.plugins,
@@ -928,6 +930,8 @@ export class V1_PropertyMappingBuilder
         this.context,
         this.tableAliasIndex,
         [],
+        undefined,
+        true,
       ),
     );
     embedded.propertyMappings = protocol.classMapping.propertyMappings.map(
@@ -992,6 +996,8 @@ export class V1_PropertyMappingBuilder
         this.context,
         this.tableAliasIndex,
         [],
+        undefined,
+        true,
       ),
     );
     otherwiseEmbedded.propertyMappings =

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/to/helpers/V1_DataElementBuilderHelper.ts

@@ -22,6 +22,9 @@ import {
   ModelStoreData,
   ModelEmbeddedData,
   ModelInstanceData,
+  RelationElementsData,
+  RelationElement,
+  RelationRowTestData,
 } from '../../../../../../../../graph/metamodel/pure/data/EmbeddedData.js';
 import {
   RelationalCSVData,
@@ -36,6 +39,7 @@ import {
   type V1_ExternalFormatData,
   type V1_ModelStoreData,
   V1_ModelEmbeddedData,
+  type V1_RelationElementsData,
 } from '../../../../model/data/V1_EmbeddedData.js';
 import type { V1_RelationalCSVData } from '../../../../model/data/V1_RelationalCSVData.js';
 import type { V1_GraphBuilderContext } from '../V1_GraphBuilderContext.js';
@@ -86,6 +90,24 @@ class V1_EmbeddedDataBuilder implements V1_EmbeddedDataVisitor<EmbeddedData> {
     return metamodel;
   }
 
+  visit_RelationElementsData(data: V1_RelationElementsData): EmbeddedData {
+    const metamodel = new RelationElementsData();
+    metamodel.relationElements = data.relationElements.map(
+      (relationElement) => {
+        const v = new RelationElement();
+        v.columns = relationElement.columns;
+        v.paths = relationElement.paths;
+        v.rows = relationElement.rows.map((row) => {
+          const r = new RelationRowTestData();
+          r.values = row.values;
+          return r;
+        });
+        return v;
+      },
+    );
+    return metamodel;
+  }
+
   visit_ModelStoreData(modelStoreData: V1_ModelStoreData): EmbeddedData {
     const metamodel = new ModelStoreData();
     if (modelStoreData.modelData?.length) {

package/src/graph-manager/protocol/pure/v1/transformation/pureGraph/to/helpers/V1_DatabaseBuilderHelper.ts

@@ -23,8 +23,12 @@ import {
   guaranteeType,
   assertTrue,
   isNonNullable,
+  AssertionError,
 } from '@finos/legend-shared';
-import { Database } from '../../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/Database.js';
+import {
+  Database,
+  INTERNAL__LakehouseGeneratedDatabase,
+} from '../../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/Database.js';
 import {
   getAllIncludedDatabases,
   getColumn,
@@ -135,6 +139,12 @@ import { TablePtr } from '../../../../../../../../graph/metamodel/pure/packageab
 import type { TabularFunction } from '../../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/TabularFunction.js';
 import type { V1_TabularFunction } from '../../../../model/packageableElements/store/relational/model/V1_TabularFunction.js';
 import { V1_buildTaggedValue } from './V1_DomainBuilderHelper.js';
+import type { IncludeStore } from '../../../../../../../../graph/metamodel/pure/packageableElements/store/relational/model/IncludeStore.js';
+import {
+  buildGeneratedIndex,
+  getOrCreateSchemaFromGeneratedDatabase,
+  getOrCreateTableFromGeneratedSchema,
+} from '../../../../../../../../graph/helpers/STO_Internal_Relational_Helper.js';
 
 const _schemaExists = (
   db: Database,
@@ -169,32 +179,80 @@ const schemaExists = (database: Database, _schema: string): boolean =>
   DEFAULT_DATABASE_SCHEMA_NAME === _schema ||
   _schemaExists(database, _schema, new Set<Database>());
 
-export const V1_findSchema = (database: Database, _schema: string): void =>
-  assertTrue(
-    schemaExists(database, _schema),
+const V1_findSchemaInGeneratedDatabase = (
+  schemaName: string,
+  db: INTERNAL__LakehouseGeneratedDatabase,
+): Schema | undefined => {
+  let schema = db.schemas.find((s) => s.name === schemaName);
+  if (!schema) {
+    schema = new Schema(schemaName, db);
+    db.schemas.push(schema);
+  }
+  db.schemas.push(schema);
+  return schema;
+};
+
+export const V1_findSchema = (database: Database, _schema: string): void => {
+  if (schemaExists(database, _schema)) {
+    return;
+  } else if (database instanceof INTERNAL__LakehouseGeneratedDatabase) {
+    V1_findSchemaInGeneratedDatabase(_schema, database);
+    return;
+  }
+  throw new AssertionError(
     `Can't find schema '${_schema}' in database '${database}'`,
   );
+};
+
+function findRelationInSchema(
+  schema: Schema,
+  tableName: string,
+): Relation | undefined {
+  let relation: Relation | undefined = schema.tables.find(
+    (table) => table.name === tableName,
+  );
+  if (!relation) {
+    relation = schema.views.find((view) => view.name === tableName);
+  }
+  if (!relation) {
+    relation = schema.tabularFunctions.find(
+      (tabularFunction) => tabularFunction.name === tableName,
+    );
+  }
+  return relation;
+}
+
+export const V1_initInternalLakehouseGeneratedDatabase = (
+  includedStore: IncludeStore,
+  owner: Database,
+): INTERNAL__LakehouseGeneratedDatabase => {
+  const generatedDatabase = new INTERNAL__LakehouseGeneratedDatabase(
+    includedStore.packageableElementPointer.value,
+    owner,
+  );
+  const defaultSchema = new Schema(
+    DEFAULT_DATABASE_SCHEMA_NAME,
+    generatedDatabase,
+  );
+  generatedDatabase.schemas.push(defaultSchema);
+  includedStore.generatedDatabase = generatedDatabase;
+  return generatedDatabase;
+};
 
 export const V1_findRelation = (
   database: Database,
   schemaName: string,
   tableName: string,
 ): Relation | undefined => {
+  if (database instanceof INTERNAL__LakehouseGeneratedDatabase) {
+    const schema = getOrCreateSchemaFromGeneratedDatabase(schemaName, database);
+    return getOrCreateTableFromGeneratedSchema(tableName, schema);
+  }
   const relations: Relation[] = [];
   getAllIncludedDatabases(database).forEach((db) => {
     const schema = db.schemas.find((_schema) => _schema.name === schemaName);
     if (schema) {
-      let relation: Relation | undefined = schema.tables.find(
-        (table) => table.name === tableName,
-      );
-      if (!relation) {
-        relation = schema.views.find((view) => view.name === tableName);
-      }
-      if (!relation) {
-        relation = schema.tabularFunctions.find(
-          (tabularFunction) => tabularFunction.name === tableName,
-        );
-      }
+      const relation = findRelationInSchema(schema, tableName);
       if (relation) {
         relations.push(relation);
       }
@@ -257,6 +315,8 @@ export const V1_buildRelationalOperationElement = (
   context: V1_GraphBuilderContext,
   tableAliasIndex: Map<string, TableAlias>,
   selfJoinTargets: TableAliasColumn[],
+  generatedDbs?: Map<string, INTERNAL__LakehouseGeneratedDatabase> | undefined,
+  allowImplicitToGeneratedDatabase?: boolean | undefined,
 ): RelationalOperationElement => {
   if (operationalElement instanceof V1_TableAliasColumn) {
     if (operationalElement.table.table === SELF_JOIN_TABLE_NAME) {
@@ -265,7 +325,11 @@ export const V1_buildRelationalOperationElement = (
       selfJoinTargets.push(selfJoin);
       return selfJoin;
     }
-    const relation = context.resolveRelation(operationalElement.table);
+    const relation = context.resolveRelation(
+      operationalElement.table,
+      generatedDbs,
+      allowImplicitToGeneratedDatabase,
+    );
     const aliasName = `${operationalElement.table.schema}.${operationalElement.tableAlias}`;
     if (!tableAliasIndex.has(aliasName)) {
       const tAlias = new TableAlias();
@@ -274,7 +338,7 @@ export const V1_buildRelationalOperationElement = (
       tableAliasIndex.set(aliasName, tAlias);
     }
     const columnReference = ColumnImplicitReference.create(
-      context.resolveDatabase(operationalElement.table.database),
+      context.resolveDatabase(operationalElement.table.database, generatedDbs),
       getColumn(relation.value, operationalElement.column),
     );
     const tableAliasColumn = new TableAliasColumn();
@@ -297,6 +361,7 @@ export const V1_buildRelationalOperationElement = (
           context,
           new Map<string, TableAlias>(),
           selfJoinTargets,
+          generatedDbs,
         );
     }
     return elementWithJoins;
@@ -308,6 +373,7 @@ export const V1_buildRelationalOperationElement = (
         context,
         tableAliasIndex,
         selfJoinTargets,
+        generatedDbs,
       ),
     );
     return dynFunc;
@@ -320,6 +386,7 @@ export const V1_buildRelationalOperationElement = (
           context,
           tableAliasIndex,
           selfJoinTargets,
+          generatedDbs,
         ),
       );
     }
@@ -334,6 +401,7 @@ export const V1_buildRelationalOperationElement = (
             context,
             tableAliasIndex,
             selfJoinTargets,
+            generatedDbs,
           ),
         );
       }
@@ -649,6 +717,7 @@ export const V1_buildDatabaseJoin = (
       context,
       tableAliasIndex,
       selfJoinTargets,
+      buildGeneratedIndex(database),
     ),
   );
   const aliases = Array.from(tableAliasIndex.values());
@@ -687,6 +756,9 @@ export const V1_buildDatabaseJoin = (
         col = existingRelationalElement.value.columns.find(
           (c) => c instanceof Column && c.name === columnName,
         ) as Column | undefined;
+      } else if (database instanceof INTERNAL__LakehouseGeneratedDatabase) {
+        col = new Column();
+        col.name = columnName as string;
       }
       assertNonNullable(col, `Can't find column '${columnName}' in the table`);
       // NOTE: this should be `implicit` because we do some inferencing