dsl-to-sql 1.0.0-fabric-1p-development.1

package/src/query-builder/QueryBuilder.ts

@@ -0,0 +1,1035 @@
+import {
+  IMDMColumnConfigWithParsedMeta,
+  MDM_COLUMN_TYPE,
+  ResolvedExpressionType,
+  CteExpression,
+} from '../js-lib/ParseContext';
+import {
+  FilterConfig,
+  JoinExpressions,
+  Rule,
+  RuleGroup,
+  ISuperFilterConfig,
+  IGroupFilter,
+  IManualFilterConfig,
+} from '../filters/filter-types';
+import { RUNTIME_TABLE_NAME } from '../runtime_var';
+import { PaginationBuilder, createPaginationBuilder } from './PaginationBuilder';
+import { From, RawSqlExpression, Select } from '../sql-types';
+import { SelectBuilder } from '../sql-lib/select';
+import { toColumn } from '../sql-lib/column';
+import { processSelect } from '../sql_query_gen';
+import {
+  getCurrentDate,
+  getCurrentDateTime,
+  getEntityNameFormatter,
+  getFormattedTableName,
+  getJoinExpressions,
+  resolveRuntimeVariables,
+  stripWhereClause,
+} from '../utils';
+import { SuperFilterBuilder } from './SuperFilterBuilder';
+
+export type SortConfig = { column: string; order: 'ASC' | 'DESC' }[];
+
+export type PaginationConfig = { skip: number; take: number };
+
+export type FilterResolverParams = {
+  filterConfig: FilterConfig;
+  schema?: IColumnSchemaItem[];
+};
+
+export interface QueryResolver {
+  filterResolver(params: FilterResolverParams): string;
+}
+
+export type RuntimeVariables = {
+  $RUNTIME_LOGGEDIN_NAME: string;
+  $RUNTIME_LOGGEDIN_EMAIL: string;
+  $RUNTIME_CURRENT_DATE?: string;
+  $RUNTIME_CURRENT_DATETIME?: string;
+};
+
+export type QueryBuilderOptions = {
+  selectedColumns: string[];
+  databaseDetails: Record<string, any>;
+  primaryKeyColumns: string[];
+  filterConfig?: FilterConfig;
+  pagination?: PaginationConfig;
+  sortConfig?: SortConfig;
+  searchExpression?: string;
+  distinct?: boolean;
+  columnConfigMap: Record<string, IMDMColumnConfigWithParsedMeta>;
+  queryResolver: QueryResolver; // instance of BaseDB in database-service-link
+  runtimeVariables: RuntimeVariables;
+  schema?: IColumnSchemaItem[];
+  includeFinalCTESelectQuery?: boolean;
+  finalResponseAsObject?: boolean;
+  superFilters?: ISuperFilterConfig;
+  skipCountForRankingFilter?: boolean;
+  joinClauses?: JoinExpressions;
+  manualFilterOptions?: IManualFilterConfig[];
+  enableLastSelectCTEWrap?: boolean;
+};
+
+type BuildState = {
+  pendingFilterConfig: FilterConfig;
+  pendingSortConfig: SortConfig;
+  isSortPending: boolean;
+  isPaginationPending: boolean;
+  isSearchPending: boolean;
+  hasFormulaColumns: boolean;
+  isSuperFilterPending: boolean;
+};
+
+type BuildOptions = Partial<Omit<QueryBuilderOptions, 'tableName' | 'queryResolver' | 'schema'>>;
+
+type MainCteExpression = {
+  type: ResolvedExpressionType.CTE;
+  cteName: string;
+  value: Select;
+};
+
+export type FinalQueryResult = {
+  finalQuery: string;
+  finalCteName?: string; // For group by feat need to pass cte name
+};
+
+type CalculationOrderResult = {
+  nativeColumns: Set<string>;
+  formulaColumnsByDependencyLevel: IMDMColumnConfigWithParsedMeta[][];
+};
+
+export enum DEFAULT_VALUE_TYPE {
+  MANUAL = 'MANUAL',
+  DERIVED = 'DERIVED',
+  NONE = 'NONE',
+}
+
+type IColumnSchemaItem = {
+  name: string;
+  type: string;
+  defaultValue?: string | number;
+  isNullable?: boolean;
+  isIdentity?: boolean;
+};
+
+export const MAX_STRING_TYPE = 'nvarchar(max)';
+
+/**
+ * Steps:
+ * 1. Separate out filters for native columns and formula columns
+ * 2. Apply filters for native columns in the main CTE. If Native columns have sorting config then -
+ *      - If SortConfig exists for Formula columns with cte, do not apply sorting
+ *      - If SortConfig exists for Formula columns without cte, apply sorting
+ *      - If SortConfig does not exist for Formula columns, apply sorting
+ * 3. Apply filters for formula columns in the CTE followed by the column specific CTEs due to alias not supported in where clause
+ * 4. Apply SortConfig and pagination in the CTE with last filter
+ * 5. Perform final projection and apply SortConfig and Pagination, if still exists
+ */
+export class QueryBuilder {
+  private readonly paginationBuilder: PaginationBuilder;
+  private readonly options: QueryBuilderOptions;
+
+  constructor(options: QueryBuilderOptions) {
+    this.options = options;
+    this.paginationBuilder = createPaginationBuilder();
+  }
+
+  build(options: BuildOptions = {}): FinalQueryResult {
+    const combinedOptions = { ...this.options, ...options };
+    const impl = new QueryBuilderImpl(combinedOptions, this.paginationBuilder);
+    return impl.build();
+  }
+
+  buildCountQuery(options: BuildOptions = {}): FinalQueryResult {
+    const combinedOptions = {
+      ...this.options,
+      ...options,
+      sortConfig: undefined, // not required for count query
+      pagination: undefined, // not required for count query
+    };
+    const impl = new QueryBuilderImpl(combinedOptions, this.paginationBuilder);
+    return impl.build(true);
+  }
+}
+
+/**
+ * Implementation class for QueryBuilder
+ *
+ * @private exposed for testing
+ */
+export class QueryBuilderImpl {
+  private readonly paginationBuilder: PaginationBuilder;
+  private readonly options: QueryBuilderOptions;
+  private buildState: BuildState;
+  private readonly entityNameFormatter: (name: string, type?: string) => string;
+
+  constructor(options: QueryBuilderOptions, paginationBuilder: PaginationBuilder) {
+    this.options = {
+      ...options,
+      searchExpression: options.searchExpression
+        ? stripWhereClause(options.searchExpression)
+        : undefined,
+      sortConfig: this.getRelevantSortConfig(options.sortConfig, new Set(options.selectedColumns)),
+    };
+    this.paginationBuilder = paginationBuilder;
+    this.buildState = {
+      pendingFilterConfig: {
+        ...(options.filterConfig ?? { combinator: 'and', not: false }),
+        rules: [],
+      },
+      hasFormulaColumns: false,
+      pendingSortConfig: [],
+      isSortPending: true,
+      isPaginationPending: true,
+      isSearchPending: options.searchExpression !== undefined,
+      isSuperFilterPending: options.superFilters !== undefined,
+    };
+    this.entityNameFormatter = getEntityNameFormatter();
+    // Set runtime variables with current date/time values
+    Object.assign(this.options.runtimeVariables, {
+      $RUNTIME_CURRENT_DATE: getCurrentDate(),
+      $RUNTIME_CURRENT_DATETIME: getCurrentDateTime(),
+    });
+  }
+
+  build(isCountQuery = false): FinalQueryResult {
+    const { joinClauses, superFilters } = this.options;
+    const { joinRelation = [] } = joinClauses || {};
+    const requiredColumns = this.filterRequiredColumns();
+    this.addColumnsFromFilterConfig(requiredColumns);
+    const { nativeColumns, formulaColumnsByDependencyLevel } =
+      this.getCalculationOrder(requiredColumns);
+    this.buildState.hasFormulaColumns = formulaColumnsByDependencyLevel.length > 0;
+    this.buildState.isSuperFilterPending = superFilters !== undefined;
+
+    // build main CTE and intermediate CTE for formula columns join relation
+    const mainCteFormulas = joinRelation.length > 0 ? [] : formulaColumnsByDependencyLevel;
+    const columnExprs = this.retrieveColumnExprsInFront(mainCteFormulas);
+    const mainCTE = this.buildMainCTE(nativeColumns, columnExprs);
+    const intermediateCTE = this.buildIntermediateCTE(
+      mainCTE.cteName,
+      formulaColumnsByDependencyLevel,
+    );
+    return this.buildFinalProjection(mainCTE, intermediateCTE, isCountQuery);
+  }
+
+  /**
+   * Returns the sort config with only the columns that are part of the requested columns
+   */
+  private getRelevantSortConfig(
+    sortConfig: SortConfig | undefined,
+    requestedColumns: Set<string>,
+  ): SortConfig | undefined {
+    return sortConfig?.filter((s) => requestedColumns.has(s.column));
+  }
+
+  /**
+   * Adds columns used in the filter config to the set of all columns
+   */
+  private addColumnsFromFilterConfig(requiredColumns: Set<string>): void {
+    const { filterConfig } = this.options;
+
+    if (!filterConfig) return;
+
+    const traverse = (rule: Rule | RuleGroup) => {
+      if ('column' in rule) {
+        requiredColumns.add(rule.column);
+      } else if ('rules' in rule) {
+        rule.rules.forEach(traverse);
+      }
+    };
+    filterConfig.rules.forEach(traverse);
+  }
+
+  private retrieveColumnExprsInFront(
+    calcOrderMembers: IMDMColumnConfigWithParsedMeta[][],
+  ): IMDMColumnConfigWithParsedMeta[] {
+    if (calcOrderMembers.length === 0) return [];
+
+    let lastColumnExprIndex = -1;
+    const firstCalcOrderMembers = calcOrderMembers[0];
+    for (let i = 0; i < firstCalcOrderMembers.length; i++) {
+      if (
+        firstCalcOrderMembers[i].columnMeta.sqlQueryProps?.resolvedExpression.type ===
+        ResolvedExpressionType.COLUMN
+      ) {
+        lastColumnExprIndex = i;
+      } else {
+        break;
+      }
+    }
+
+    if (lastColumnExprIndex === -1) {
+      return [];
+    }
+
+    const columnExprsInFront = firstCalcOrderMembers.splice(0, lastColumnExprIndex + 1);
+    if (firstCalcOrderMembers.length === 0) {
+      calcOrderMembers.shift();
+    }
+    return columnExprsInFront;
+  }
+
+  private buildMainCTE(
+    nativeColumns: Set<string>,
+    columnExprs: IMDMColumnConfigWithParsedMeta[],
+  ): MainCteExpression {
+    const columnsInMainCTE = new Set([...nativeColumns, ...columnExprs.map((c) => c.columnName)]);
+    const selectBuilder = new SelectBuilder();
+    const { joinClauses } = this.options;
+    const { alias } = joinClauses || {};
+    const tableAlias = alias ? `${this.entityNameFormatter(alias)}.` : '';
+
+    const columns = Array.from(nativeColumns).map((c) => {
+      const schemaMatch = this.options.schema?.find((s) => s.name === c);
+      return toColumn({
+        type: 'sql_expr',
+        value: `${tableAlias}` + this.entityNameFormatter(c, schemaMatch?.type ?? ''),
+        meta: { rawColumnName: c },
+      });
+    });
+
+    if (columnExprs.length) {
+      columnExprs.forEach((c) => {
+        const schemaMatch = this.options.schema?.find((s) => s.name === c.columnName);
+        let resolvedExpressionValue =
+          c.columnMeta.sqlQueryProps?.resolvedExpression?.value?.join(', ');
+        if (resolvedExpressionValue) {
+          resolvedExpressionValue = resolveRuntimeVariables(
+            resolvedExpressionValue,
+            this.options.runtimeVariables,
+          );
+        }
+        columns.push(
+          toColumn(
+            {
+              type: 'sql_expr',
+              value: resolvedExpressionValue ?? '',
+              meta: { rawColumnName: c.columnName },
+            },
+            this.entityNameFormatter(c.columnName, schemaMatch?.type ?? ''),
+          ),
+        );
+      });
+    }
+
+    const formattedTableName = getFormattedTableName(this.options.databaseDetails);
+    selectBuilder.addColumns(columns);
+    selectBuilder.addFrom([
+      {
+        type: 'from',
+        table: `${formattedTableName}${alias ? ` AS ${this.entityNameFormatter(alias)}` : ''}`,
+      } as From,
+    ]);
+
+    // Build filter condition
+    const where = [];
+    const filterCondition = this.getFilterExpression(columnsInMainCTE);
+    if (filterCondition) {
+      where.push(filterCondition);
+    }
+
+    if (
+      this.buildState.isSearchPending &&
+      this.buildState.isSuperFilterPending === false &&
+      this.buildState.hasFormulaColumns === false
+    ) {
+      this.buildState.isSearchPending = false;
+      where.push(this.options.searchExpression);
+    }
+    if (where.length) {
+      selectBuilder.addWhere({
+        type: 'sql_expr',
+        value: where.join(' AND '),
+      } as RawSqlExpression);
+    }
+
+    // Build sort expression
+    const sortExpression = this.getSortExpression(columnsInMainCTE);
+    if (sortExpression) {
+      selectBuilder.addOrderby([{ type: 'sql_expr', value: sortExpression } as RawSqlExpression]);
+    }
+
+    // Build pagination expression
+    const paginationExpression = this.getPaginationExpression();
+    if (paginationExpression) {
+      selectBuilder.addLimit({
+        type: 'sql_expr',
+        value: paginationExpression,
+      } as RawSqlExpression);
+    }
+
+    return {
+      type: ResolvedExpressionType.CTE,
+      cteName: 'cte_main',
+      value: selectBuilder.build(),
+    };
+  }
+
+  /**
+   * Returns the filter expression for the selected columns. If column type is formula,
+   * stores the rules in pending filter config and process it in the next call to getFilterExpression
+   * else processes immediately.
+   *
+   *  Note - Pass empty set to process any pending filter config
+   */
+  private getFilterExpression(columns: Set<string>): string | undefined {
+    const { filterConfig, columnConfigMap, queryResolver, databaseDetails } = this.options;
+    if (!filterConfig) {
+      return;
+    }
+
+    // extract pending filter config
+    const filterConfigToProcess = this.buildState.pendingFilterConfig;
+
+    // Reset pending filter config
+    this.buildState.pendingFilterConfig = {
+      rules: [],
+      combinator: filterConfig.combinator,
+      not: filterConfig.not,
+    };
+
+    if (columns.size) {
+      const nativeColumnRules: (Rule | RuleGroup)[] = [];
+      const formulaColumnRules: (Rule | RuleGroup)[] = [];
+      // Get rules for selected columns from original filter config
+      filterConfig.rules.forEach((rule) => {
+        const columnName = (rule as Rule).column ?? (rule as RuleGroup).rules[0].column;
+        if (columns.has(columnName)) {
+          if (columnConfigMap[columnName].columnType === MDM_COLUMN_TYPE.FORMULA) {
+            formulaColumnRules.push(rule);
+
+            const schema = this.options.schema ?? [];
+            const exists = schema.some((col) => col.name === columnName);
+            if (!exists) {
+              schema.push({
+                name: columnName,
+                type: MAX_STRING_TYPE,
+              });
+            }
+          } else {
+            nativeColumnRules.push(rule);
+          }
+        }
+      });
+
+      // if column type is formula, store the rules in pending filter config else process immediately
+      if (formulaColumnRules.length) {
+        this.buildState.pendingFilterConfig.rules = formulaColumnRules;
+      }
+      if (nativeColumnRules.length) {
+        filterConfigToProcess.rules = filterConfigToProcess.rules.concat(nativeColumnRules);
+      }
+    }
+
+    if (filterConfigToProcess.rules.length) {
+      const resolvedFilterQuery = queryResolver.filterResolver({
+        filterConfig: filterConfigToProcess,
+        schema: this.options.schema,
+      });
+      return stripWhereClause(resolvedFilterQuery);
+    }
+  }
+
+  /**
+   * Returns the sort expression if each column in the sort config is part of selected columns
+   */
+  private getSortExpression(columns: Set<string>): string | undefined {
+    const { sortConfig, joinClauses } = this.options;
+    const { alias } = joinClauses || {};
+    if (!sortConfig || sortConfig.length === 0) {
+      this.buildState.isSortPending = false;
+      return;
+    }
+
+    if (this.buildState.isSortPending === false) {
+      return;
+    }
+
+    const selectedSortColumns = sortConfig.filter((s) => columns.has(s.column));
+    this.buildState.pendingSortConfig = (this.buildState.pendingSortConfig ?? []).concat(
+      selectedSortColumns,
+    );
+
+    const hasPendingFilters = this.buildState.pendingFilterConfig.rules.length > 0;
+    if (
+      this.buildState.pendingSortConfig.length === sortConfig.length &&
+      !hasPendingFilters &&
+      !this.buildState.isSearchPending &&
+      !this.buildState.isSuperFilterPending
+    ) {
+      this.buildState.isSortPending = false;
+      return `${sortConfig.map((s) => `${alias && !this.buildState.isSuperFilterPending ? `${this.entityNameFormatter(alias)}.` : ''}${this.entityNameFormatter(s.column)} ${s.order}`).join(', ')}`;
+    }
+  }
+
+  /**
+   * Returns the pagination expression if the pagination config is set and sort is not pending.
+   * It must be called after getSortExpression.
+   */
+  private getPaginationExpression(): string | undefined {
+    const { pagination } = this.options;
+    if (!pagination) {
+      this.buildState.isPaginationPending = false;
+      return;
+    }
+
+    // If pagination is already processed, return
+    if (this.buildState.isPaginationPending === false) {
+      return;
+    }
+
+    // If sort is not processed, throw error
+    if (this.buildState.pendingSortConfig === undefined) {
+      throw new Error('getSortExpression() must be called before getPaginationExpression()');
+    }
+
+    // If sort is not complete, return
+    if (this.buildState.isSortPending || this.buildState.isSearchPending) {
+      this.buildState.isPaginationPending = true;
+      return;
+    }
+
+    // If pagination can be completed, return
+    this.buildState.isPaginationPending = false;
+    return this.paginationBuilder.generateQuery(pagination.skip, pagination.take);
+  }
+
+  /**
+   * Builds the intermediate CTEs for the formula columns
+   * 1. Replace table name with previous cte name
+   * 2. Apply filter condition
+   * 3. Apply sort expression, if applicable
+   * 4. Apply pagination, if applicable
+   * 5. Return CTEs/Column expressions
+   */
+  private buildIntermediateCTE(
+    previousCTEName: string,
+    formulaColumnsInCalcOrder: IMDMColumnConfigWithParsedMeta[][],
+  ): CteExpression {
+    let parentCTEName = previousCTEName;
+    const tableNameRegex = new RegExp(`\\${RUNTIME_TABLE_NAME}`, 'g');
+
+    const cteParts: string[] = [];
+    formulaColumnsInCalcOrder.forEach((stageMembers) => {
+      const pendingColumns: IMDMColumnConfigWithParsedMeta[] = [];
+
+      stageMembers.forEach((column) => {
+        const {
+          columnName,
+          columnMeta: { sqlQueryProps },
+        } = column;
+        const isCteExpression =
+          sqlQueryProps?.resolvedExpression?.type === ResolvedExpressionType.CTE;
+        const intermediateParts = [];
+
+        if (isCteExpression) {
+          // add staging cte to process any pending filters, sort, pagination
+          const stagingCte = this.addStagingCTE(parentCTEName, pendingColumns);
+          if (stagingCte) {
+            pendingColumns.length = 0;
+            parentCTEName = stagingCte!.cteName;
+            intermediateParts.push(stagingCte.expr);
+          }
+
+          // process current column cte
+          const sqlParts = this.buildSqlQueryParts([columnName]);
+          const expr = sqlQueryProps.resolvedExpression as CteExpression;
+          let resolvedCte = expr.value.join(', ').replace(tableNameRegex, parentCTEName);
+          if (resolvedCte) {
+            resolvedCte = resolveRuntimeVariables(resolvedCte, this.options.runtimeVariables);
+          }
+          parentCTEName = expr.cteName;
+          intermediateParts.push(resolvedCte);
+          if (sqlParts.length) {
+            const cteName = `cte_post_${parentCTEName}`;
+            const postCte = `${cteName} AS (SELECT * FROM ${parentCTEName}${sqlParts.length ? ' ' + sqlParts.join(' ') : ''})`;
+            intermediateParts.push(postCte);
+            parentCTEName = cteName;
+          }
+          cteParts.push(intermediateParts.join(', '));
+        } else {
+          pendingColumns.push(column);
+        }
+      });
+
+      // add staging cte if there are pending columns at the end of stage.
+      if (pendingColumns.length) {
+        const stagingCte = this.addStagingCTE(parentCTEName, pendingColumns);
+        pendingColumns.length = 0;
+        if (stagingCte) {
+          cteParts.push(stagingCte.expr);
+          parentCTEName = stagingCte.cteName;
+        }
+      }
+    });
+
+    return { type: ResolvedExpressionType.CTE, cteName: parentCTEName, value: cteParts };
+  }
+
+  private addStagingCTE(
+    parentCTEName: string,
+    columnConfigs: IMDMColumnConfigWithParsedMeta[],
+  ): { cteName: string; expr: string } | undefined {
+    const pendingColumnsSet = new Set(columnConfigs.map((config) => config.columnName));
+    const sqlParts = this.buildSqlQueryParts(Array.from(pendingColumnsSet));
+
+    if (sqlParts.length !== 0 || columnConfigs.length !== 0) {
+      const cteName = `cte_staging_${parentCTEName}`;
+      const columnsToProject = [
+        '*',
+        ...columnConfigs.map((config) => {
+          let resolvedExpressionValue =
+            config.columnMeta.sqlQueryProps?.resolvedExpression?.value?.join(', ');
+          if (resolvedExpressionValue) {
+            resolvedExpressionValue = resolveRuntimeVariables(
+              resolvedExpressionValue,
+              this.options.runtimeVariables,
+            );
+          }
+          return `${resolvedExpressionValue} AS ${this.entityNameFormatter(config.columnName)}`;
+        }),
+      ].join(', ');
+      const stagingCte = `${cteName} AS (SELECT ${columnsToProject} FROM ${parentCTEName}${sqlParts.length ? ' ' + sqlParts.join(' ') : ''})`;
+      return { cteName, expr: stagingCte };
+    }
+  }
+
+  private buildFinalProjection(
+    mainCTE: MainCteExpression,
+    intermediateCTE: CteExpression,
+    isCountQuery: boolean = false,
+  ): FinalQueryResult {
+    const {
+      distinct,
+      selectedColumns,
+      includeFinalCTESelectQuery = true,
+      finalResponseAsObject = false,
+      joinClauses,
+      enableLastSelectCTEWrap = false,
+    } = this.options;
+    const { joinRelation, alias } = joinClauses || {};
+    const mainTableAlias = alias ? `${this.entityNameFormatter(alias)}` : '';
+
+    // final query is select from main cte and intermediate cte if exists and isFilterPending is false
+    const finalSqlParts = this.buildState.isSuperFilterPending ? false : true;
+    const sqlParts = this.buildSqlQueryParts([], finalSqlParts);
+
+    if (
+      sqlParts.length === 0 &&
+      intermediateCTE.value.length === 0 &&
+      !this.buildState.isSuperFilterPending
+    ) {
+      if (joinRelation?.length) {
+        mainCTE.value.from = this.applyJoinExpressionToMainCTE(mainCTE.value);
+      }
+      if (distinct) {
+        this.applyDistinctExpressionToMainCTE(mainCTE);
+      }
+      const query = isCountQuery
+        ? this.applyCountExpressionToMainCTE(mainCTE)
+        : processSelect(mainCTE.value);
+      return {
+        finalQuery: query,
+      };
+    } else {
+      let currentCteName = mainCTE.cteName;
+      const cteParts = [`cte_main AS (${processSelect(mainCTE.value)})`];
+      if (intermediateCTE.value.length > 0) {
+        currentCteName = intermediateCTE.cteName;
+        cteParts.push(intermediateCTE.value.join(', '));
+      }
+      let tableName = currentCteName;
+      if (joinRelation?.length) {
+        const joinExpr = getJoinExpressions(
+          joinClauses as JoinExpressions,
+          this.options?.databaseDetails,
+        ).join(' ');
+        sqlParts.splice(0, 0, joinExpr);
+      }
+      let column = '*';
+      if (distinct) {
+        tableName = `${tableName}${mainTableAlias ? ` AS ${mainTableAlias}` : ''}`;
+
+        column = `DISTINCT ${selectedColumns
+          .map((c) => {
+            const schemaMatch = this.options.schema?.find((s) => s.name === c);
+            return (
+              `${mainTableAlias ? `${mainTableAlias}.` : ''}` +
+              this.entityNameFormatter(c, schemaMatch?.type ?? '')
+            );
+          })
+          .join(', ')}`;
+      }
+      if (isCountQuery) {
+        if (distinct) {
+          const distinctCountCte = this.getDistinctCountCte(currentCteName, sqlParts);
+          sqlParts.length = 0;
+          tableName = distinctCountCte.cteName;
+          cteParts.push(distinctCountCte.value.join(', '));
+          column = `COUNT(*) AS row_count`;
+        } else {
+          if (joinRelation?.length) {
+            const primaryKeyColumns = this.options.primaryKeyColumns?.[0];
+            if (mainTableAlias && primaryKeyColumns) {
+              column = `DISTINCT ${mainTableAlias}.${this.entityNameFormatter(primaryKeyColumns)}`;
+            }
+          }
+          column = `COUNT(${column}) AS row_count`;
+        }
+      }
+      if (!distinct) {
+        tableName = `${tableName}${mainTableAlias ? ` AS ${mainTableAlias}` : ''}`;
+      }
+      let selectedColumn =
+        column === '*' && mainTableAlias ? `DISTINCT ${mainTableAlias}.*` : column;
+
+      if (isCountQuery && this.buildState.isSuperFilterPending) {
+        selectedColumn = mainTableAlias && !distinct ? `DISTINCT ${mainTableAlias}.*` : '*';
+      }
+
+      const selectedProjection = `SELECT ${selectedColumn} FROM ${tableName}${sqlParts.length ? ' ' + sqlParts.join(' ') : ''}`;
+
+      let finalProjection = '';
+      if (this.buildState.isSuperFilterPending || enableLastSelectCTEWrap) {
+        const { cteDefinition, finalQuery, finalCteName } = this.buildFinalQueryComponents(
+          currentCteName,
+          selectedProjection,
+          isCountQuery,
+          distinct,
+        );
+        currentCteName = finalCteName;
+        cteParts.push(cteDefinition);
+        finalProjection = finalQuery;
+      } else {
+        finalProjection = selectedProjection;
+      }
+      const cteJoins = enableLastSelectCTEWrap
+        ? cteParts.join(', ')
+        : `WITH ${cteParts.join(', ')}`;
+      let finalQuery = cteJoins;
+      if (includeFinalCTESelectQuery) {
+        finalQuery += ` ${finalProjection}`;
+      }
+
+      if (finalResponseAsObject) {
+        return {
+          finalQuery: finalQuery,
+          finalCteName: currentCteName,
+        };
+      }
+
+      return {
+        finalQuery,
+      };
+    }
+  }
+
+  private applyDistinctExpressionToMainCTE(mainCTE: MainCteExpression) {
+    const { selectedColumns } = this.options;
+    if (mainCTE.value.columns.length > selectedColumns.length) {
+      mainCTE.value.columns = mainCTE.value.columns.filter((c) => {
+        return selectedColumns.includes(c.expr.meta?.rawColumnName);
+      });
+    }
+    mainCTE.value.distinct = 'DISTINCT';
+  }
+
+  private applyCountExpressionToMainCTE(mainCTE: MainCteExpression): string {
+    if (mainCTE.value.distinct) {
+      const mainCteResult = `WITH cte_main AS (${processSelect(mainCTE.value)})`;
+      return `${mainCteResult} SELECT COUNT(*) AS row_count FROM cte_main`;
+    }
+
+    const columnValue =
+      mainCTE.value.columns.length > 1 ? '*' : `${mainCTE.value.columns[0].expr.value}`;
+    mainCTE.value.columns[0].expr.value = `COUNT(${columnValue})`;
+    mainCTE.value.columns[0].as = `row_count`;
+    mainCTE.value.columns.length = 1;
+    return processSelect(mainCTE.value);
+  }
+
+  private getDistinctCountCte(parentCTEName: string, sqlParts: string[]): CteExpression {
+    const { selectedColumns, joinClauses } = this.options;
+    const { alias } = joinClauses || {};
+    const cteName = `cte_distinct_${parentCTEName}`;
+    const fromName = `${parentCTEName}${alias ? ` AS ${this.entityNameFormatter(alias)}` : ''}`;
+    return {
+      type: ResolvedExpressionType.CTE,
+      cteName,
+      value: [
+        `${cteName} AS (SELECT DISTINCT ${selectedColumns.map((c) => `${alias ? `${this.entityNameFormatter(alias)}.` : ''}` + this.entityNameFormatter(c)).join(', ')} FROM ${fromName}${sqlParts.length ? ' ' + sqlParts.join(' ') : ''})`,
+      ],
+    };
+  }
+
+  private buildSqlQueryParts(columns: string[], isFinalProjection = false): string[] {
+    const sqlParts = [];
+
+    const where = [];
+    const filterCondition = this.getFilterExpression(new Set(columns));
+    if (filterCondition) {
+      where.push(filterCondition);
+    }
+    if (columns.length === 0 && this.buildState.isSuperFilterPending) {
+      const superFilterCondition = this.getSuperFilterExpression(isFinalProjection);
+      if (superFilterCondition) {
+        where.push(superFilterCondition);
+      }
+    }
+    if (columns.length === 0 && this.buildState.isSearchPending) {
+      this.buildState.isSearchPending = false;
+      where.push(this.options.searchExpression);
+    }
+    if (where.length) {
+      sqlParts.push(`WHERE ${where.join(' AND ')}`);
+    }
+
+    const sortExpression = this.getSortExpression(new Set(columns));
+    if (sortExpression) {
+      sqlParts.push(`ORDER BY ${sortExpression}`);
+    }
+
+    const paginationExpression = this.getPaginationExpression();
+    if (paginationExpression) {
+      sqlParts.push(paginationExpression);
+    }
+
+    return sqlParts;
+  }
+
+  /**
+   * Returns the calculation order for the given columns
+   * @private exposed for testing
+   */
+  public getCalculationOrder(columns: Set<string>): CalculationOrderResult {
+    const state = {
+      dependencyLevel: -1,
+      dependencyLevelMap: new Map<string, number>(),
+      nativeColumns: new Set<string>(),
+      visited: new Set<string>(),
+    };
+
+    // trigger dependency tracing for each column
+    columns.forEach((columnName) => {
+      state.dependencyLevel = -1; // reset dependency level for each column
+      this.dfs(columnName, state);
+    });
+
+    // build output
+    const formulaColumnsByDependencyLevel: IMDMColumnConfigWithParsedMeta[][] = [];
+    for (const [columnName, level] of state.dependencyLevelMap) {
+      const columnConfig = this.options.columnConfigMap[columnName];
+      if (level >= formulaColumnsByDependencyLevel.length) {
+        formulaColumnsByDependencyLevel.push([]);
+      }
+      formulaColumnsByDependencyLevel[level].push(columnConfig);
+    }
+
+    return {
+      nativeColumns: state.nativeColumns,
+      formulaColumnsByDependencyLevel,
+    };
+  }
+
+  private dfs(
+    columnName: string,
+    state: {
+      dependencyLevel: number;
+      dependencyLevelMap: Map<string, number>;
+      nativeColumns: Set<string>;
+      visited: Set<string>;
+    },
+  ) {
+    // If dependency level is already resolved, return
+    const level = state.dependencyLevelMap.get(columnName);
+    if (level !== undefined) {
+      if (level > state.dependencyLevel) {
+        state.dependencyLevel = level;
+      }
+      return;
+    }
+
+    if (state.visited.has(columnName)) {
+      throw new Error(`Cycle detected in formula dependencies: ${columnName}`);
+    }
+
+    const columnConfig = this.options.columnConfigMap[columnName];
+    const { columnType, columnMeta } = columnConfig;
+    const { sqlQueryProps, isCustomColumn, deps, defaultValueType } = columnMeta;
+
+    const isNativeColumn =
+      columnType !== MDM_COLUMN_TYPE.FORMULA && defaultValueType !== DEFAULT_VALUE_TYPE.DERIVED;
+    const isFormulaDbColumn =
+      columnType === MDM_COLUMN_TYPE.FORMULA && !sqlQueryProps && !isCustomColumn && !deps;
+    const isDerivedDbColumn =
+      defaultValueType === DEFAULT_VALUE_TYPE.DERIVED && !sqlQueryProps && !deps;
+
+    if (isNativeColumn || isFormulaDbColumn || isDerivedDbColumn) {
+      state.nativeColumns.add(columnName);
+      return;
+    }
+
+    // Skip processing to maintain backward compatibility with older formula visual columns
+    if (columnType === MDM_COLUMN_TYPE.FORMULA && isCustomColumn && !sqlQueryProps) {
+      return;
+    }
+
+    state.visited.add(columnName);
+    deps?.forEach((colName) => {
+      this.dfs(colName, state);
+    });
+
+    state.visited.delete(columnName);
+    state.dependencyLevelMap.set(columnName, ++state.dependencyLevel);
+  }
+
+  /**
+   * Applies the join positions from the query builder to the given Select AST.
+   * The join positions are added to the FROM list of the AST.
+   *
+   * @param {Select} ast - The Select AST to apply the join positions to.
+   * @returns {string[]} - An empty array, for consistency with other methods.
+   */
+  private applyJoinExpressionToMainCTE(ast: Select): From[] {
+    const { joinClauses, databaseDetails } = this.options;
+    const joinExpressions = getJoinExpressions(joinClauses as JoinExpressions, databaseDetails);
+    const joins: From[] = joinExpressions.map((j) => {
+      const raw: RawSqlExpression = { type: 'sql_expr', value: j };
+      return raw as unknown as From;
+    });
+    if (joins.length) {
+      ast.distinct = 'DISTINCT';
+    }
+    const joinFroms = ast.from as From[];
+    // set the FROM list as base table + joins
+    return joinFroms?.concat(joins);
+  }
+
+  /**
+   * Gets the super filter expression.
+   * If super filters are present and the pending filter config is not empty,
+   * it will process the super filter condition and return the result.
+   * If the super filter expression is empty, it will return an empty string.
+   * @returns {string} - The super filter expression.
+   */
+  private getSuperFilterExpression(isFinalProjection?: boolean): string {
+    if (!this.options.superFilters || !this.options.superFilters.children.length) return '';
+    const { superFilters, manualFilterOptions } = this.options;
+
+    const { children } = superFilters;
+    const { rankingFilters, nonRankingFilters } = children.reduce(
+      (acc, child) => {
+        if ('groupId' in child && 'children' in child) {
+          acc.nonRankingFilters.push(child);
+        } else if ('filters' in child) {
+          const filterType = (child as { filters: IGroupFilter }).filters.filterType;
+          if (filterType === 'RANKING') {
+            acc.rankingFilters.push(child);
+          } else {
+            acc.nonRankingFilters.push(child);
+          }
+        }
+        return acc;
+      },
+      {
+        rankingFilters: [] as typeof children,
+        nonRankingFilters: [] as typeof children,
+      },
+    );
+
+    let activeFilters = { ...superFilters, children: nonRankingFilters };
+    let joinClauses = this.options.joinClauses;
+
+    // Process ranking filters if pending and available
+    if (rankingFilters.length && isFinalProjection && this.buildState.isSuperFilterPending) {
+      activeFilters = { ...superFilters, children: rankingFilters };
+      joinClauses = { alias: '', joinRelation: [] };
+    }
+    if (rankingFilters.length === 0) {
+      this.buildState.isSuperFilterPending = false;
+    }
+    if (activeFilters.children.length) {
+      const sqlQueryBuilder = new SuperFilterBuilder(
+        activeFilters,
+        this.options.databaseDetails,
+        this.options.skipCountForRankingFilter,
+        joinClauses,
+        manualFilterOptions,
+      );
+      return sqlQueryBuilder.build();
+    }
+
+    return '';
+  }
+
+  private buildFinalQueryComponents(
+    currentCteName: string,
+    selectProjection: string,
+    isCountQuery: boolean,
+    distinct: boolean,
+  ): { cteDefinition: string; finalQuery: string; finalCteName: string } {
+    const { selectedColumns, enableLastSelectCTEWrap } = this.options;
+    let selectQuery = '*';
+    if (distinct && !isCountQuery) {
+      selectQuery = `DISTINCT ${selectedColumns
+        .map((c) => {
+          const schemaMatch = this.options.schema?.find((s) => s.name === c);
+          return this.entityNameFormatter(c, schemaMatch?.type ?? '');
+        })
+        .join(', ')}`;
+    }
+
+    let finalCteName = `${currentCteName}_main_cte`;
+    let cteDefinition = `${finalCteName} AS (${selectProjection})`;
+
+    const columns = isCountQuery ? `COUNT(${selectQuery}) AS row_count` : `${selectQuery}`;
+    const sqlParts = this.buildSqlQueryParts([], true);
+
+    const finalQuery = `SELECT ${columns} FROM ${finalCteName}${sqlParts.length ? ' ' + sqlParts.join(' ') : ''}`;
+
+    if (enableLastSelectCTEWrap && this.buildState.isSuperFilterPending) {
+      finalCteName = `${finalCteName}_main_cte`;
+      cteDefinition = ` ${cteDefinition},${finalCteName} AS (${finalQuery})`;
+    }
+
+    this.buildState.isSuperFilterPending = false;
+
+    return { cteDefinition, finalQuery, finalCteName };
+  }
+
+  private filterRequiredColumns(): Set<string> {
+    const {
+      selectedColumns,
+      superFilters,
+      primaryKeyColumns,
+      sortConfig = [],
+      joinClauses,
+    } = this.options;
+    const requiredColumn = new Set<string>(selectedColumns);
+    if (superFilters?.children?.length) {
+      // Collect all columnIds from superFilters
+      (superFilters?.children as { filters: IGroupFilter }[])?.forEach((child) => {
+        const filterColumn = child.filters?.columnIds?.[0];
+        if (filterColumn) requiredColumn.add(filterColumn);
+        child?.filters?.rankingFilter?.forEach?.((rank) => {
+          if (rank?.rankByColumnName) requiredColumn.add(rank.rankByColumnName);
+        });
+      });
+      primaryKeyColumns.forEach((col) => {
+        requiredColumn.add(col);
+      });
+      sortConfig.forEach((col) => {
+        requiredColumn.add(col?.column);
+      });
+    }
+    // Add all columnIds from joinRelation
+    for (const { id } of joinClauses?.joinRelation ?? []) {
+      if (id != null) requiredColumn.add(id);
+    }
+
+    return requiredColumn;
+  }
+}