@quillsql/react 2.13.35 → 2.13.36

package/dist/esm/ReportBuilder.js

@@ -31,7 +31,7 @@ import { astToFilterTree, filterTreeToAst, getFieldFromExpression, } from './uti
 import useAstToFilterTree from './hooks/useAstToFilterTree';
 import { filterStackToFilterTree, uniqueValuesToStringMap, } from './utils/filterProcessing';
 import { QuillMultiSelectComponentWithCombo } from './components/QuillMultiSelectWithCombo';
-import { DEFAULT_PAGINATION, shouldFetchMore, shouldSortInMemory, } from './utils/paginationProcessing';
+import { DEFAULT_PAGINATION, shouldFetchMore, } from './utils/paginationProcessing';
 import { EMPTY_INTERNAL_REPORT, fetchReportBuilderDataFromAST, formatRowsFromReport, } from './utils/report';
 import { TEMP_REPORT_ID, } from './models/Report';
 import equal from 'fast-deep-equal';
@@ -102,7 +102,15 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
     const [rows, setRows] = useState([]);
     const [formattedRows, setFormattedRows] = useState([]);
     const [columns, setColumns] = useState([]);
-    const [tempReport, setTempReport] = useState(EMPTY_INTERNAL_REPORT);
+    const REPORT_BUILDER_PAGINATION = {
+        page: 0,
+        rowsPerPage: 20,
+        rowsPerRequest: 100,
+    };
+    const [tempReport, setTempReport] = useState({
+        ...EMPTY_INTERNAL_REPORT,
+        pagination: REPORT_BUILDER_PAGINATION,
+    });
     const [uniqueValues, setUniqueValues] = useState({});
     const [uniqueValuesIsLoading, setUniqueValuesIsLoading] = useState(false);
     const [pivot, setPivot] = useState(null);
@@ -138,11 +146,6 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
     const { filterTree, filterStack } = useAstToFilterTree(formData, client, columns); // Stores the state of filters
     const [removingFilter, setRemovingFilter] = useState(false);
     const [pivotRecommendationsEnabledState, setPivotRecommendationsEnabledState,] = useState(pivotRecommendationsEnabled);
-    const REPORT_BUILDER_PAGINATION = {
-        page: 0,
-        rowsPerPage: 20,
-        rowsPerRequest: 1000,
-    };
     const [unresolvedReportMessage, setUnresolvedReportMessage] = useState('');
     const dashboardName = useMemo(() => {
         if (destinationDashboard) {
@@ -154,12 +157,20 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
         return dashboard[reportId]?.dashboardName;
     }, [reportId, dashboard, destinationDashboard]);
     const [filtersEnabled, setFiltersEnabled] = useState(!!reportId);
+    const mssqlSortWarning = useMemo(() => {
+        if (!client || client?.databaseType !== 'mssql') {
+            return undefined;
+        }
+        else if (!pivot && !baseAst?.top) {
+            return 'Please add a limit.';
+        }
+    }, [client, baseAst, pivot]);
     useEffect(() => {
         if (!client) {
             return;
         }
-        if (client.recommendedPivotsDisabled !== undefined) {
-            setPivotRecommendationsEnabledState(!client.recommendedPivotsDisabled);
+        if (client.featureFlags?.['recommendedPivotsDisabled'] !== undefined) {
+            setPivotRecommendationsEnabledState(!client.featureFlags?.['recommendedPivotsDisabled']);
         }
         if (!initialLoad && client.publicKey) {
             clearAllState();
@@ -205,6 +216,14 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
         setPivotError(undefined);
         // @ts-ignore
         newPivot[fieldKey] = changeField;
+        if (fieldKey === 'columnField' &&
+            !changeField &&
+            pivotValueField === pivotValueField2) {
+            setPivotValueField2(undefined);
+            newPivot.valueField2 = undefined;
+        }
+        newPivot.rowLimit = undefined;
+        newPivot.sort = undefined;
         if (fieldKey === 'rowField') {
             // check to see if the new rowField value is a date field
             const column = columns.find((c) => c.field === changeField);
@@ -220,11 +239,15 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                 newPivot.sort = undefined;
             }
         }
+        setPivot(newPivot);
         const { valid, reason } = isValidPivot(newPivot);
         if (!valid) {
             setPivotError(reason);
             return;
         }
+        resetLimit();
+        resetSort();
+        setPreviousPage(0);
         setTableLoading(true);
         let dateBucket = undefined;
         const tempDateRange = dateRanges && dateRanges[newPivot.rowField || ''];
@@ -234,16 +257,20 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
         let distinctValuesForQuery = {};
         const prevPivot = pivot;
         const prevPivotData = pivotData;
-        setPivot(newPivot);
         setPivotHint('');
         if (newPivot.columnField) {
-            distinctValuesForQuery = await getUniqueValuesByColumns([
-                {
-                    field: newPivot.columnField,
-                    label: newPivot.columnField,
-                    format: 'string',
-                },
-            ], activeQuery, [], client, tenants, schemaData.customFields ?? [], undefined, dashboardName);
+            if (uniqueValues?.[newPivot.columnField]) {
+                distinctValuesForQuery = uniqueValues;
+            }
+            else {
+                distinctValuesForQuery = await getUniqueValuesByColumns([
+                    {
+                        field: newPivot.columnField,
+                        label: newPivot.columnField,
+                        format: 'string',
+                    },
+                ], activeQuery, [], client, tenants, schemaData.customFields ?? [], undefined, dashboardName);
+            }
         }
         try {
             const pivotedData = await generatePivotTable({
@@ -254,7 +281,9 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                 uniqueValues: distinctValuesForQuery,
                 dashboardName,
                 tenants,
+                additionalProcessing: { page: REPORT_BUILDER_PAGINATION },
             });
+            resetProcessing();
             setPivotData(pivotedData || []);
             const formattedRows = formatRows(pivotedData.rows, columns, true, newPivot.aggregationType);
             setFormattedRows(formattedRows);
@@ -286,6 +315,65 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
             setTableLoading(false);
         }
     };
+    const resetLimit = (fetchData) => {
+        const newAst = {
+            ...baseAst,
+            limit: null,
+            top: null,
+            orderby: client?.databaseType === 'mssql' ? null : baseAst.orderby,
+        };
+        setBaseAst((prevAst) => ({
+            ...prevAst,
+            limit: null,
+            top: null,
+            orderby: client?.databaseType === 'mssql' ? null : prevAst.orderby,
+        }));
+        setPivot((oldPivot) => {
+            if (!oldPivot)
+                return null;
+            return {
+                ...oldPivot,
+                rowLimit: undefined,
+            };
+        });
+        if (fetchData) {
+            fetchReportFromASTHelper({
+                baseAst: newAst,
+                curPivot: pivot ? { ...pivot, rowLimit: undefined } : undefined,
+            });
+        }
+    };
+    const resetSort = (fetchData) => {
+        if (pivot) {
+            setPivot((oldPivot) => {
+                if (!oldPivot)
+                    return null;
+                return {
+                    ...oldPivot,
+                    sort: undefined,
+                    sortField: undefined,
+                    sortDirection: undefined,
+                    sortFieldType: undefined,
+                };
+            });
+        }
+        const newAst = { ...baseAst, orderby: null };
+        setBaseAst((prevAst) => ({ ...prevAst, orderby: null }));
+        if (fetchData) {
+            fetchReportFromASTHelper({
+                baseAst: newAst,
+                curPivot: pivot
+                    ? {
+                        ...pivot,
+                        sort: undefined,
+                        sortField: undefined,
+                        sortDirection: undefined,
+                        sortFieldType: undefined,
+                    }
+                    : undefined,
+            });
+        }
+    };
     const enforceOrderOnColumns = (columnNames) => {
         if (pivot) {
             const rowName = pivot.rowField;
@@ -434,7 +522,10 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
             const data = await response.json();
             setActiveQuery(data.query);
             if (fetchData) {
-                fetchReportFromASTHelper(ast, formData);
+                fetchReportFromASTHelper({
+                    baseAst: ast,
+                    newFormData: formData,
+                });
             }
             return data.query;
         }
@@ -554,7 +645,13 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                 }
                 const { ast: newAst, pivot: newPivot, schema: curSchema, } = await fetchASTFromQuillReport(report, client, schemaData.schema);
                 setBaseAst({ ...newAst, where: null });
-                await fetchReportFromASTHelper({ ...newAst, where: null }, newAst.where, newPivot, curSchema, report);
+                await fetchReportFromASTHelper({
+                    baseAst: { ...newAst, where: null },
+                    newFormData: newAst.where,
+                    curPivot: newPivot,
+                    curSchema,
+                    previousReport: report,
+                });
                 await onSchemaChange();
                 setReportInfo(report);
                 const query = await fetchSqlQuery(newAst, null, false);
@@ -636,7 +733,7 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
             ];
         }
         else {
-            return [{ label: snakeAndCamelCaseToTitleCase(pivot.valueField) }];
+            return [{ label: snakeAndCamelCaseToTitleCase(pivot?.valueField || '') }];
         }
     };
     const [previousPage, setPreviousPage] = useState(0);
@@ -646,12 +743,15 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
     const [numberOfRows, setNumberOfRows] = useState(0);
     const [rowCountIsLoading, setRowCountIsLoading] = useState(false);
     const [tableLoading, setTableLoading] = useState(false);
+    const resetProcessing = () => {
+        setCurrentProcessing({ page: REPORT_BUILDER_PAGINATION });
+    };
     const onPageChange = (page, initiator = 'ReportBuilder') => {
         const pagination = initiator === 'ReportBuilder'
             ? REPORT_BUILDER_PAGINATION
             : DEFAULT_PAGINATION;
         if (currentProcessing.page &&
-            shouldFetchMore(pagination, page, previousPage)) {
+            shouldFetchMore(pagination, page, previousPage, pivotData ? pivotData.rows.length : rows.length)) {
             const newPagination = { ...currentProcessing.page, page };
             const updatedProcessing = { ...currentProcessing, page: newPagination };
             setCurrentProcessing(updatedProcessing);
@@ -661,40 +761,130 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
             setPreviousPage(page);
         }
     };
-    const onSortChange = (sort) => {
-        if (shouldSortInMemory(REPORT_BUILDER_PAGINATION, numberOfRows) ||
-            (pivot && pivot.rowField !== sort.field)) {
-            return;
+    const onSortChange = (sort, isDelete) => {
+        if (pivot) {
+            let newPivot = null;
+            if (isDelete) {
+                setPivot((oldPivot) => {
+                    if (!oldPivot)
+                        return null;
+                    newPivot = {
+                        ...oldPivot,
+                        sort: undefined,
+                        sortField: undefined,
+                        sortDirection: undefined,
+                        sortFieldType: undefined,
+                    };
+                    return newPivot;
+                });
+            }
+            else {
+                setPivot((oldPivot) => {
+                    if (!oldPivot)
+                        return null;
+                    newPivot = {
+                        ...oldPivot,
+                        sort: true,
+                        sortField: sort.field,
+                        sortDirection: sort.direction,
+                        sortFieldType: getColumnTypeByName(sort.field),
+                    };
+                    return newPivot;
+                });
+            }
+            fetchReportFromASTHelper({
+                baseAst: baseAst,
+                curPivot: newPivot,
+            });
         }
-        const newAst = { ...baseAst };
-        if (!newAst.orderby) {
-            newAst.orderby = [];
+        else {
+            const newAst = { ...baseAst };
+            if (!newAst.orderby) {
+                newAst.orderby = [];
+            }
+            const existingSortIndex = newAst.orderby.findIndex((item) => getFieldFromExpression(item.expr) === sort.field);
+            if (isDelete) {
+                if (existingSortIndex !== -1) {
+                    newAst.orderby.splice(existingSortIndex, 1);
+                }
+            }
+            else if (existingSortIndex !== -1) {
+                newAst.orderby[existingSortIndex] = {
+                    expr: { type: 'column_ref', column: sort.field },
+                    type: sort.direction.toUpperCase(),
+                };
+            }
+            else {
+                newAst.orderby.push({
+                    expr: { type: 'column_ref', column: sort.field },
+                    type: sort.direction.toUpperCase(),
+                });
+            }
+            setBaseAst(deepCopy(newAst));
+            fetchReportFromASTHelper({
+                baseAst: newAst,
+            });
+            setPreviousPage(0);
         }
-        const existingSortIndex = newAst.orderby.findIndex((item) => getFieldFromExpression(item.expr) === sort.field);
-        if (existingSortIndex !== -1) {
-            newAst.orderby[existingSortIndex] = {
-                expr: { type: 'column_ref', column: sort.field },
-                type: sort.direction.toUpperCase(),
-            };
+    };
+    const onLimitChange = (limit) => {
+        if (limit) {
+            if (pivot) {
+                setPivot((oldPivot) => {
+                    if (!oldPivot)
+                        return null;
+                    const newPivot = { ...oldPivot, rowLimit: limit };
+                    fetchReportFromASTHelper({ baseAst, curPivot: newPivot });
+                    return newPivot;
+                });
+            }
+            else {
+                const newAst = { ...baseAst };
+                if (client.databaseType.toLowerCase() === 'mssql') {
+                    newAst.top = {
+                        value: limit,
+                    };
+                }
+                else {
+                    newAst.limit = {
+                        seperator: '',
+                        value: [
+                            {
+                                type: 'number',
+                                value: limit,
+                            },
+                        ],
+                    };
+                }
+                setBaseAst(deepCopy(newAst));
+                fetchSqlQuery(deepCopy(newAst), undefined, true);
+            }
         }
         else {
-            newAst.orderby.push({
-                expr: { type: 'column_ref', column: sort.field },
-                type: sort.direction.toUpperCase(),
-            });
+            resetLimit(true);
         }
-        setBaseAst(deepCopy(newAst));
-        fetchReportFromASTHelper(newAst, newAst.where);
-        setPreviousPage(0);
+        setOpenPopover(null);
     };
     const fetchRowCount = async (processing, includeFilters) => {
         if (!client || !activeQuery) {
             return;
         }
         setRowCountIsLoading(true);
-        const tableInfo = await fetchResultsByQuery(activeQuery, client, tenants, processing, schemaData.customFields, includeFilters ? specificDashboardFilters : undefined, includeFilters
-            ? (tempReport.dateField ?? reportInfo?.dateField)
-            : undefined, false, true, undefined, dashboardName);
+        const tableInfo = await fetchResultsByQuery({
+            query: activeQuery,
+            client,
+            tenants,
+            processing,
+            customFields: schemaData.customFields,
+            filters: includeFilters ? specificDashboardFilters : undefined,
+            dateField: includeFilters
+                ? (tempReport.dateField ?? reportInfo?.dateField)
+                : undefined,
+            rowsOnly: false,
+            rowCountOnly: true,
+            filterMap: undefined,
+            dashboardName,
+        });
         if (tableInfo.rowCount) {
             setNumberOfRows(tableInfo.rowCount);
             // @ts-ignore
@@ -720,29 +910,62 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
     };
     const handleRunQuery = async (processing, resetRows = false, includeFilters = false) => {
         try {
+            const isPivotPagination = !!(pivot && pivotData);
             setErrorMessage('');
             setTableLoading(true);
-            const tableInfo = await fetchResultsByQuery(activeQuery, client, tenants, processing, schemaData.customFields, undefined, undefined, true, undefined, undefined, dashboardName);
+            const tableInfo = await fetchResultsByQuery({
+                query: isPivotPagination ? pivotData.pivotQuery : activeQuery,
+                comparisonQuery: pivot && pivotData ? pivotData.comparisonPivotQuery : undefined,
+                client,
+                tenants,
+                processing,
+                customFields: schemaData.customFields,
+                rowsOnly: true,
+                dashboardName,
+                pivot: pivot,
+                getPivotRowCount: false,
+            });
             if (tableInfo.error) {
                 throw new Error(tableInfo.error);
             }
             else if (tableInfo.rows.length === 0) {
                 throw new Error('No data found');
             }
-            fetchRowCount(processing, includeFilters);
-            let tempRows = [...rows, ...tableInfo.rows];
-            if (resetRows) {
-                tempRows = tableInfo.rows;
-                setPreviousPage(0);
+            if (!isPivotPagination) {
+                // fetching row count for non-pivot query
+                fetchRowCount(processing, includeFilters);
+                let tempRows = [...rows, ...tableInfo.rows];
+                if (resetRows) {
+                    tempRows = tableInfo.rows;
+                    setPreviousPage(0);
+                }
+                setRows(tempRows);
+                setFormattedRows(formatRowsFromReport({ rows: tempRows, columns: tableInfo.columns }));
+                setTempReport((tempReport) => ({
+                    ...tempReport,
+                    rows: tempRows,
+                    rowCount: tableInfo.rowCount ?? tempReport.rowCount,
+                }));
+                setColumns(tableInfo.columns);
+            }
+            else {
+                let tempRows = [...pivotData.rows, ...tableInfo.rows];
+                if (resetRows) {
+                    tempRows = tableInfo.rows;
+                    setPreviousPage(0);
+                }
+                setPivotData((oldPivotData) => {
+                    if (oldPivotData) {
+                        return {
+                            ...oldPivotData,
+                            rows: tempRows,
+                            columns: tableInfo.columns,
+                        };
+                    }
+                    return null;
+                });
+                setFormattedRows(formatRowsFromReport({ rows: tempRows, columns: tableInfo.columns }));
             }
-            setRows(tempRows);
-            setFormattedRows(formatRowsFromReport({ rows: tempRows, columns: tableInfo.columns }));
-            setTempReport((tempReport) => ({
-                ...tempReport,
-                rows: tempRows,
-                rowCount: tableInfo.rowCount ?? tempReport.rowCount,
-            }));
-            setColumns(tableInfo.columns);
             setTableLoading(false);
         }
         catch (e) {
@@ -783,12 +1006,31 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
         updateFieldValuesMap(uniqueStringsObj, table);
         return uniqueStringsObj;
     };
-    const fetchReportFromASTHelper = async (baseAst, newFormData, curPivot, curSchema, previousReport, keepPivotHint = false) => {
+    const fetchReportFromASTHelper = async ({ baseAst, newFormData, curPivot, curSchema, previousReport, keepPivotHint = false, }) => {
         const curFormData = newFormData !== undefined ? newFormData : formData;
         let reportBuilderInfo = undefined;
         try {
             setLoading(true);
-            reportBuilderInfo = await fetchReportBuilderDataFromAST(baseAst, curFormData, curSchema ?? schemaData.schema, client, tenants, curPivot ?? pivot, formData, currentTable, { uniqueStrings: uniqueValues, dateRanges: dateRanges ?? {} }, previousReport ?? reportInfo ?? undefined, schemaData.customFields, true, true, { page: REPORT_BUILDER_PAGINATION }, dashboardName);
+            reportBuilderInfo = await fetchReportBuilderDataFromAST({
+                baseAst,
+                formData: curFormData,
+                schema: curSchema ?? schemaData.schema,
+                client,
+                tenants,
+                pivot: curPivot ?? pivot,
+                previousFormData: formData,
+                currentTable,
+                previousRelevant: {
+                    uniqueStrings: uniqueValues,
+                    dateRanges: dateRanges ?? {},
+                },
+                report: previousReport ?? reportInfo ?? undefined,
+                customFields: schemaData.customFields,
+                skipUniqueValues: true,
+                skipRowCount: true,
+                processing: { page: REPORT_BUILDER_PAGINATION },
+                dashboardName,
+            });
             if (reportBuilderInfo.error) {
                 throw new Error(reportBuilderInfo.error);
             }
@@ -808,7 +1050,10 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
             setErrorMessage('Failed to fetch');
             return;
         }
-        setTempReport(reportBuilderInfo.report);
+        setTempReport({
+            ...reportBuilderInfo.report,
+            pagination: REPORT_BUILDER_PAGINATION,
+        });
         fetchRowCountFromAST(baseAst, curFormData);
         const reportTable = reportBuilderInfo.table;
         const reportBuilderInfoColumns = reportBuilderInfo.columns.map((column) => {
@@ -826,7 +1071,6 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
         if (prevTable !== reportBuilderInfo.table) {
             setSelectedOrderedColumns([]); // reset selected ordered columns
         }
-        setNumberOfRows(reportBuilderInfo.rowCount);
         setPivot(reportBuilderInfo.pivot);
         if (!keepPivotHint) {
             setPivotHint('');
@@ -902,7 +1146,14 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                 }
                 setUniqueValuesIsLoading(false);
                 if (pivotChanged) {
-                    fetchReportFromASTHelper(baseAst, curFormData, newPivot, curSchema, previousReport, true);
+                    fetchReportFromASTHelper({
+                        baseAst,
+                        newFormData: curFormData,
+                        curPivot: newPivot,
+                        curSchema,
+                        previousReport,
+                        keepPivotHint: true,
+                    });
                 }
                 else {
                     setLoading(false);
@@ -934,7 +1185,7 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
             setLoading(true);
             setAskAILoading(true);
             setErrorMessage('');
-            astInfo = await fetchAndProcessASTFromPrompt(prompt, schemaData.schema, client, pivot, activeQuery, currentTable, dashboardName);
+            astInfo = await fetchAndProcessASTFromPrompt(prompt, schemaData.schema, client, pivot ?? undefined, activeQuery, currentTable, dashboardName, tenants);
             if (astInfo.error) {
                 throw new Error(astInfo.error);
             }
@@ -969,7 +1220,11 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
         setBaseAst(astInfo.ast);
         fetchSqlQuery(astInfo.ast, cleanAst, false);
         setAskAILoading(false);
-        await fetchReportFromASTHelper(astInfo.ast, cleanAst, astInfo.pivot);
+        await fetchReportFromASTHelper({
+            baseAst: astInfo.ast,
+            newFormData: cleanAst,
+            curPivot: astInfo.pivot,
+        });
     };
     const makePivotValid = (pivot, uniqueValuesForPivot, reportTable, reportColumns) => {
         if (!pivot) {
@@ -1173,6 +1428,7 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                                                     ast.where = null;
                                                     ast.orderby = null;
                                                     ast.limit = null;
+                                                    ast.top = null;
                                                     setBaseAst(ast);
                                                     fetchSqlQuery(ast, null);
                                                 }
@@ -1231,23 +1487,29 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                                             setPivot(null);
                                             setPivotHint('');
                                             setPivotData(null);
+                                            resetLimit();
+                                            resetSort(true);
+                                            setPreviousPage(0);
                                             const formattedRows = formatRows(rows, columns, false);
                                             setFormattedRows(formattedRows);
                                         }, selectPivot: async (selectedPivot, uniqueValues, dateRange, pivotTable) => {
                                             if (!selectedPivot)
                                                 return;
-                                            const newAst = { ...baseAst };
-                                            newAst.orderby = null;
+                                            // const newAst = { ...baseAst };
+                                            // newAst.orderby = null;
                                             if (selectedPivot.rowFieldType === 'date') {
                                                 selectedPivot['sort'] = true;
                                                 selectedPivot['sortDirection'] = 'ASC';
                                             }
-                                            setBaseAst(newAst); // trigger refetch
+                                            // setBaseAst(newAst); // trigger refetch
                                             let dateBucket = undefined;
                                             if (dateRange) {
                                                 dateBucket = getDateBucketFromRange(dateRange);
                                             }
                                             setPivot(selectedPivot);
+                                            resetLimit();
+                                            resetSort();
+                                            setPreviousPage(0);
                                             setPivotHint('');
                                             try {
                                                 if (!pivotTable) {
@@ -1260,8 +1522,12 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                                                         uniqueValues,
                                                         dashboardName,
                                                         tenants,
+                                                        additionalProcessing: {
+                                                            page: REPORT_BUILDER_PAGINATION,
+                                                        },
                                                     });
                                                 }
+                                                resetProcessing();
                                                 setPivotData(pivotTable || []);
                                                 const formattedRows = formatRows(pivotTable.rows, columns, true, selectedPivot.aggregationType, dateBucket);
                                                 setFormattedRows(formattedRows);
@@ -1299,167 +1565,48 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                                             setPivot(null);
                                             setPivotError(undefined);
                                             setPivotHint('');
-                                            setPivotData([]);
+                                            setPivotData(null);
+                                            resetLimit();
+                                            resetSort();
+                                            setPreviousPage(0);
                                             const formattedRows = formatRows(rows, columns, false);
                                             setFormattedRows(formattedRows);
-                                        }, isLoading: tableLoading || loading, pivotRowField: pivotRowField, pivotColumnField: pivotColumnField, pivotValueField: pivotValueField, pivotValueField2: pivotValueField2, pivotAggregation: pivotAggregation, SecondaryButtonComponent: SecondaryButtonComponent, SelectComponent: SelectComponent, PivotColumnContainer: PivotColumnContainer, pivotHint: pivotHint }))] }), _jsxs("div", { style: { width: '100%' }, children: [_jsx(SidebarHeadingComponent, { label: "Sort" }), pivot && pivot.sort && (_jsx("div", { style: {
+                                        }, isLoading: tableLoading || loading, pivotRowField: pivotRowField, pivotColumnField: pivotColumnField, pivotValueField: pivotValueField, pivotValueField2: pivotValueField2, pivotAggregation: pivotAggregation, SecondaryButtonComponent: SecondaryButtonComponent, SelectComponent: SelectComponent, PivotColumnContainer: PivotColumnContainer, pivotHint: pivotHint }))] }), _jsxs("div", { style: { width: '100%' }, children: [_jsx(SidebarHeadingComponent, { label: "Sort" }), pivot && pivot.sort && pivot.sortField && (_jsx("div", { style: {
                                             display: 'flex',
                                             flexDirection: 'column',
                                             gap: 8,
                                             marginBottom: 12,
-                                        }, children: _jsx(SortSentence, { sortData: {
-                                                type: pivot.sortDirection,
-                                                expr: { type: 'column_ref', column: pivot.sortField },
-                                            }, columns: pivot
-                                                ? pivot.columnField
-                                                    ? [`.${pivot.rowField}`]
-                                                    : [
-                                                        `.${pivot.rowField}`,
-                                                        `.${pivot.valueField || 'count'}`,
-                                                    ]
-                                                : selectedColumns, setIsPending: () => { }, setEditPopoverKey: () => { }, setActiveEditItem: setActiveEditItem, setOpenPopover: setOpenPopover, SortPopover: SortPopoverComponent, EditPopover: AddSortPopover, handleDelete: async () => {
-                                                if (!pivot) {
-                                                    setBaseAst(deepCopy(baseAst));
-                                                    fetchSqlQuery(deepCopy(baseAst));
-                                                    return;
-                                                }
-                                                const tempPivot = { ...pivot, sort: false };
-                                                let dateBucket = undefined;
-                                                const tempDateRange = dateRanges &&
-                                                    pivot.rowField &&
-                                                    dateRanges[pivot.rowField];
-                                                if (tempDateRange) {
-                                                    dateBucket = getDateBucketFromRange(tempDateRange.dateRange);
-                                                }
-                                                setPivot(tempPivot);
-                                                setPivotHint('');
-                                                try {
-                                                    const pivotedData = await generatePivotTable({
-                                                        pivot: tempPivot,
-                                                        rowLimit: -1,
-                                                        dateBucket,
-                                                        report: tempReport,
-                                                        client,
-                                                        uniqueValues: uniqueValues[currentTable],
-                                                        dashboardName,
-                                                        tenants,
-                                                    });
-                                                    setPivotData(pivotedData || []);
-                                                    const formattedRows = formatRows(pivotedData.rows, columns, true, pivot.aggregationType, dateBucket);
-                                                    setFormattedRows(formattedRows);
-                                                    setErrorMessage('');
-                                                }
-                                                catch (e) {
-                                                    if (e instanceof Error)
-                                                        setPivotError(e.message);
-                                                }
-                                                finally {
-                                                    setTableLoading(false);
-                                                }
+                                        }, children: _jsx(SortSentence, { sortField: pivot.sortField, sortDirection: pivot.sortDirection || 'ASC', columns: pivotData?.columns ?? [], setIsPending: () => { }, setEditPopoverKey: () => { }, setActiveEditItem: setActiveEditItem, setOpenPopover: setOpenPopover, SortPopover: SortPopoverComponent, EditPopover: AddSortPopover, handleDelete: async () => {
+                                                onSortChange({
+                                                    field: pivot.sortField ?? '',
+                                                    direction: pivot.sortDirection ?? 'ASC',
+                                                }, true);
                                             }, onSave: async (column, direction) => {
-                                                if (!pivot) {
-                                                    setOpenPopover(null);
-                                                    setBaseAst(deepCopy(baseAst));
-                                                    fetchSqlQuery(deepCopy(baseAst));
-                                                    return;
-                                                }
-                                                const sortFieldType = column === (pivot.valueField || 'count')
-                                                    ? 'number'
-                                                    : pivot.rowFieldType;
-                                                const tempPivot = {
-                                                    ...pivot,
-                                                    sort: true,
-                                                    sortDirection: direction,
-                                                    sortField: column,
-                                                    sortFieldType: sortFieldType,
-                                                };
-                                                setPivot(tempPivot);
-                                                setPivotHint('');
-                                                let dateBucket = undefined;
-                                                const tempDateRange = dateRanges &&
-                                                    pivot.rowField &&
-                                                    dateRanges[pivot.rowField];
-                                                if (tempDateRange) {
-                                                    dateBucket = getDateBucketFromRange(tempDateRange.dateRange);
-                                                }
-                                                try {
-                                                    const pivotedData = await generatePivotTable({
-                                                        pivot: tempPivot,
-                                                        rowLimit: -1,
-                                                        dateBucket,
-                                                        report: tempReport,
-                                                        client,
-                                                        uniqueValues: uniqueValues[currentTable],
-                                                        dashboardName,
-                                                        tenants,
-                                                    });
-                                                    setPivotData(pivotedData || []);
-                                                    const formattedRows = formatRows(pivotedData.rows, columns, true, pivot.aggregationType, dateBucket);
-                                                    setFormattedRows(formattedRows);
-                                                    setErrorMessage('');
-                                                }
-                                                catch (e) {
-                                                    if (e instanceof Error)
-                                                        setPivotError(e.message);
-                                                }
+                                                onSortChange({ field: column, direction });
                                             }, Select: SelectComponent, Button: ButtonComponent, SecondaryButton: SecondaryButtonComponent, disabled: tableLoading || loading }, `sort-sentence-pivot`) })), baseAst && baseAst.orderby && baseAst.orderby.length > 0 && (_jsx("div", { style: {
                                             display: 'flex',
                                             flexDirection: 'column',
                                             gap: 8,
                                             marginBottom: 12,
-                                        }, children: baseAst.orderby.map((sortData, id) => (_jsx(SortSentence, { sortData: sortData, columns: selectedColumns, setIsPending: () => { }, setEditPopoverKey: () => { }, setActiveEditItem: setActiveEditItem, setOpenPopover: setOpenPopover, SortPopover: SortPopoverComponent, EditPopover: AddSortPopover, handleDelete: () => {
-                                                if (pivot) {
-                                                    setPivot({ ...pivot, sort: false });
-                                                    setPivotHint('');
-                                                    return;
-                                                }
-                                                const newAst = { ...baseAst };
-                                                newAst.orderby.splice(id, 1);
-                                                if (newAst.orderby.length === 0) {
-                                                    newAst.orderby = null;
-                                                }
-                                                setBaseAst(deepCopy(newAst));
-                                                fetchSqlQuery(deepCopy(newAst));
+                                        }, children: baseAst.orderby.map((sortData, id) => (_jsx(SortSentence, { sortField: sortData.expr?.column || sortData.expr?.value, sortDirection: sortData.type, columns: columns, setIsPending: () => { }, setEditPopoverKey: () => { }, setActiveEditItem: setActiveEditItem, setOpenPopover: setOpenPopover, SortPopover: SortPopoverComponent, EditPopover: AddSortPopover, handleDelete: () => {
+                                                onSortChange({
+                                                    field: sortData.expr?.column || sortData.expr?.value,
+                                                    direction: sortData.type,
+                                                }, true);
                                             }, onSave: (column, direction) => {
-                                                if (pivot) {
-                                                    const sortFieldType = column === (pivot.valueField || 'count')
-                                                        ? 'number'
-                                                        : pivot.rowFieldType;
-                                                    setPivot({
-                                                        ...pivot,
-                                                        sort: true,
-                                                        sortDirection: direction,
-                                                        sortField: column,
-                                                        sortFieldType: sortFieldType,
-                                                    });
-                                                    setPivotHint('');
-                                                    return;
-                                                }
-                                                setActiveEditItem(null);
-                                                setOpenPopover(null);
-                                                if (column === '')
-                                                    return;
-                                                const newAst = { ...baseAst };
-                                                newAst.orderby[id] = {
-                                                    expr: {
-                                                        type: 'column_ref',
-                                                        table: null,
-                                                        column: column,
-                                                    },
-                                                    type: direction,
-                                                };
-                                                // look through the columns
-                                                setOpenPopover(null);
-                                                setBaseAst(deepCopy(newAst));
-                                                fetchSqlQuery(deepCopy(newAst));
-                                            }, Select: SelectComponent, Button: ButtonComponent, SecondaryButton: SecondaryButtonComponent, disabled: tableLoading || loading }, `sort-sentence-${id}`))) })), _jsx(SecondaryButtonComponent, { disabled: !baseAst || !dataDisplayed || loading || tableLoading, onClick: () => {
+                                                onSortChange({ field: column, direction });
+                                            }, Select: SelectComponent, Button: ButtonComponent, SecondaryButton: SecondaryButtonComponent, disabled: tableLoading || loading }, `sort-sentence-${id}`))) })), _jsx(SecondaryButtonComponent, { disabled: !baseAst ||
+                                            !dataDisplayed ||
+                                            loading ||
+                                            tableLoading ||
+                                            mssqlSortWarning, onClick: () => {
                                             if (!selectedColumns || selectedColumns.length === 0) {
                                                 return;
                                             }
                                             if (!openPopover) {
                                                 setOpenPopover('AddSortPopover');
                                             }
-                                        }, label: 'Add sort' }), _jsx("div", { style: {
+                                        }, label: 'Add sort', tooltipText: mssqlSortWarning }), _jsx("div", { style: {
                                             position: 'relative',
                                             ...(openPopover === 'AddSortPopover' && { top: 12 }),
                                         }, children: _jsx(PopoverComponent, { isOpen: openPopover === 'AddSortPopover', setIsOpen: (isOpen) => {
@@ -1467,14 +1614,7 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                                                     setActiveEditItem(null);
                                                     setOpenPopover(null);
                                                 }
-                                            }, popoverTitle: "Sort by", popoverChildren: _jsx(AddSortPopover, { columns: pivot
-                                                    ? pivot.columnField
-                                                        ? [`.${pivot.rowField}`]
-                                                        : [
-                                                            `.${pivot.rowField}`,
-                                                            `.${pivot.valueField || 'count'}`,
-                                                        ]
-                                                    : selectedColumns, Select: SelectComponent, Button: ButtonComponent, SecondaryButton: SecondaryButtonComponent, onSave: async (column, direction) => {
+                                            }, popoverTitle: "Sort by", popoverChildren: _jsx(AddSortPopover, { columns: pivotData ? pivotData.columns : columns, Select: SelectComponent, Button: ButtonComponent, SecondaryButton: SecondaryButtonComponent, onSave: async (column, direction) => {
                                                     if (column === '')
                                                         return;
                                                     if (pivot) {
@@ -1506,7 +1646,11 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                                                                 uniqueValues: uniqueValues[currentTable],
                                                                 dashboardName,
                                                                 tenants,
+                                                                additionalProcessing: {
+                                                                    page: REPORT_BUILDER_PAGINATION,
+                                                                },
                                                             });
+                                                            resetProcessing();
                                                             setErrorMessage('');
                                                             setPivotData(pivotedData || []);
                                                             const formattedRows = formatRows(pivotedData.rows, columns, true, pivot.aggregationType, dateBucket);
@@ -1520,50 +1664,43 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                                                         setBaseAst(deepCopy(baseAst));
                                                         return;
                                                     }
-                                                    const newAst = { ...baseAst };
-                                                    if (!newAst.orderby)
-                                                        newAst.orderby = [];
-                                                    const existingSortIndex = newAst.orderby.findIndex((item) => getFieldFromExpression(item.expr) === column);
-                                                    if (existingSortIndex !== -1) {
-                                                        newAst.orderby[existingSortIndex] = {
-                                                            expr: { type: 'column_ref', column },
-                                                            type: direction,
-                                                        };
-                                                    }
                                                     else {
-                                                        newAst.orderby.push({
-                                                            expr: { type: 'column_ref', column },
-                                                            type: direction,
-                                                        });
+                                                        const newAst = { ...baseAst };
+                                                        if (!newAst.orderby)
+                                                            newAst.orderby = [];
+                                                        const existingSortIndex = newAst.orderby.findIndex((item) => getFieldFromExpression(item.expr) === column);
+                                                        if (existingSortIndex !== -1) {
+                                                            newAst.orderby[existingSortIndex] = {
+                                                                expr: { type: 'column_ref', column },
+                                                                type: direction,
+                                                            };
+                                                        }
+                                                        else {
+                                                            newAst.orderby.push({
+                                                                expr: { type: 'column_ref', column },
+                                                                type: direction,
+                                                            });
+                                                        }
+                                                        // look through the columns
+                                                        setOpenPopover(null);
+                                                        setBaseAst(deepCopy(newAst));
+                                                        fetchSqlQuery(deepCopy(newAst));
                                                     }
-                                                    // look through the columns
-                                                    setOpenPopover(null);
-                                                    setBaseAst(deepCopy(newAst));
-                                                    fetchSqlQuery(deepCopy(newAst));
-                                                } }) }) })] }), _jsxs("div", { style: { width: '100%' }, children: [_jsx(SidebarHeadingComponent, { label: "Limit" }), baseAst && baseAst.limit && baseAst.limit.value?.length > 0 ? (_jsx("div", { style: {
+                                                } }) }) })] }), _jsxs("div", { style: { width: '100%' }, children: [_jsx(SidebarHeadingComponent, { label: "Limit" }), (baseAst &&
+                                        ((baseAst.limit && baseAst.limit.value?.length > 0) ||
+                                            (baseAst.top && baseAst.top.value))) ||
+                                        pivot?.rowLimit ? (_jsx("div", { style: {
                                             display: 'flex',
                                             flexDirection: 'column',
                                             gap: 8,
                                             marginBottom: 12,
-                                        }, children: _jsx(LimitSentence, { limit: baseAst.limit, setOpenPopover: setOpenPopover, LimitPopover: LimitPopoverComponent, EditPopover: AddLimitPopover, handleDelete: () => {
-                                                const newAst = { ...baseAst };
-                                                newAst.limit = null;
-                                                setBaseAst(deepCopy(newAst));
-                                                fetchSqlQuery(deepCopy(newAst));
+                                        }, children: _jsx(LimitSentence, { limit: baseAst.limit?.value?.[0]?.value ||
+                                                baseAst.top?.value ||
+                                                pivot?.rowLimit ||
+                                                0, setOpenPopover: setOpenPopover, LimitPopover: LimitPopoverComponent, EditPopover: AddLimitPopover, handleDelete: () => {
+                                                onLimitChange(null);
                                             }, onSave: (limit) => {
-                                                const newAst = { ...baseAst };
-                                                newAst.limit = {
-                                                    seperator: '',
-                                                    value: [
-                                                        {
-                                                            type: 'number',
-                                                            value: limit,
-                                                        },
-                                                    ],
-                                                };
-                                                setOpenPopover(null);
-                                                setBaseAst(deepCopy(newAst));
-                                                fetchSqlQuery(deepCopy(newAst));
+                                                onLimitChange(limit);
                                             }, TextInput: TextInputComponent, Button: ButtonComponent, SecondaryButton: SecondaryButtonComponent, disabled: tableLoading || loading }) })) : (_jsxs(_Fragment, { children: [_jsx(SecondaryButtonComponent, { disabled: !baseAst || !dataDisplayed || loading || tableLoading, onClick: () => {
                                                     if (!selectedColumns || selectedColumns.length === 0) {
                                                         return;
@@ -1583,19 +1720,7 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                                                             setOpenPopover(null);
                                                         }
                                                     }, popoverTitle: "Add limit", popoverChildren: _jsx(AddLimitPopover, { TextInputComponent: TextInputComponent, Button: ButtonComponent, SecondaryButton: SecondaryButtonComponent, onSave: (limit) => {
-                                                            const newAst = { ...baseAst };
-                                                            newAst.limit = {
-                                                                seperator: '',
-                                                                value: [
-                                                                    {
-                                                                        type: 'number',
-                                                                        value: Number(limit),
-                                                                    },
-                                                                ],
-                                                            };
-                                                            setOpenPopover(null);
-                                                            setBaseAst(deepCopy(newAst));
-                                                            fetchSqlQuery(deepCopy(newAst));
+                                                            onLimitChange(limit);
                                                         } }) }) })] }))] })] }), _jsxs(ContainerComponent, { children: [isAIEnabled && (_jsx("form", { ref: askAIContainerRef, onSubmit: (event) => {
                                     event.preventDefault();
                                 }, style: {
@@ -1611,7 +1736,7 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                                                 ? 'Ask a follow-up question...'
                                                 : 'Ask a question...' }), _jsx(ButtonComponent, { onClick: () => {
                                                 fetchAstFromPromptHelper();
-                                            }, isLoading: askAILoading && !baseAst, label: 'Ask AI' }), ((baseAst && dataDisplayed) || initialLoad) && !reportId && (_jsx(SecondaryButtonComponent, { label: 'New report', onClick: clearAllState }))] }) })), baseAst && (_jsx(TableComponent, { isLoading: tableLoading || (loading && errorMessage.length === 0), rows: formattedRows, rowCount: pivot ? undefined : numberOfRows, rowCountIsLoading: rowCountIsLoading, rowsPerPage: 20, columns: pivot
+                                            }, isLoading: askAILoading && !baseAst, label: 'Ask AI' }), ((baseAst && dataDisplayed) || initialLoad) && !reportId && (_jsx(SecondaryButtonComponent, { label: 'New report', onClick: clearAllState }))] }) })), baseAst && (_jsx(TableComponent, { isLoading: tableLoading || (loading && errorMessage.length === 0), rows: formattedRows, rowCount: pivot ? pivotData?.rowCount : numberOfRows, rowCountIsLoading: rowCountIsLoading, rowsPerPage: 20, columns: pivot
                                     ? pivotData?.columns || emptyPivotColumns()
                                     : enforceOrderOnColumns(Object.keys(rows[0] ?? {})).map((c) => {
                                         return {
@@ -1620,7 +1745,7 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                                         };
                                     }), onPageChange: onPageChange, onSortChange: (sort) => {
                                     onSortChange(sort);
-                                }, containerStyle: {
+                                }, disableSort: !!mssqlSortWarning, containerStyle: {
                                     maxHeight: Math.max(window.innerHeight - 290, 75 + Math.min(Math.max(10, rows.length), 20) * 37),
                                 } })), _jsxs("div", { style: {
                                     display: 'flex',
@@ -1668,8 +1793,10 @@ SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectCol
                                                                 rows: rows,
                                                                 pivotRows: pivotData?.rows,
                                                                 pivotColumns: pivotData?.columns,
-                                                                flags: reportInfo?.flags ??
-                                                                    tempReport?.flags,
+                                                                pivotRowCount: pivotData?.rowCount,
+                                                                pivotQuery: pivotData?.pivotQuery,
+                                                                comparisonPivotQuery: pivotData?.comparisonPivotQuery,
+                                                                flags: reportInfo?.flags ?? tempReport?.flags,
                                                             }
                                                             : {
                                                                 ...tempReport,