@quillsql/react 2.12.27 → 2.12.29

package/dist/esm/ReportBuilder.js

@@ -8,7 +8,7 @@ import { CSS as DND_CSS } from '@dnd-kit/utilities';
 import { getQuarter } from 'date-fns';
 import { ClientContext, CustomFieldContext, SchemaContext, ThemeContext, } from './Context';
 import { getTableAliases, getTableNames, isBoolColumnType, isDateishColumnType, isNumericColumnType, isTextColumnType, } from './components/ReportBuilder/ast';
-import { ChartBuilderWithModal } from './ChartBuilder';
+import { ChartBuilderWithModal, createInitialFormData } from './ChartBuilder';
 import { QuillTextInput } from './components/UiComponents';
 import { QuillSidebar, CustomContainer, QuillSelectColumn, QuillDraggableColumn, QuillSidebarHeading, QuillFilterPopover, QuillSortPopover, TagWrapper, EditPopover, AddFilterPopover, QuillLimitPopover, } from './components/ReportBuilder/ui';
 import { generateCurrentPeriodPostgres, generateEqualsPostgres, generateLastNPeriodsPostgres, generatePreviousPeriodPostgres, } from './components/ReportBuilder/postgres';
@@ -28,13 +28,15 @@ import { QuillCard } from './components/QuillCard';
 import { getData } from './utils/dataFetcher';
 import { DATE_FORMAT_TYPES, quillFormat } from './utils/valueFormatter';
 import { getPossiblePivotFieldOptions, pivotToSql, } from './utils/pivotProcessing';
-import { getUniqueValuesByColumns } from './utils/tableProcessing';
+import { getUniqueValuesByColumns, getDateRangeByColumns, getCountsByColumns, } from './utils/tableProcessing';
 import { useQuill } from './hooks/useQuill';
 import { getDataFromCloud } from './utils/dataFetcher';
 import { convertColumnInfoToColumnInternal, convertPostgresColumn, } from './utils/columnProcessing';
 import { getSelectFromAST, processApostrophe, processStarColumn, } from './utils/astProcessing';
 import PivotForm from './internals/ReportBuilder/PivotForm';
 import { getSchemaInfoWithCustomFields } from './utils/schema';
+import { MAX_COLUMN_ROWS_LIMIT } from './utils/constants';
+import { getDateBucketFromRange } from './utils/dates';
 export const QUILL_SERVER = (typeof process !== 'undefined' && process?.env?.QUILL_SERVER_HOST) ||
     'https://quill-344421.uc.r.appspot.com';
 /**
@@ -68,7 +70,7 @@ export const QUILL_SERVER = (typeof process !== 'undefined' && process?.env?.QUI
  * ### Report Builder API
  * @see https://docs.quillsql.com/components/report-builder
  */
-export default function ReportBuilder({ initialTableName = '', onSubmitEditReport = (_) => void null, onSubmitCreateReport = (_) => void null, destinationDashboard = undefined, organizationName = '', ButtonComponent = MemoizedButton, SecondaryButtonComponent = MemoizedSecondaryButton, DeleteButtonComponent = MemoizedDeleteButton, ModalComponent = MemoizedModal, TextInputComponent = QuillTextInput, SelectComponent = QuillSelectComponent, TableComponent = QuillTableSQLEditorComponent, PopoverComponent = MemoizedPopover, TabsComponent = QuillTabs, CheckboxComponent = MemoizedCheckbox, SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectColumnComponent = QuillSelectColumn, DraggableColumnComponent = QuillDraggableColumn, SidebarHeadingComponent = QuillSidebarHeading, FilterPopoverComponent = QuillFilterPopover, SortPopoverComponent = QuillSortPopover, LimitPopoverComponent = QuillLimitPopover, CardComponent = QuillCard, LabelComponent = MemoizedLabel, HeaderComponent = MemoizedHeader, SubHeaderComponent = MemoizedSubHeader, TextComponent = MemoizedText, ErrorMessageComponent = QuillErrorMessageComponent, ChartBuilderInputRowContainer = QuillChartBuilderInputRowContainer, ChartBuilderInputColumnContainer = QuillChartBuilderInputColumnContainer, PivotRowContainer = QuillPivotRowContainer, PivotColumnContainer = QuillPivotColumnContainer, LoadingComponent = QuillLoadingComponent, ColumnSearchEmptyState = QuillColumnSearchEmptyState, ChartBuilderFormContainer = QuillChartBuilderFormContainer, ChartBuilderModalComponent = MemoizedModal, isAdminEnabled = false, isAIEnabled = true, showChartBuilderTableFormatOptions = true, containerStyle, className, pivotRecommendationsEnabled = true, reportId, hideCopySQL = true, isChartBuilderHorizontalView = true, onClickChartElement }) {
+export default function ReportBuilder({ initialTableName = '', onSubmitEditReport = (_) => void null, onSubmitCreateReport = (_) => void null, destinationDashboard = undefined, organizationName = '', ButtonComponent = MemoizedButton, SecondaryButtonComponent = MemoizedSecondaryButton, DeleteButtonComponent = MemoizedDeleteButton, ModalComponent = MemoizedModal, TextInputComponent = QuillTextInput, SelectComponent = QuillSelectComponent, TableComponent = QuillTableSQLEditorComponent, PopoverComponent = MemoizedPopover, TabsComponent = QuillTabs, CheckboxComponent = MemoizedCheckbox, SidebarComponent = QuillSidebar, ContainerComponent = CustomContainer, SelectColumnComponent = QuillSelectColumn, DraggableColumnComponent = QuillDraggableColumn, SidebarHeadingComponent = QuillSidebarHeading, FilterPopoverComponent = QuillFilterPopover, SortPopoverComponent = QuillSortPopover, LimitPopoverComponent = QuillLimitPopover, CardComponent = QuillCard, LabelComponent = MemoizedLabel, HeaderComponent = MemoizedHeader, SubHeaderComponent = MemoizedSubHeader, TextComponent = MemoizedText, ErrorMessageComponent = QuillErrorMessageComponent, ChartBuilderInputRowContainer = QuillChartBuilderInputRowContainer, ChartBuilderInputColumnContainer = QuillChartBuilderInputColumnContainer, PivotRowContainer = QuillPivotRowContainer, PivotColumnContainer = QuillPivotColumnContainer, LoadingComponent = QuillLoadingComponent, ColumnSearchEmptyState = QuillColumnSearchEmptyState, ChartBuilderFormContainer = QuillChartBuilderFormContainer, ChartBuilderModalComponent = MemoizedModal, isAdminEnabled = false, isAIEnabled = true, showChartBuilderTableFormatOptions = true, containerStyle, className, pivotRecommendationsEnabled = true, reportId, hideCopySQL = true, isChartBuilderHorizontalView = true, onClickChartElement, }) {
     const { data: report } = useQuill(reportId || '');
     const [aiPrompt, setAiPrompt] = useState('');
     const [errorMessage, setErrorMessage] = useState('');
@@ -91,6 +93,7 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
     const [rows, setRows] = useState([]);
     const [formattedRows, setFormattedRows] = useState([]);
     const [columns, setColumns] = useState([]);
+    const [tempReport, setTempReport] = useState({});
     const [topLevelBinaryOperator, setTopLevelBinaryOperator] = useState('AND');
     const [editPopoverKey, setEditPopoverKey] = useState(null);
     const [uniqueValues, setUniqueValues] = useState({});
@@ -116,19 +119,23 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
     const [pivotColumnField, setPivotColumnField] = useState(undefined);
     const [pivotValueField, setPivotValueField] = useState(undefined);
     const [pivotAggregation, setPivotAggregation] = useState(undefined);
+    const [dateRanges, setDateRanges] = useState(null);
     // eslint-disable-next-line no-unused-vars
     const [client, _setClient] = useContext(ClientContext);
     // JANK: This is temp and stupid
     const [overrideRecommendations, setOverrideRecommendations] = useState(true);
     const [customFields, setCustomFields] = useContext(CustomFieldContext);
     useEffect(() => {
+        if (!client) {
+            return;
+        }
         if (client.publicKey === '663416663aa9bc716e59a89d') {
             setOverrideRecommendations(false);
         }
         if (!loadingSchema) {
             fetchSchema();
         }
-    }, [client.publicKey, client.customerId]);
+    }, [client]);
     useEffect(() => {
         updateFirstChildWidth(askAIContainerRef, setAskAIInputWidth, { gap: 12 });
         updateFirstChildWidth(askAILoadingContainerRef, setAskAILoadingContainerWidth, { gap: 12 });
@@ -146,8 +153,9 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
             window.removeEventListener('resize', handleResize);
         };
     }, []);
-    const updatePivot = (changeField, fieldKey) => {
+    const updatePivot = async (changeField, fieldKey) => {
         const newPivot = pivot;
+        setTableLoading(true);
         // @ts-ignore
         newPivot[fieldKey] = changeField;
         if (fieldKey === 'rowField') {
@@ -165,11 +173,17 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                 newPivot.sort = undefined;
             }
         }
-        setPivot(newPivot);
-        const pivotedData = generatePivotTable(newPivot, rows, undefined, false);
+        let dateBucket = undefined;
+        const tempDateRange = dateRanges && dateRanges[newPivot.rowField];
+        if (tempDateRange) {
+            dateBucket = getDateBucketFromRange(tempDateRange.dateRange);
+        }
+        const pivotedData = await generatePivotTable(pivot, rows, undefined, false, -1, undefined, dateBucket, tempReport, client, uniqueValues[currentTable]);
         setPivotData(pivotedData || []);
         const formattedRows = formatRows(pivotedData.rows, columns, true, newPivot.aggregationType);
+        setPivot(newPivot);
         setFormattedRows(formattedRows);
+        setTableLoading(false);
     };
     const enforceOrderOnColumns = (columnNames) => {
         if (pivot) {
@@ -214,10 +228,13 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
         }, 0);
     };
     useEffect(() => {
+        if (!client) {
+            return;
+        }
         if (!initialLoad && client.publicKey) {
             clearAllState();
         }
-    }, [client.publicKey, client.customerId]);
+    }, [client]);
     useEffect(() => {
         if (activePath !== null) {
             // update the modal with the new subtree
@@ -354,6 +371,34 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
             console.error(error);
         }
     };
+    const getUniqueStringValues = async (columns, tableName) => {
+        const convertedStringColumns = columns
+            .filter((column) => {
+            return isTextColumnType(column.fieldType);
+        })
+            .map((column) => convertColumnInfoToColumnInternal(column));
+        const stringNames = convertedStringColumns.map((column) => column.field);
+        const smallStringColumns = await getCountsByColumns(convertedStringColumns, `Select ${stringNames.join(', ')} from ${tableName}`, client, customFields);
+        const smallStringNames = smallStringColumns.map((column) => column.field);
+        const newUniqueValues = await getUniqueValuesByColumns(smallStringColumns, `Select ${smallStringNames.join(', ')} from ${tableName}`, [], client, customFields);
+        const joinedUniqueValues = deepCopy(uniqueValues);
+        joinedUniqueValues[tableName] = newUniqueValues;
+        return joinedUniqueValues;
+    };
+    const getDateRanges = async (columns, tableName) => {
+        const dateColumns = columns.filter((column) => {
+            return column.fieldType === 'date';
+        });
+        if (dateColumns.length === 0) {
+            return {};
+        }
+        const dateColumnNames = dateColumns.map((column) => {
+            //@ts-ignore
+            return column.field || column.name;
+        });
+        const dateRanges = await getDateRangeByColumns(dateColumns, `Select ${dateColumnNames.join(', ')} from ${tableName}`, client, customFields);
+        return dateRanges;
+    };
     // It's just like getColumnsInPivot but we expand the columnField
     // if there is one to include all the variants just like it would
     // show up in the table. (eg. category -> ...[Fuel, Food, Other])
@@ -380,18 +425,12 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
         setLoading(true);
         const tableInfo = tables.find((tableInfo) => tableInfo.name === initialTableName);
         if (tableInfo) {
-            const convertedStringColumns = tableInfo.columns
-                .filter((column) => {
-                return isTextColumnType(column.fieldType);
-            })
-                .map((column) => convertColumnInfoToColumnInternal(column));
-            const stringNames = convertedStringColumns.map((column) => column.field);
-            const newUniqueValues = await getUniqueValuesByColumns(convertedStringColumns, `Select ${stringNames.join(', ')} from ${initialTableName}`, [], client, customFields);
-            const joinedUniqueValues = deepCopy(uniqueValues);
-            joinedUniqueValues[initialTableName] = newUniqueValues;
-            if (hashCode(uniqueValues) !== hashCode(joinedUniqueValues)) {
-                setUniqueValues(joinedUniqueValues);
+            const newUniqueValues = await getUniqueStringValues(tableInfo.columns, initialTableName);
+            if (hashCode(uniqueValues) !== hashCode(newUniqueValues)) {
+                setUniqueValues(newUniqueValues);
             }
+            const dateRangesTemp = await getDateRanges(tableInfo.columns, initialTableName);
+            setDateRanges(dateRangesTemp);
         }
         const columnsForTable = tables
             .find((t) => t.name === initialTableName)
@@ -483,17 +522,13 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
             setBaseAst(newAst);
             const initialRows = await fetchUponChange(newAst, undefined);
             const tableInfo = schemaInfo.find((table) => table.name === tableName);
-            const stringColumns = tableInfo.columns
-                .filter((column) => {
-                return column.fieldType === 'varchar' || column.fieldType === 'text';
-            })
-                .map((column) => convertColumnInfoToColumnInternal(column));
-            if (stringColumns.length !== 0 && tableName) {
-                const stringNames = stringColumns.map((column) => column.field);
-                const newUniqueValues = await getUniqueValuesByColumns(stringColumns, `Select ${stringNames.join(', ')} from ${tableName}`, [], client, customFields);
-                const joinedUniqueValues = deepCopy(uniqueValues);
-                joinedUniqueValues[tableName] = newUniqueValues;
-                setUniqueValues(joinedUniqueValues);
+            let newUniqueValues = undefined;
+            let dateRangesTemp = undefined;
+            if (tableName) {
+                newUniqueValues = await getUniqueStringValues(tableInfo.columns, tableName);
+                setUniqueValues(newUniqueValues);
+                dateRangesTemp = await getDateRanges(tableInfo.columns, tableName);
+                setDateRanges(dateRangesTemp);
             }
             if (groupByPivot) {
                 // @ts-ignore
@@ -505,9 +540,16 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                 // @ts-ignore
                 setPivotValueField(groupByPivot.valueField);
                 setPivot(groupByPivot);
-                const pivotedData = generatePivotTable(
+                let dateBucket = undefined;
+                const tempDateRange = dateRangesTemp &&
+                    groupByPivot.rowField &&
+                    dateRangesTemp[groupByPivot.rowField];
+                if (tempDateRange) {
+                    dateBucket = getDateBucketFromRange(tempDateRange.dateRange);
+                }
+                const pivotedData = await generatePivotTable(
                 // @ts-ignore
-                groupByPivot, initialRows, undefined, false);
+                groupByPivot, initialRows, tempDateRange, false, -1, undefined, dateBucket, report, client, newUniqueValues[tableName]);
                 setPivotData(pivotedData || []);
                 const formattedRows = formatRows(pivotedData.rows, report.columns, true, 
                 // @ts-ignore
@@ -915,6 +957,89 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
             return [{ label: snakeAndCamelCaseToTitleCase(pivot.valueField) }];
         }
     };
+    const [previousPage, setPreviousPage] = useState(0);
+    const [currentProcessing, setCurrentProcessing] = useState({
+        page: { currentPage: 0, rowsPerPage: 20 },
+    });
+    const [numberOfRows, setNumberOfRows] = useState(0);
+    const [tableLoading, setTableLoading] = useState(false);
+    const onPageChange = (page) => {
+        if (pivot) {
+            return;
+        }
+        if ((previousPage < page.currentPage &&
+            (page.currentPage * page.rowsPerPage) % MAX_COLUMN_ROWS_LIMIT === 0) ||
+            (previousPage > page.currentPage &&
+                (previousPage * page.rowsPerPage) % MAX_COLUMN_ROWS_LIMIT === 0)) {
+            handleRunQuery({ ...currentProcessing, page });
+        }
+        setPreviousPage(page.currentPage);
+    };
+    const onSortChange = (sort) => {
+        if (pivot) {
+            return;
+        }
+        handleRunQuery({ sort, page: { currentPage: 0, rowsPerPage: 20 } });
+        setPreviousPage(0);
+    };
+    const handleRunQuery = async (processing) => {
+        try {
+            setErrorMessage('');
+            setTableLoading(true);
+            const hostedBody = {
+                metadata: {
+                    query: activeQuery,
+                    task: 'query',
+                    orgId: client.customerId || '*',
+                    clientId: client.publicKey,
+                    databaseType: client?.databaseType,
+                    getCustomFields: !client.customerId || client.customerId === '*' ? false : true,
+                    customFieldsByTable: customFields,
+                    additionalProcessing: processing,
+                    useUpdatedDataGathering: true,
+                },
+            };
+            const cloudBody = { activeQuery };
+            const resp = await getData(client, 'dashquery', 'same-origin', hostedBody, cloudBody);
+            if (resp && resp.errorMessage) {
+                setTableLoading(false);
+                setErrorMessage('Failed to run SQL query: ' + resp.errorMessage);
+                setRows([]);
+                setColumns([]);
+                return;
+            }
+            if (resp.rowCount) {
+                setNumberOfRows(resp.rowCount);
+            }
+            setErrorMessage('');
+            setCurrentProcessing(processing);
+            if (resp.rows.length === 0) {
+                setErrorMessage('No data found');
+            }
+            const temp_rows = resp.rows && resp.rows.length ? resp.rows : [];
+            const processedFields = resp.fields.map((elem) => convertPostgresColumn(elem));
+            setRows(temp_rows);
+            // setNumberOfRows(resp.compareRows[0]?.count || temp_rows.length);
+            setFormattedRows(temp_rows.map((row) => {
+                return processedFields.reduce((formattedRow, column) => {
+                    // Apply the format function to each field in the row
+                    const formattedValue = quillFormat({
+                        value: row[column.field],
+                        format: column.format,
+                    });
+                    formattedRow[column.field] = formattedValue;
+                    return formattedRow;
+                }, {});
+            }));
+            setColumns(processedFields);
+            setTableLoading(false);
+        }
+        catch (e) {
+            setTableLoading(false);
+            console.log('ERROR: ', e);
+            return;
+        }
+    };
     /**
      * Render form fields based on the type of the node
      * @param node the AST or subtree to render recursively
@@ -1780,6 +1905,8 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                         task: 'patterns',
                         getCustomFields: false,
                         customFields,
+                        additionalProcessing: { page: { currentPage: 0, rowsPerPage: 20 } },
+                        useUpdatedDataGathering: true,
                     },
                 };
                 const cloudBody = {};
@@ -1788,23 +1915,23 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                     throw new Error(data2.errorMessage);
                 }
                 rows = data2.rows;
+                if (data2.rowCount) {
+                    setNumberOfRows(data2.rowCount);
+                }
                 const tables = getTableNames(baseAst);
                 const table = tables.length >= 1 ? tables[0] : initialTableName;
+                let newUniqueValues = uniqueValues;
+                let dateRangesTemp = dateRanges;
+                let curReport = undefined;
                 if (table !== currentTable) {
                     const tableInfo = schema.find((tableInfo) => tableInfo.name === table);
                     if (tableInfo) {
-                        const convertedStringColumns = tableInfo.columns
-                            .filter((column) => {
-                            return isTextColumnType(column.fieldType);
-                        })
-                            .map((column) => convertColumnInfoToColumnInternal(column));
-                        const stringNames = convertedStringColumns.map((column) => column.field);
-                        const newUniqueValues = await getUniqueValuesByColumns(convertedStringColumns, `Select ${stringNames.join(', ')} from ${table}`, rows, client, customFields);
-                        const joinedUniqueValues = deepCopy(uniqueValues);
-                        joinedUniqueValues[table] = newUniqueValues;
-                        if (hashCode(uniqueValues) !== hashCode(joinedUniqueValues)) {
-                            setUniqueValues(joinedUniqueValues);
+                        newUniqueValues = await getUniqueStringValues(tableInfo.columns, table);
+                        if (hashCode(uniqueValues) !== hashCode(newUniqueValues)) {
+                            setUniqueValues(newUniqueValues);
                         }
+                        dateRangesTemp = await getDateRanges(tableInfo.columns, table);
+                        setDateRanges(dateRangesTemp || {});
                     }
                     setCurrentTable(table);
                 }
@@ -1817,6 +1944,22 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                             setPivotData(null);
                             setRows(data2.rows);
                             setColumns(processedFields);
+                            if (data2.rowCount) {
+                                const processedFormData = report
+                                    ? report
+                                    : createInitialFormData(processedFields);
+                                setNumberOfRows(data2.rowCount);
+                                curReport = {
+                                    ...formData,
+                                    ...processedFormData,
+                                    itemQuery: data2.itemQuery,
+                                    rowCount: data2.rowCount,
+                                    filtersApplied: [],
+                                    rows: data2.rows,
+                                    columns: processedFields,
+                                };
+                                setTempReport(curReport);
+                            }
                             const formattedRows = formatRows(data2.rows, processedFields, false);
                             setFormattedRows(formattedRows);
                             return;
@@ -1828,12 +1971,36 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                             return acc;
                         }, {});
                         uniqueFormatted[pivot.columnField] = uniqueRecords;
-                        const pivotedData = generatePivotTable(pivot, data2.rows, undefined, false);
+                        let dateBucket = undefined;
+                        const tempDateRange = dateRangesTemp &&
+                            pivot.rowField &&
+                            dateRangesTemp[pivot.rowField];
+                        if (tempDateRange) {
+                            dateBucket = getDateBucketFromRange(tempDateRange.dateRange);
+                        }
+                        const pivotedData = await generatePivotTable(
+                        // @ts-ignore
+                        pivot, data2.rows, undefined, false, -1, undefined, dateBucket, curReport, client, uniqueFormatted[pivot.columnField]);
                         console.info(`%c[Pivot]: ${JSON.stringify(pivot)}`, 'color: dimgray');
                         const processedFields = data2.fields.map((elem) => convertPostgresColumn(elem));
                         setPivotData(pivotedData);
                         setRows(data2.rows);
                         setColumns(processedFields);
+                        if (data2.rowCount) {
+                            const processedFormData = report
+                                ? report
+                                : createInitialFormData(processedFields);
+                            setNumberOfRows(data2.rowCount);
+                            setTempReport({
+                                ...formData,
+                                ...processedFormData,
+                                itemQuery: data2.itemQuery,
+                                rowCount: data2.rowCount,
+                                filtersApplied: [],
+                                rows: data2.rows,
+                                columns: processedFields,
+                            });
+                        }
                         const formattedRows = formatRows(pivotedData.rows, processedFields, true, pivot.aggregationType);
                         setSelectedColumns(processedFields.map((column) => {
                             return `${table}.${column.field}`;
@@ -1844,6 +2011,21 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                         const processedFields = data2.fields.map((elem) => convertPostgresColumn(elem));
                         setRows(data2.rows);
                         setColumns(processedFields);
+                        if (data2.rowCount) {
+                            const processedFormData = report
+                                ? report
+                                : createInitialFormData(processedFields);
+                            setNumberOfRows(data2.rowCount);
+                            setTempReport({
+                                ...formData,
+                                ...processedFormData,
+                                itemQuery: data2.itemQuery,
+                                rowCount: data2.rowCount,
+                                filtersApplied: [],
+                                rows: data2.rows,
+                                columns: processedFields,
+                            });
+                        }
                         setSelectedColumns(processedFields.map((column) => {
                             return `${table}.${column.field}`;
                         }));
@@ -2078,6 +2260,9 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                     publicKey: client.publicKey,
                     orgId: client.customerId,
                     task: 'patterns',
+                    additionalProcessing: { page: { currentPage: 0, rowsPerPage: 20 } },
+                    useUpdatedDataGathering: true,
+                    pivot: groupByPivot,
                 },
             };
             const cloudBody = {};
@@ -2086,20 +2271,16 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                 throw new Error('Error querying data from patterns');
             }
             let currentUniqueValues = uniqueValues;
+            let dateRangesTemp = dateRanges;
             if (table !== currentTable) {
                 const tableInfo = schema.find((tableInfo) => tableInfo.name === table);
                 if (tableInfo) {
-                    const convertedStringColumns = tableInfo.columns
-                        .filter((column) => {
-                        return isTextColumnType(column.fieldType);
-                    })
-                        .map((column) => convertColumnInfoToColumnInternal(column));
-                    const stringNames = convertedStringColumns.map((column) => column.field);
-                    const newUniqueValues = await getUniqueValuesByColumns(convertedStringColumns, `Select ${stringNames.join(', ')} from ${table}`, rows, client, customFields);
-                    currentUniqueValues[table] = newUniqueValues;
-                    if (hashCode(uniqueValues) !== hashCode(currentUniqueValues)) {
-                        setUniqueValues(currentUniqueValues);
+                    const newUniqueValues = await getUniqueStringValues(tableInfo.columns, table);
+                    if (hashCode(uniqueValues) !== hashCode(newUniqueValues)) {
+                        setUniqueValues(newUniqueValues);
                     }
+                    dateRangesTemp = await getDateRanges(tableInfo.columns, table);
+                    setDateRanges(dateRangesTemp);
                 }
                 setCurrentTable(table);
             }
@@ -2142,15 +2323,43 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                     errored = true;
                 }
                 if (groupByPivot && possiblePivot) {
-                    setPivotRowField(groupByPivot?.rowField);
-                    setPivotColumnField(groupByPivot?.columnField);
-                    setPivotValueField(groupByPivot?.valueField);
-                    setPivotAggregation(groupByPivot?.aggregationType);
-                    const pivotedData = generatePivotTable(groupByPivot, data2.rows, undefined, false);
+                    let curReport = report ? report : undefined;
+                    if (data2.rowCount) {
+                        const processedFormData = report
+                            ? report
+                            : createInitialFormData(processedFields);
+                        setNumberOfRows(data2.rowCount);
+                        curReport = {
+                            ...formData,
+                            ...processedFormData,
+                            itemQuery: data2.itemQuery,
+                            rowCount: data2.rowCount,
+                            filtersApplied: [],
+                            rows: data2.rows,
+                            columns: processedFields,
+                        };
+                        setTempReport(curReport || null);
+                    }
+                    let dateBucket = undefined;
+                    const tempDateRange = dateRangesTemp &&
+                        groupByPivot.rowField &&
+                        dateRangesTemp[groupByPivot.rowField];
+                    if (tempDateRange) {
+                        dateBucket = getDateBucketFromRange(tempDateRange.dateRange);
+                    }
+                    const pivotedData = await generatePivotTable(
+                    // @ts-ignore
+                    groupByPivot, data2.rows, undefined, false, -1, undefined, dateBucket, curReport, client, groupByPivot.columnField
+                        ? currentUniqueValues[groupByPivot.columnField]
+                        : undefined);
                     console.info(`%c[Pivot]: ${JSON.stringify(groupByPivot)}`, 'color: dimgray');
                     setPivotData(pivotedData);
                     setPivot(groupByPivot);
                     setRows(data2.rows);
+                    setPivotRowField(groupByPivot?.rowField);
+                    setPivotColumnField(groupByPivot?.columnField);
+                    setPivotValueField(groupByPivot?.valueField);
+                    setPivotAggregation(groupByPivot?.aggregationType);
                     setColumns(processedFields);
                     const formattedRows = formatRows(pivotedData.rows, processedFields, true, groupByPivot.aggregationType);
                     setFormattedRows(formattedRows);
@@ -2159,6 +2368,17 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                     const processedFields = data2.fields.map((elem) => convertPostgresColumn(elem));
                     setRows(data2.rows);
                     setColumns(processedFields);
+                    if (data2.rowCount) {
+                        setNumberOfRows(data2.rowCount);
+                        setTempReport({
+                            ...formData,
+                            itemQuery: data2.itemQuery,
+                            rowCount: data2.rowCount,
+                            filtersApplied: [],
+                            rows: data2.rows,
+                            columns: processedFields,
+                        });
+                    }
                     const formattedRows = formatRows(data2.rows, processedFields, false);
                     setFormattedRows(formattedRows);
                 }
@@ -2433,17 +2653,8 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                                     }, 
                                     // TODOs
                                     selectPivot: (pivot) => {
-                                        if (!pivot)
-                                            return;
-                                        const newAst = { ...baseAst };
-                                        newAst.orderby = null;
-                                        setBaseAst(newAst); // trigger refetch
-                                        setPivot(pivot);
-                                        const pivotedData = generatePivotTable(pivot, rows, undefined, false);
-                                        setPivotData(pivotedData || []);
-                                        const formattedRows = formatRows(pivotedData.rows, columns, true, pivot.aggregationType);
-                                        setFormattedRows(formattedRows);
-                                    }, selectPivotOnEdit: true, showTrigger: !pivot, theme: theme, LabelComponent: LabelComponent, HeaderComponent: HeaderComponent, dateRange: undefined, pivotCountRequest: 4, SecondaryButtonComponent: SecondaryButtonComponent, query: activeQuery, initialUniqueValues: uniqueValues[currentTable], disabled: !loading && (!baseAst || !dataDisplayed), pivotRecommendationsEnabled: pivotRecommendationsEnabled && overrideRecommendations }), pivot && (_jsx(PivotForm, { columns: columns, uniqueValues: uniqueValues[currentTable], setPivotRowField: (value) => {
+                                        return;
+                                    }, selectPivotOnEdit: true, showTrigger: !pivot, theme: theme, LabelComponent: LabelComponent, HeaderComponent: HeaderComponent, dateRange: undefined, pivotCountRequest: 4, SecondaryButtonComponent: SecondaryButtonComponent, query: activeQuery, initialUniqueValues: uniqueValues[currentTable], disabled: !loading && (!baseAst || !dataDisplayed), pivotRecommendationsEnabled: pivotRecommendationsEnabled && overrideRecommendations, report: tempReport ?? report }), pivot && (_jsx(PivotForm, { columns: columns, uniqueValues: uniqueValues[currentTable], setPivotRowField: (value) => {
                                         setPivotRowField(value);
                                     }, setPivotColumnField: setPivotColumnField, setPivotValueField: setPivotValueField, setPivotAggregation: setPivotAggregation, pivotRowField: pivotRowField, pivotColumnField: pivotColumnField, pivotValueField: pivotValueField, pivotAggregation: pivotAggregation, onDelete: () => {
                                         setPivot(null);
@@ -2737,7 +2948,7 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                                             setPivotData(null);
                                             const formattedRows = formatRows(rows, columns, false);
                                             setFormattedRows(formattedRows);
-                                        }, selectPivot: (pivot) => {
+                                        }, selectPivot: async (pivot, uniqueValues, dateRange, pivotTable) => {
                                             if (!pivot)
                                                 return;
                                             const newAst = { ...baseAst };
@@ -2747,13 +2958,19 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                                                 pivot['sortDirection'] = 'ASC';
                                             }
                                             setBaseAst(newAst); // trigger refetch
+                                            let dateBucket = undefined;
+                                            if (dateRange) {
+                                                dateBucket = getDateBucketFromRange(dateRange);
+                                            }
+                                            if (!pivotTable) {
+                                                pivotTable = await generatePivotTable(pivot, rows, undefined, false, -1, undefined, dateBucket, tempReport, client, uniqueValues);
+                                            }
+                                            setPivotData(pivotTable || []);
                                             setPivot(pivot);
-                                            const pivotedData = generatePivotTable(pivot, rows, undefined, false);
-                                            setPivotData(pivotedData || []);
-                                            const formattedRows = formatRows(pivotedData.rows, columns, true, pivot.aggregationType);
+                                            const formattedRows = formatRows(pivotTable.rows, columns, true, pivot.aggregationType);
                                             setFormattedRows(formattedRows);
                                             setErrorMessage('');
-                                        }, selectPivotOnEdit: true, showTrigger: !pivot, theme: theme, LabelComponent: LabelComponent, HeaderComponent: HeaderComponent, dateRange: undefined, pivotCountRequest: 4, query: activeQuery, initialUniqueValues: uniqueValues[currentTable], disabled: !baseAst || !dataDisplayed, pivotRecommendationsEnabled: pivotRecommendationsEnabled && overrideRecommendations }), pivot && (_jsx(PivotForm, { columns: columns, uniqueValues: uniqueValues[currentTable], setPivotRowField: (value) => {
+                                        }, selectPivotOnEdit: true, showTrigger: !pivot, theme: theme, LabelComponent: LabelComponent, HeaderComponent: HeaderComponent, dateRange: undefined, pivotCountRequest: 4, query: activeQuery, initialUniqueValues: uniqueValues[currentTable], disabled: !baseAst || !dataDisplayed, pivotRecommendationsEnabled: pivotRecommendationsEnabled && overrideRecommendations, report: tempReport }), pivot && (_jsx(PivotForm, { columns: columns, uniqueValues: uniqueValues[currentTable], setPivotRowField: (value) => {
                                             setPivotRowField(value);
                                             updatePivot(value, 'rowField');
                                         }, setPivotColumnField: (value) => {
@@ -2785,10 +3002,18 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                                                         `.${pivot.rowField}`,
                                                         `.${pivot.valueField || 'count'}`,
                                                     ]
-                                                : selectedColumns, setIsPending: setIsPending, setEditPopoverKey: setEditPopoverKey, setActiveEditItem: setActiveEditItem, setActivePath: setActivePath, setOpenPopover: setOpenPopover, SortPopover: SortPopoverComponent, EditPopover: AddSortPopover, handleDelete: () => {
+                                                : selectedColumns, setIsPending: setIsPending, setEditPopoverKey: setEditPopoverKey, setActiveEditItem: setActiveEditItem, setActivePath: setActivePath, setOpenPopover: setOpenPopover, SortPopover: SortPopoverComponent, EditPopover: AddSortPopover, handleDelete: async () => {
                                                 if (pivot) {
-                                                    setPivot({ ...pivot, sort: false });
-                                                    const pivotedData = generatePivotTable({ ...pivot, sort: false }, rows, undefined, false);
+                                                    const tempPivot = { ...pivot, sort: false };
+                                                    let dateBucket = undefined;
+                                                    const tempDateRange = dateRanges &&
+                                                        pivot.rowField &&
+                                                        dateRanges[pivot.rowField];
+                                                    if (tempDateRange) {
+                                                        dateBucket = getDateBucketFromRange(tempDateRange.dateRange);
+                                                    }
+                                                    setPivot(tempPivot);
+                                                    const pivotedData = await generatePivotTable(tempPivot, rows, undefined, false, -1, undefined, dateBucket, tempReport, client, uniqueValues[currentTable]);
                                                     setPivotData(pivotedData || []);
                                                     const formattedRows = formatRows(pivotedData.rows, columns, true, pivot.aggregationType);
                                                     setFormattedRows(formattedRows);
@@ -2797,25 +3022,27 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                                                 }
                                                 setBaseAst(deepCopy(baseAst));
                                                 fetchSqlQuery(deepCopy(baseAst));
-                                            }, onSave: (column, direction) => {
+                                            }, onSave: async (column, direction) => {
                                                 if (pivot) {
                                                     const sortFieldType = column === (pivot.valueField || 'count')
                                                         ? 'number'
                                                         : pivot.rowFieldType;
-                                                    setPivot({
-                                                        ...pivot,
-                                                        sort: true,
-                                                        sortDirection: direction,
-                                                        sortField: column,
-                                                        sortFieldType: sortFieldType,
-                                                    });
-                                                    const pivotedData = generatePivotTable({
+                                                    const tempPivot = {
                                                         ...pivot,
                                                         sort: true,
                                                         sortDirection: direction,
                                                         sortField: column,
                                                         sortFieldType: sortFieldType,
-                                                    }, rows, undefined, false);
+                                                    };
+                                                    setPivot(tempPivot);
+                                                    let dateBucket = undefined;
+                                                    const tempDateRange = dateRanges &&
+                                                        pivot.rowField &&
+                                                        dateRanges[pivot.rowField];
+                                                    if (tempDateRange) {
+                                                        dateBucket = getDateBucketFromRange(tempDateRange.dateRange);
+                                                    }
+                                                    const pivotedData = await generatePivotTable(tempPivot, rows, undefined, false, -1, undefined, dateBucket, tempReport, client, uniqueValues[currentTable]);
                                                     setPivotData(pivotedData || []);
                                                     const formattedRows = formatRows(pivotedData.rows, columns, true, pivot.aggregationType);
                                                     setFormattedRows(formattedRows);
@@ -2896,27 +3123,29 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                                                             `.${pivot.rowField}`,
                                                             `.${pivot.valueField || 'count'}`,
                                                         ]
-                                                    : selectedColumns, Select: SelectComponent, Button: ButtonComponent, SecondaryButton: SecondaryButtonComponent, onSave: (column, direction) => {
+                                                    : selectedColumns, Select: SelectComponent, Button: ButtonComponent, SecondaryButton: SecondaryButtonComponent, onSave: async (column, direction) => {
                                                     if (column === '')
                                                         return;
                                                     if (pivot) {
                                                         const sortFieldType = column === (pivot.valueField || 'count')
                                                             ? 'number'
                                                             : pivot.rowFieldType;
-                                                        setPivot({
+                                                        const tempPivot = {
                                                             ...pivot,
                                                             sort: true,
                                                             sortDirection: direction,
                                                             sortField: column,
                                                             sortFieldType: sortFieldType,
-                                                        });
-                                                        const pivotedData = generatePivotTable({
-                                                            ...pivot,
-                                                            sort: true,
-                                                            sortDirection: direction,
-                                                            sortField: column,
-                                                            sortFieldType: sortFieldType,
-                                                        }, rows, undefined, false);
+                                                        };
+                                                        setPivot(tempPivot);
+                                                        let dateBucket = undefined;
+                                                        const tempDateRange = dateRanges &&
+                                                            pivot.rowField &&
+                                                            dateRanges[pivot.rowField];
+                                                        if (tempDateRange) {
+                                                            dateBucket = getDateBucketFromRange(tempDateRange.dateRange);
+                                                        }
+                                                        const pivotedData = await generatePivotTable(tempPivot, rows, undefined, false, -1, undefined, dateBucket, tempReport, client, uniqueValues[currentTable]);
                                                         setErrorMessage('');
                                                         setPivotData(pivotedData || []);
                                                         const formattedRows = formatRows(pivotedData.rows, columns, true, pivot.aggregationType);
@@ -3009,14 +3238,16 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                                                 ? askAIInputWidth
                                                 : askAILoadingContainerWidth, onChange: (e) => setAiPrompt(e.target.value), placeholder: askedAQuestion
                                                 ? 'Ask a follow-up question...'
-                                                : 'Ask a question...' }), _jsx(ButtonComponent, { onClick: handleAsk, label: 'Ask AI' }), ((baseAst && dataDisplayed) || initialLoad) && (_jsx(SecondaryButtonComponent, { label: 'New report', onClick: clearAllState }))] }) })), baseAst && (_jsx(TableComponent, { isLoading: (loading && errorMessage.length === 0) || initalChartLoad, rows: formattedRows, columns: pivot
+                                                : 'Ask a question...' }), _jsx(ButtonComponent, { onClick: handleAsk, label: 'Ask AI' }), ((baseAst && dataDisplayed) || initialLoad) && (_jsx(SecondaryButtonComponent, { label: 'New report', onClick: clearAllState }))] }) })), baseAst && (_jsx(TableComponent, { isLoading: tableLoading ||
+                                    (loading && errorMessage.length === 0) ||
+                                    initalChartLoad, rows: formattedRows, rowCount: pivot ? undefined : numberOfRows, columns: pivot
                                     ? pivotData?.columns || emptyPivotColumns()
                                     : enforceOrderOnColumns(Object.keys(rows[0] ?? {})).map((c) => {
                                         return {
                                             label: snakeAndCamelCaseToTitleCase(c),
                                             field: c,
                                         };
-                                    }) })), _jsxs("div", { style: {
+                                    }), onPageChange: onPageChange, onSortChange: onSortChange })), _jsxs("div", { style: {
                                     display: 'flex',
                                     flexDirection: 'row',
                                     gap: '12px',
@@ -3032,6 +3263,7 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                                                 }, label: report ? 'Save changes' : 'Add to dashboard' })] }))] })] }), _jsx("style", { children: `body{margin:0;}` })] })), (!isChartBuilderHorizontalView || isChartBuilderOpen) && (_jsx(ChartBuilderWithModal, { report: report
                     ? {
                         ...report,
+                        ...tempReport,
                         pivot: pivot,
                         yAxisFields: report.pivot && !pivot ? [] : report.yAxisFields,
                         columns: report.columns.filter((col) => {
@@ -3042,5 +3274,5 @@ export default function ReportBuilder({ initialTableName = '', onSubmitEditRepor
                         queryString: activeQuery,
                         rows: rows,
                     }
-                    : undefined, rows: rows, columns: columns, pivot: pivot, query: activeQuery, showTableFormatOptions: showChartBuilderTableFormatOptions, showDateFieldOptions: isAdminEnabled, showAccessControlOptions: isAdminEnabled, title: report ? 'Save changes' : 'Add to dashboard', isHorizontalView: true, isOpen: isChartBuilderOpen, setIsOpen: setIsChartBuilderOpen, onAddToDashboardComplete: report ? onSubmitEditReport : onSubmitCreateReport, destinationDashboard: destinationDashboard, organizationName: organizationName, pivotData: pivotData, initialUniqueValues: uniqueValues[currentTable], pivotRecommendationsEnabled: pivotRecommendationsEnabled && overrideRecommendations, SelectComponent: SelectComponent, TextInputComponent: TextInputComponent, ButtonComponent: ButtonComponent, SecondaryButtonComponent: SecondaryButtonComponent, HeaderComponent: HeaderComponent, SubHeaderComponent: SubHeaderComponent, LabelComponent: LabelComponent, TextComponent: TextComponent, CardComponent: CardComponent, ModalComponent: ChartBuilderModalComponent, PopoverComponent: PopoverComponent, TableComponent: TableComponent, DeleteButtonComponent: DeleteButtonComponent, LoadingComponent: LoadingComponent, ChartBuilderInputRowContainer: ChartBuilderInputRowContainer, ChartBuilderInputColumnContainer: ChartBuilderInputColumnContainer, FormContainer: ChartBuilderFormContainer, hideDateRangeFilter: true, buttonLabel: report ? 'Save changes' : 'Add to dashboard', onClickChartElement: onClickChartElement }))] }));
+                    : tempReport, rows: rows, columns: columns, pivot: pivot, query: activeQuery, showTableFormatOptions: showChartBuilderTableFormatOptions, showDateFieldOptions: isAdminEnabled, showAccessControlOptions: isAdminEnabled, title: report ? 'Save changes' : 'Add to dashboard', isHorizontalView: true, isOpen: isChartBuilderOpen, setIsOpen: setIsChartBuilderOpen, onAddToDashboardComplete: report ? onSubmitEditReport : onSubmitCreateReport, destinationDashboard: destinationDashboard, organizationName: organizationName, pivotData: pivotData, initialUniqueValues: uniqueValues[currentTable], pivotRecommendationsEnabled: pivotRecommendationsEnabled && overrideRecommendations, SelectComponent: SelectComponent, TextInputComponent: TextInputComponent, ButtonComponent: ButtonComponent, SecondaryButtonComponent: SecondaryButtonComponent, HeaderComponent: HeaderComponent, SubHeaderComponent: SubHeaderComponent, LabelComponent: LabelComponent, TextComponent: TextComponent, CardComponent: CardComponent, ModalComponent: ChartBuilderModalComponent, PopoverComponent: PopoverComponent, TableComponent: TableComponent, DeleteButtonComponent: DeleteButtonComponent, LoadingComponent: LoadingComponent, ChartBuilderInputRowContainer: ChartBuilderInputRowContainer, ChartBuilderInputColumnContainer: ChartBuilderInputColumnContainer, FormContainer: ChartBuilderFormContainer, hideDateRangeFilter: true, buttonLabel: report ? 'Save changes' : 'Add to dashboard', onClickChartElement: onClickChartElement }))] }));
 }