@quillsql/react 2.12.29 → 2.12.31

package/dist/esm/components/ReportBuilder/convert.js

@@ -1,6 +1,8 @@
+import { isValid, parseISO } from 'date-fns';
 import { generatePivotTitle } from '../../internals/ReportBuilder/PivotModal';
-import { extractColumnish, isSelect, isLogicalBinaryExpr, isBigQuery, isNumericColumnType, } from './ast';
-import { generateColumnExpr, isColumnDateish, isIdColumn } from './util';
+import { extractColumnish, isSelect, isLogicalBinaryExpr, isBigQuery, isNumericColumnType, getColumnName, } from './ast';
+import { generateColumnExpr, isIdColumn } from './util';
+import { convertColumnInfoToColumnInternal } from '../../utils/columnProcessing';
 // Helper functions that preprocess the AST.
 // Find and replace certain pre-set date comparison queries with their
 // more stable counterparts.
@@ -65,7 +67,7 @@ export function recursiveSearchAndReplaceByKey(node, search, replace, searchKey)
 export function convertBigQuery(ast) {
     // For BigQuery, the info we want is nested inside the 'select' key
     if (isBigQuery(ast)) {
-        let newAst = ast.select;
+        const newAst = ast.select;
         newAst.from?.forEach((tbl) => {
             tbl.table = tbl.table.replaceAll('`', '');
         });
@@ -86,7 +88,7 @@ export function convertBigQuery(ast) {
         recursiveSearchAndReplaceByKey(newAst.where, 'week', '* 7 DAY', 'unit');
         recursiveSearchAndReplaceByKey(newAst.where, 'month', '* 30 DAY', 'unit');
         // TODO: handle joins nicely
-        if (newAst.from?.length > 1) {
+        if (newAst.from && newAst.from.length > 1) {
             newAst.from = [newAst.from[0]];
         }
         // TODO: handle UNION ALL joins
@@ -97,7 +99,7 @@ export function convertBigQuery(ast) {
     }
     else {
         // TODO: handle joins nicely
-        if (ast.from?.length > 1) {
+        if (ast.from && ast.from.length > 1) {
             ast.from = [ast.from[0]];
         }
         // TODO: handle UNION ALL joins
@@ -107,12 +109,18 @@ export function convertBigQuery(ast) {
         return ast;
     }
 }
+function columnExistsInNewColumns(columns, col) {
+    return columns.find((c) => [c.expr?.value, c.expr?.column, c.expr?.column?.expr?.value, c.as].includes(col));
+}
 /**
  * Replaces the wildcard column selector with their literal column names.
  */
 export function convertWildcardColumns(ast, schemaTables) {
-    if (ast.columns !== '*' && ast.columns.length !== 0)
+    if (ast.columns !== '*' &&
+        ast.columns.length !== 0 &&
+        (!ast.columns[0].expr || ast.columns[0].expr.column !== '*')) {
         return ast;
+    }
     if (!ast.from)
         return ast;
     // Map SELECT(*) to their actual columns
@@ -134,348 +142,63 @@ export function convertWildcardColumns(ast, schemaTables) {
     };
 }
 /**
- * Processes the AST and builds up a new pivot, given the old AST, the old pivot
- * and a list of tables in the schema.
+ * Take a list of AST column objects of any form and turn them into the generic select form
  */
-export function convertGroupBy(ast, prevPivot = null, schemaTables) {
-    if (!isSelect(ast))
-        return { ast, pivot: prevPivot };
-    let newAst = {};
-    let pivot = {};
-    let newColumns = [];
-    let aliasMap = {};
-    const tableNamesInQuery = (ast.from ?? []).map((tbl) => {
-        // This assumes that all the entries in the from clause are BaseFrom.
-        // TODO: Handle Join, TableExpr, and Dual (maybe not the last one).
-        return tbl.table;
+export function convertColumnToGenericColumns(astColumns) {
+    const genericColumnReferences = [];
+    astColumns.forEach((column) => {
+        const columnName = getColumnName(column);
+        if (!columnName) {
+            return;
+        }
+        genericColumnReferences.push({
+            type: 'expr',
+            expr: {
+                type: 'column_ref',
+                table: null,
+                column: columnName,
+            },
+            as: columnName,
+        });
     });
-    const columns = schemaTables
-        .filter((t) => tableNamesInQuery.includes(t.displayName))
-        .flatMap((table) => table.columns)
-        .map((column) => ({
-        field: column.displayName,
-        fieldType: column.fieldType,
-    }));
-    // If there is not GROUP BY clause in the AST, we still need to process the
-    // AST but it should pass-through in nature.
-    if (!ast.groupby) {
-        // If there was a pivot before, evaluate the pivot.
-        if (prevPivot) {
-            // Make sure that the AST includes all the columns on the prevPivot
-            const columnAlreadyExists = (col) => ast.columns.find((c) => [c.expr?.value, c.expr?.column, c.as].includes(col));
-            if (prevPivot.valueField) {
-                const existingCol = columnAlreadyExists(prevPivot.valueField);
-                newColumns.push(existingCol ?? generateColumnExpr(prevPivot.valueField));
-            }
-            if (prevPivot.rowField) {
-                const existingCol = columnAlreadyExists(prevPivot.rowField);
-                newColumns.push(existingCol ?? generateColumnExpr(prevPivot.rowField));
-            }
-            if (prevPivot.columnField) {
-                const existingCol = columnAlreadyExists(prevPivot.columnField);
-                newColumns.push(existingCol ?? generateColumnExpr(prevPivot.columnField));
-            }
-            ast.columns = newColumns;
-            return { pivot: prevPivot, ast };
+    return genericColumnReferences;
+}
+function cleanASTColumns(ast, columns) {
+    let newColumns = convertColumnToGenericColumns(ast.columns);
+    const columnNames = [];
+    newColumns = newColumns.filter((col) => {
+        if (!columnNames.includes(col.as)) {
+            columnNames.push(col.as);
+            return true;
         }
-        for (let i = 0; i < ast.columns.length; i++) {
-            const column = ast.columns[i];
-            // Extract out the column name, note that there are several ways
-            // that a column node could be structured so we need to check.
-            let columnName;
-            let columnAs = null;
-            let columnTable = null;
-            if (column.type === 'expr') {
-                columnAs = column.as;
-                columnTable = column.expr.table;
-                if (column.expr.type === 'column_ref') {
-                    columnName = column.expr.column;
-                }
-                else if (column.expr.type === 'string') {
-                    columnName = column.expr.value;
-                }
-                else if (column.expr.type === 'double_quote_string') {
-                    columnName = column.expr?.value;
-                }
-                else if (column.expr?.args?.expr) {
-                    columnName = column.expr?.args?.expr?.value;
-                }
-                if (column.expr.type === 'aggr_func' && !isIdColumn(columnName)) {
-                    pivot.aggregationType = column.expr.name.toLowerCase();
-                    if (pivot.aggregationType === 'avg') {
-                        pivot.aggregationType = 'average';
-                    }
-                    pivot.valueField =
-                        column.expr.args?.expr?.value || column.expr.args?.expr?.column;
-                    if (column.expr.args.expr.type === 'column_ref') {
-                        columnTable = column.expr.args.expr.table;
-                        columnName = column.expr.args.expr.column;
-                    }
-                    else if (column.expr.args.expr.type === 'double_quote_string') {
-                        columnName = column.expr.args.expr?.value;
-                    }
-                    else if (column.expr?.args?.expr) {
-                        columnName = column.expr?.args?.expr?.value;
-                    }
-                }
-            }
-            if (columnName === '*') {
-                // Use the first numeric column found, if none found, use the first column
-                let columnForValueField = columns.find((c) => {
-                    return isNumericColumnType(c.fieldType) && !isIdColumn(c.field);
-                });
-                if (!columnForValueField) {
-                    columnForValueField = columns[0];
-                }
-                for (let i = 0; i < columns.length; i++) {
-                    if (!isIdColumn(columns[i]?.field) &&
-                        isNumericColumnType(columns[i]?.fieldType)) {
-                        columnForValueField = columns[i];
-                        break;
-                    }
-                }
-                pivot.valueField = columnForValueField.field;
-                // pivot.valueFieldType = 'number';
-                columnName = columnForValueField.field;
-            }
-            newColumns.push({
-                type: 'expr',
-                expr: {
-                    type: 'column_ref',
-                    table: columnTable,
-                    column: columnName,
-                },
-                as: columnName,
-            });
+        else {
+            return false;
         }
-        newAst = { ...ast };
-        newAst.columns = newColumns;
-        const newPivot = Object.keys(pivot).length === 0 ? null : pivot;
-        return { pivot: newPivot, ast: newAst };
+    });
+    newColumns = newColumns.filter((col) => columns.find((c) => c.field === col.as));
+    return newColumns;
+}
+function addAllPivotColumnsToColumnAST(astColumns, pivot) {
+    if (!pivot) {
+        return;
     }
-    // Iterate over columns to find aggr_func -> set pivot.aggregationType and pivot.valueField
-    for (let i = 0; i < ast.columns.length; i++) {
-        const column = ast.columns[i];
-        const field = columns.find((c) => c.field === column.expr?.args?.expr?.column ||
-            c.field === column.expr?.args?.expr?.value);
-        if (column?.expr?.type === 'aggr_func') {
-            // CASE WHEN
-            if (column.expr?.args?.expr?.type.toLowerCase() === 'case' &&
-                ['double_quote_string', 'column_ref'].includes(column.expr?.args?.expr?.args[0]?.result?.type)) {
-                // The result of the CASE is the column to use
-                const columnNode = column.expr?.args?.expr?.args[0]?.result;
-                const columnName = columnNode?.value ?? columnNode?.column;
-                if (!isIdColumn(columnName)) {
-                    pivot.valueField = columnName;
-                    pivot.aggregationType = column?.expr?.name?.toLowerCase();
-                    if (pivot.aggregationType === 'avg') {
-                        pivot.aggregationType = 'average';
-                    }
-                }
-                const findByName = (x) => [x.expr?.value, x.expr?.column].includes(columnName);
-                if (!newColumns.find(findByName)) {
-                    newColumns.push({
-                        type: 'expr',
-                        expr: {
-                            type: 'column_ref',
-                            table: null,
-                            column: columnName,
-                        },
-                        as: null,
-                    });
-                }
-                // We need to extract a column name from the condition expression of
-                // the CASE and we'll use that as the columnField in our pivot.
-                const condNode = column.expr?.args?.expr?.args[0]?.cond;
-                const condColumnName = condNode.left?.column ?? condNode.left?.value;
-                if (!isIdColumn(condColumnName)) {
-                    pivot.columnField = condColumnName;
-                }
-                const findByColName = (x) => [x.expr?.value, x.expr?.column].includes(condColumnName);
-                if (!newColumns.find(findByColName)) {
-                    // Make sure the column we extracted is in our column list.
-                    newColumns.push({
-                        type: 'expr',
-                        expr: {
-                            type: 'column_ref',
-                            table: null,
-                            column: condColumnName,
-                        },
-                        as: null,
-                    });
-                }
-                // REGULAR
-            }
-            else if (
-            // sum("amount")
-            column.expr?.args?.expr?.type === 'double_quote_string' ||
-                // sum(amount)
-                column.expr?.args?.expr?.type === 'column_ref' ||
-                // count(*)
-                column.expr?.args?.expr?.type === 'star') {
-                const columnNode = column.expr?.args?.expr;
-                const columnName = columnNode?.value || columnNode?.column;
-                // if count(*), make the value field an actual column
-                if (columnName === '*') {
-                    let columnForValueField = columns.find((c) => {
-                        return isNumericColumnType(c.fieldType) && !isIdColumn(c.field);
-                    });
-                    // if there aren't any numeric columns use the first column
-                    if (!column) {
-                        columnForValueField = columns[0];
-                    }
-                    pivot.valueField = columnForValueField?.field;
-                    // pivot.valueFieldType = 'number';
-                    pivot.aggregationType = column?.expr?.name.toLowerCase();
-                    if (pivot.aggregationType === 'avg') {
-                        pivot.aggregationType = 'average';
-                    }
-                    const findByName = (x) => [x.expr?.value, x.expr?.column].includes(columnForValueField.field);
-                    if (!newColumns.find(findByName)) {
-                        newColumns.push({
-                            type: 'expr',
-                            expr: {
-                                type: 'column_ref',
-                                table: null,
-                                column: columnForValueField?.field || '',
-                            },
-                            as: null,
-                        });
-                    }
-                }
-                else {
-                    // ONLY SET VALUE FIELD IF THE FIELD TYPE IS A NUMBER
-                    if (isNumericColumnType(field?.fieldType) &&
-                        !isIdColumn(columnName)) {
-                        pivot.valueField = columnName;
-                        pivot.aggregationType = column?.expr?.name.toLowerCase();
-                        if (pivot.aggregationType === 'avg') {
-                            pivot.aggregationType = 'average';
-                        }
-                    }
-                    const findByName = (x) => [x.expr?.value, x.expr?.column].includes(columnName);
-                    if (!newColumns.find(findByName)) {
-                        newColumns.push({
-                            type: 'expr',
-                            expr: {
-                                type: 'column_ref',
-                                table: null,
-                                column: columnName,
-                            },
-                            as: null,
-                        });
-                    }
-                }
-            }
-            else {
-                // normal column?
-                const columnName = column.expr?.column || column.expr?.value;
-                const findByName = (x) => [x.expr?.value, x.expr?.column].includes(columnName);
-                if (!newColumns.find(findByName)) {
-                    newColumns.push(column);
-                }
-            }
+    [pivot.valueField, pivot.rowField, pivot.columnField].forEach((field) => {
+        if (!field || columnExistsInNewColumns(astColumns, field)) {
+            return;
         }
-        else if (column.expr?.type === 'function') {
-            // date_trunc("month", "created_at") as month
-            if (column.type === 'expr' &&
-                column.expr?.type.toLowerCase() === 'function' &&
-                column.expr?.name.toLowerCase() === 'date_trunc' &&
-                column.expr?.args?.type === 'expr_list' &&
-                column.as) {
-                const columnObj = column.expr?.args?.value[1];
-                const periodObj = column.expr?.args?.value[0];
-                aliasMap[column.as] = columnObj?.value ?? columnObj.column;
-                newColumns.push({
-                    type: 'expr',
-                    expr: {
-                        type: 'function',
-                        name: 'DATE_TRUNC',
-                        args: {
-                            type: 'expr_list',
-                            value: [
-                                { type: 'single_quote_string', value: periodObj?.value },
-                                {
-                                    type: 'column_ref',
-                                    table: null,
-                                    column: columnObj?.value ?? columnObj.column,
-                                },
-                            ],
-                        },
-                    },
-                    as: columnObj?.value ?? columnObj.column,
-                });
-            }
-            else if (column.type === 'expr' &&
-                column.expr?.type.toLowerCase() === 'function' &&
-                column.expr?.name.toLowerCase() === 'date' &&
-                column.expr?.args?.type === 'expr_list' &&
-                column.as) {
-                const columnObj = column.expr?.args?.value[0];
-                aliasMap[column.as] = columnObj?.value ?? columnObj.column;
-                newColumns.push({
-                    type: 'expr',
-                    expr: {
-                        type: 'function',
-                        name: 'DATE_TRUNC',
-                        args: {
-                            type: 'expr_list',
-                            value: [
-                                { type: 'single_quote_string', value: 'day' },
-                                {
-                                    type: 'column_ref',
-                                    table: null,
-                                    column: columnObj?.value ?? columnObj.column,
-                                },
-                            ],
-                        },
-                    },
-                    as: columnObj?.value ?? columnObj.column,
-                });
-            }
-            else {
-                newColumns.push(column);
-            }
-        }
-        else if (column.type === 'expr' &&
-            column.expr?.type.toLowerCase() === 'extract' &&
-            column.as) {
-            const cast_type = column.expr?.args.cast_type;
-            const source = column.expr?.args.source;
-            const field = column.expr?.args.field;
-            aliasMap[column.as] = source?.value ?? source.column;
-            newColumns.push({
-                type: 'expr',
-                expr: {
-                    type: 'extract',
-                    args: { field, cast_type, source },
-                },
-                as: source?.value ?? source.column,
-            });
-        }
-        else {
-            const columnNode = column.expr;
-            newColumns.push({
-                type: 'expr',
-                expr: {
-                    type: 'column_ref',
-                    table: columnNode.table,
-                    column: columnNode.args?.length
-                        ? columnNode.args?.value.find((elem) => ['double_quote_string', 'column_ref'].includes(elem.type))?.value
-                        : columnNode.column ?? columnNode?.value,
-                },
-                as: null,
-            });
+        if (field && !astColumns.find((col) => col.as === field)) {
+            astColumns.push(generateColumnExpr(field));
         }
+    });
+}
+function convertGroupByToPivotData(ast, columns, aliasMap) {
+    if (!ast.groupby) {
+        return {};
     }
-    // Look for a date_trunc in the GROUP BY and set it to the rowField.
-    // If there are no date_trunc in the GROUP BY, we also need to check
-    // to see if there was a date_trunc column that was given an alias and
-    // that alias is used in the GROUP BY (which is also set to rowField).
+    const pivot = {};
     const isDateTrunc = (item) => ['function', 'date_trunc'].includes(item.type);
     const isExtract = (item) => 'extract' === item.type;
     const isCol = (x) => ['double_quote_string', 'column_ref'].includes(x.type);
-    const isAlias = (x) => isCol(x) && aliasMap[extractColumnish(x)];
     const dateTruncGroup = ast.groupby.find(isDateTrunc);
     const extractGroup = ast.groupby.find(isExtract);
     if (dateTruncGroup) {
@@ -490,15 +213,6 @@ export function convertGroupBy(ast, prevPivot = null, schemaTables) {
         pivot.rowField = columnName;
         pivot.rowFieldType = columns.find((c) => c.field === columnName)?.fieldType;
     }
-    else if (aliasMap) {
-        const match = ast.groupby.find(isAlias);
-        const matchColumnish = extractColumnish(match);
-        if (matchColumnish) {
-            const actualColumnName = aliasMap[matchColumnish];
-            pivot.rowField = actualColumnName;
-            pivot.rowFieldType = columns.find((c) => c.field === actualColumnName)?.fieldType;
-        }
-    }
     // Otherwise, set the first groupby to rowField then set
     // next field to columnField (if there is one)
     for (let j = 0; j < ast.groupby.length; j++) {
@@ -512,6 +226,12 @@ export function convertGroupBy(ast, prevPivot = null, schemaTables) {
             groupValue = aliasMap[groupValue];
         }
         // Try to set the rowField and columnField.
+        const column = columns.find((c) => c.field === groupValue);
+        if (!column ||
+            isNumericColumnType(column.fieldType) ||
+            isIdColumn(column.field)) {
+            continue;
+        }
         if (!pivot.rowField) {
             pivot.rowField = groupValue;
             pivot.rowFieldType = columns.find((c) => c.field === groupValue)?.fieldType;
@@ -520,154 +240,207 @@ export function convertGroupBy(ast, prevPivot = null, schemaTables) {
             pivot.columnField = groupValue;
         }
     }
-    newAst = { ...ast };
-    newAst.groupby = null;
-    newAst.columns = newColumns;
-    const isColumnDate = columns.find((column) => column.field === pivot?.columnField)?.fieldType ===
-        'date';
-    if (pivot && pivot.columnField && isColumnDate) {
-        [pivot.rowField, pivot.columnField] = [pivot.columnField, pivot.rowField];
-        pivot.rowFieldType = 'date';
-    }
-    // Automatically order by the rowField if this is a 1D pivot (ie. no columnField)
-    // we also want to automatically order by rowField if the rowField is date-ish.
-    // TODO: dateish check breaks for plain date-type columns (ie. non date_trunc/extract)
-    const isPivot1D = pivot.rowField && !pivot.columnField;
-    const isRowFieldDateish = isColumnDateish(newAst.columns.find((c) => c.as === pivot?.rowField));
-    const isAutoOrderBy = isPivot1D || isRowFieldDateish;
-    newAst.orderby = isAutoOrderBy
-        ? [
-            {
-                expr: { type: 'column_ref', table: null, column: pivot.rowField },
-                type: newAst.orderby !== null
-                    ? newAst.orderby[0]?.type ?? 'DESC'
-                    : 'DESC',
-            },
-        ]
-        : null;
-    // If the AI didn't generate an aggregate, add one. DO NOT ADD ID FOR VALUE FIELD
-    if (!pivot.valueField) {
-        for (let i = 0; i < newAst.columns.length; i++) {
-            const columnName = newAst.columns[i].expr?.column ||
-                newAst.columns[i].expr?.value ||
-                undefined;
-            if (columnName &&
-                !isIdColumn(columnName) &&
-                isNumericColumnType(columns.find((c) => c.field === columnName)?.fieldType)) {
-                pivot.valueField = columnName;
-                pivot.aggregationType = 'sum';
+    return pivot;
+}
+function convertCaseWhenToPivotData(column) {
+    const pivot = {};
+    const columnNode = column.expr?.args?.expr?.args[0]?.result;
+    const columnName = columnNode?.value ?? columnNode?.column?.expr?.value ?? columnNode?.column;
+    if (columnName && typeof columnName === 'string') {
+        if (!isIdColumn(columnName)) {
+            pivot.valueField = columnName;
+            pivot.aggregationType = column?.expr?.name?.toLowerCase();
+            if (pivot.aggregationType === 'avg') {
+                pivot.aggregationType = 'average';
             }
         }
     }
-    if (pivot.rowFieldType === 'date') {
-        pivot.sort = true;
-        pivot.sortDirection = 'ASC';
-        pivot.sortField = pivot.rowField;
-        pivot.sortFieldType = pivot.rowFieldType;
+    // We need to extract a column name from the condition expression of
+    // the CASE and we'll use that as the columnField in our pivot.
+    const condNode = column.expr?.args?.expr?.args[0]?.cond;
+    const condColumnName = condNode.left?.column?.expr?.value ??
+        condNode.left?.column ??
+        condNode.left?.value;
+    if (!isIdColumn(condColumnName)) {
+        pivot.columnField = condColumnName;
     }
-    // @ts-ignore
-    const title = generatePivotTitle(pivot);
-    return { pivot: { ...pivot, title }, ast: newAst };
+    return pivot;
 }
-/**
- * Applies a pivot to an object with {rows, fields} keys, returning
- * a new object with the aggragation applied to the rows and the
- * corresponding fields updated to reflect the accurate types.
- */
-export function applyPivot(data, pivot, schema) {
-    const newRows = [];
-    const { rows, fields } = data;
-    const { aggregationType, valueField, rowField, columnField } = pivot;
-    if (rows.length === 0)
-        return { rows, fields };
-    if ((rowField || columnField) && !valueField)
-        return { rows, fields };
-    if (columnField && !Object.keys(schema).includes(columnField))
-        return { rows, fields };
-    const AGGREGATORS = {
-        sum: (prev, next, _count = 0) => Number(prev) + Number(next),
-        max: (prev, next, _count = 0) => Math.max(Number(prev), Number(next)),
-        min: (prev, next, _count = 0) => Math.min(Number(prev), Number(next)),
-        count: (_prev, _next, count = 0) => Number(count) + 1,
-        avg: (prev, next, count = 0) => (Number(prev) * count + Number(next)) / (count + 1),
-    };
-    if (!columnField) {
-        // Simple 1D pivot
-        const keys = Object.keys(rows[0]);
-        for (const row of rows) {
-            let newRow = newRows.find((r) => r[rowField] === row[rowField]);
-            if (newRow) {
-                // Aggregate the existing row with this row
-                newRow[valueField] = AGGREGATORS[aggregationType](newRow[valueField], row[valueField], newRow.count ?? 0);
-                newRow.count = (newRow.count ?? 0) + 1;
+export function convertASTToPivotData(ast, columnInfo) {
+    const newPivot = {};
+    const aliasMap = {};
+    if (ast.columns === '*') {
+        return null;
+    }
+    ast.columns.forEach((column) => {
+        const columnName = getColumnName(column);
+        if (!columnName) {
+            return;
+        }
+        const alias = column.as || columnName;
+        aliasMap[alias] = columnName;
+        if (column.expr?.args?.expr?.type.toLowerCase() === 'case' &&
+            ['double_quote_string', 'column_ref'].includes(column.expr?.args?.expr?.args[0]?.result?.type)) {
+            const caseWhenPivot = convertCaseWhenToPivotData(column);
+            newPivot.valueField = caseWhenPivot.valueField;
+            newPivot.aggregationType = caseWhenPivot.aggregationType;
+            newPivot.columnField = caseWhenPivot.columnField;
+        }
+        else if (column.expr.type === 'aggr_func' && !isIdColumn(columnName)) {
+            const aggObj = column.expr;
+            newPivot.aggregationType = aggObj.name.toLowerCase();
+            if (newPivot.aggregationType === 'avg') {
+                newPivot.aggregationType = 'average';
             }
-            else {
-                // Build up a new row and add it to our new rows array.
-                newRow = keys.reduce((obj, key) => {
-                    obj[key] = row[key];
-                    return obj;
-                }, {});
-                newRow.count = 1;
-                newRows.push(newRow);
+            newPivot.valueField = columnName;
+        }
+        else if (column.expr.type === 'function') {
+            const functionInfo = column.expr.name;
+            if (Array.isArray(functionInfo.name) &&
+                functionInfo.name[0].value &&
+                ['timestamp_trunc', 'date_trunc', 'date_part'].includes(functionInfo.name[0].value.toLowerCase())) {
+                newPivot.rowField = columnName;
             }
         }
-        // Remove the count property before returning
-        for (const row of newRows) {
-            delete row.count;
+    });
+    const pivotData = convertGroupByToPivotData(ast, columnInfo, aliasMap);
+    const finalPivot = { ...pivotData, ...newPivot };
+    const isColumnDate = columnInfo.find((column) => column.field === finalPivot?.columnField)
+        ?.fieldType === 'date';
+    if (finalPivot) {
+        // Make sure date fields are set to the rowField
+        const isPivot1D = finalPivot.rowField && !finalPivot.columnField;
+        if (finalPivot.columnField && isColumnDate) {
+            [finalPivot.rowField, finalPivot.columnField] = [
+                finalPivot.columnField,
+                finalPivot.rowField,
+            ];
+            finalPivot.rowFieldType = 'date';
         }
-        return { rows: newRows, fields };
-    }
-    else {
-        // 2D Pivot with rowField and columnField
-        const columnVariants = Object.keys(schema[columnField]);
-        const rowFieldColumnType = fields.find((f) => f.name === rowField);
-        const valueFieldColumnType = fields.find((f) => f.name === valueField);
-        const newFields = [
-            rowFieldColumnType,
-            ...columnVariants.map((name) => {
-                return { ...valueFieldColumnType, name, label: name };
-            }),
-        ];
-        for (const row of rows) {
-            let newRow = newRows.find((r) => r[rowField] === row[rowField]);
-            if (newRow) {
-                // Aggregate the existing row with this row
-                newRow[row[columnField]] = AGGREGATORS[aggregationType](newRow[row[columnField]], row[valueField], newRow[`${row[columnField]}_count`] ?? 0);
-                newRow[`${row[columnField]}_count`] += 1;
+        // Make sure pivot value field is of numeric type
+        if (finalPivot.valueField) {
+            const column = columnInfo.find((c) => c.field === finalPivot.valueField);
+            if (!column || !isNumericColumnType(column.fieldType)) {
+                // @ts-ignore
+                finalPivot.valueField = null;
+                finalPivot.aggregationType = 'count';
             }
-            else {
-                // Build up a new row and add it to our new rows array.
-                const defaultAllColumnVariants = columnVariants.reduce((obj, key) => {
-                    // set each variant to the default value (eg. 0)
-                    obj[key] = aggregationType === 'min' ? Number.MAX_VALUE : 0;
-                    obj[`${key}_count`] = 0;
-                    return obj;
-                }, {});
-                newRow = {
-                    [rowField]: row[rowField],
-                    ...defaultAllColumnVariants,
-                };
-                newRow[row[columnField]] =
-                    aggregationType === 'count' ? 1 : row[valueField];
-                newRow[`${row[columnField]}_count`] = 1;
-                newRows.push(newRow);
+        }
+        if (finalPivot.rowFieldType === 'date') {
+            finalPivot.sort = true;
+            finalPivot.sortDirection = 'ASC';
+            finalPivot.sortField = finalPivot.rowField;
+            finalPivot.sortFieldType = finalPivot.rowFieldType;
+        }
+        else if (isPivot1D) {
+            finalPivot.sort = true;
+            finalPivot.sortDirection = 'ASC';
+            finalPivot.sortField = finalPivot.rowField;
+            finalPivot.sortFieldType = finalPivot.rowFieldType;
+        }
+    }
+    return finalPivot;
+}
+function addValueFieldToPivot(newAst, pivot, columns) {
+    if (!pivot) {
+        return;
+    }
+    if (!pivot.valueField) {
+        let firstNumericNonIdColumn = undefined;
+        newAst.columns.forEach((col) => {
+            if (!isIdColumn(col.as) &&
+                isNumericColumnType(columns.find((c) => c.field === col.as)?.fieldType)) {
+                firstNumericNonIdColumn = { ...col };
             }
+        });
+        if (firstNumericNonIdColumn) {
+            pivot.valueField = firstNumericNonIdColumn.as;
+            pivot.aggregationType = 'sum';
         }
-        // Remove the _count properties before returning
-        for (const row of newRows) {
-            for (const key in row) {
-                if (key.endsWith('_count')) {
-                    // Show a special output for no entries so that we
-                    // don't show our dummy defaults like 0 or Number.MAX_VALUE
-                    if (row[key] === 0) {
-                        row[key.replace('_count', '')] = '-';
-                    }
-                    delete row[key];
-                }
+        else {
+            pivot.aggregationType = 'count';
+        }
+    }
+}
+/**
+ * Processes the AST and builds up a new pivot, given the old AST, the old pivot
+ * and a list of tables in the schema.
+ */
+export function convertGroupBy(ast, prevPivot = null, schemaTables) {
+    if (!isSelect(ast))
+        return { ast, pivot: prevPivot };
+    let newAst = {};
+    let pivot;
+    const newColumns = [];
+    const tableNamesInQuery = (ast.from ?? []).map((tbl) => {
+        // This assumes that all the entries in the from clause are BaseFrom.
+        // TODO: Handle Join, TableExpr, and Dual (maybe not the last one).
+        return tbl.table;
+    });
+    const columns = schemaTables
+        .filter((t) => tableNamesInQuery.includes(t.displayName))
+        .flatMap((table) => table.columns)
+        .map((column) => {
+        return convertColumnInfoToColumnInternal(column);
+    });
+    // If there is not GROUP BY clause in the AST, we still need to process the
+    // AST but it should pass-through in nature.
+    if (!ast.groupby) {
+        // If there was a pivot before, evaluate the pivot.
+        if (prevPivot) {
+            // Make sure that the AST includes all the columns on the prevPivot
+            const columnAlreadyExists = (col) => ast.columns.find((c) => [
+                c.expr?.value,
+                c.expr?.column,
+                c.expr?.column?.expr?.value,
+                c.as,
+            ].includes(col));
+            if (prevPivot.valueField) {
+                const existingCol = columnAlreadyExists(prevPivot.valueField);
+                newColumns.push(existingCol ?? generateColumnExpr(prevPivot.valueField));
+            }
+            if (prevPivot.rowField) {
+                const existingCol = columnAlreadyExists(prevPivot.rowField);
+                newColumns.push(existingCol ?? generateColumnExpr(prevPivot.rowField));
             }
+            if (prevPivot.columnField) {
+                const existingCol = columnAlreadyExists(prevPivot.columnField);
+                newColumns.push(existingCol ?? generateColumnExpr(prevPivot.columnField));
+            }
+            ast.columns = newColumns;
+            return { pivot: prevPivot, ast };
         }
-        return { rows: newRows, fields: newFields };
+        pivot = convertASTToPivotData(ast, columns);
+        if (pivot && Object.keys(pivot).length === 0) {
+            pivot = null;
+        }
+        addAllPivotColumnsToColumnAST(ast.columns, pivot);
+        newAst = { ...ast };
+        newAst.columns = cleanASTColumns(ast, columns);
+        addValueFieldToPivot(newAst, pivot, columns);
+        return { pivot, ast: newAst };
     }
+    pivot = convertASTToPivotData(ast, columns);
+    if (!pivot || pivot == null) {
+        throw new Error('Pivot is null');
+    }
+    addAllPivotColumnsToColumnAST(ast.columns, pivot);
+    newAst = { ...ast };
+    newAst.groupby = null;
+    newAst.columns = cleanASTColumns(ast, columns);
+    // ADD sort if pivot has sort
+    if (pivot.sort) {
+        newAst.orderby = [
+            {
+                expr: { type: 'column_ref', table: null, column: pivot.rowField },
+                type: newAst.orderby !== null ? newAst.orderby[0]?.type ?? 'DESC' : 'DESC',
+            },
+        ];
+    }
+    addValueFieldToPivot(newAst, pivot, columns);
+    // @ts-ignore
+    const title = generatePivotTitle(pivot);
+    return { pivot: { ...pivot, title }, ast: newAst };
 }
 export function convertStringComparison(node, databaseType) {
     // Function to recursively search and replace the pattern in the object
@@ -681,7 +454,7 @@ export function convertStringComparison(node, databaseType) {
         else if (typeof obj === 'object' && obj !== null) {
             // If the current node is an object, check for the pattern
             if (obj.type === 'binary_expr' &&
-                (obj.operator === '=' ||
+                ((obj.operator === '=' && !isValid(parseISO(obj.right.value))) || // don't do it on date objects
                     obj.operator === 'LIKE' ||
                     obj.operator === 'ILIKE') &&
                 obj.left &&
@@ -693,25 +466,49 @@ export function convertStringComparison(node, databaseType) {
                 obj.operator = 'LIKE';
                 obj.left = {
                     type: 'function',
-                    name: 'LOWER',
+                    name: {
+                        name: [
+                            {
+                                type: 'default',
+                                value: 'LOWER',
+                            },
+                        ],
+                    },
                     args: {
                         type: 'expr_list',
                         value: [
                             databaseType === 'BigQuery'
                                 ? {
                                     type: 'column_ref',
-                                    value: obj.left.column || obj.left.value,
+                                    value: obj.left.column?.expr?.value ||
+                                        obj.left.column ||
+                                        obj.left.value,
                                 }
                                 : {
-                                    type: 'double_quote_string',
-                                    value: obj.left.column || obj.left.value,
+                                    type: 'column_ref',
+                                    table: null,
+                                    column: {
+                                        expr: {
+                                            type: 'default',
+                                            value: obj.left.column?.expr?.value ||
+                                                obj.left.column ||
+                                                obj.left.value,
+                                        },
+                                    },
                                 },
                         ],
                     },
                 };
                 obj.right = {
                     type: 'function',
-                    name: 'LOWER',
+                    name: {
+                        name: [
+                            {
+                                type: 'default',
+                                value: 'LOWER',
+                            },
+                        ],
+                    },
                     args: {
                         type: 'expr_list',
                         value: [
@@ -737,25 +534,43 @@ export function convertStringComparison(node, databaseType) {
                 obj.operator = 'NOT LIKE';
                 obj.left = {
                     type: 'function',
-                    name: 'LOWER',
+                    name: {
+                        name: [
+                            {
+                                type: 'default',
+                                value: 'LOWER',
+                            },
+                        ],
+                    },
                     args: {
                         type: 'expr_list',
                         value: [
                             databaseType === 'BigQuery'
                                 ? {
                                     type: 'column_ref',
-                                    value: obj.left.column || obj.left.value,
+                                    value: obj.left.column?.expr?.value ||
+                                        obj.left.column ||
+                                        obj.left.value,
                                 }
                                 : {
                                     type: 'double_quote_string',
-                                    value: obj.left.column || obj.left.value,
+                                    value: obj.left.column?.expr?.value ||
+                                        obj.left.column ||
+                                        obj.left.value,
                                 },
                         ],
                     },
                 };
                 obj.right = {
                     type: 'function',
-                    name: 'LOWER',
+                    name: {
+                        name: [
+                            {
+                                type: 'default',
+                                value: 'LOWER',
+                            },
+                        ],
+                    },
                     args: {
                         type: 'expr_list',
                         value: [
@@ -779,18 +594,28 @@ export function convertStringComparison(node, databaseType) {
                 obj.right.value[0].type === 'single_quote_string') {
                 obj.left = {
                     type: 'function',
-                    name: 'LOWER',
+                    name: {
+                        name: [],
+                        schema: {
+                            type: 'default',
+                            value: 'LOWER',
+                        },
+                    },
                     args: {
                         type: 'expr_list',
                         value: [
                             databaseType === 'BigQuery'
                                 ? {
                                     type: 'column_ref',
-                                    value: obj.left.column || obj.left.value,
+                                    value: obj.left.column?.expr?.value ||
+                                        obj.left.column ||
+                                        obj.left.value,
                                 }
                                 : {
                                     type: 'double_quote_string',
-                                    value: obj.left.column || obj.left.value,
+                                    value: obj.left.column?.expr?.value ||
+                                        obj.left.column ||
+                                        obj.left.value,
                                 },
                         ],
                     },
@@ -800,7 +625,13 @@ export function convertStringComparison(node, databaseType) {
                     // convert NOT IN ('fuel', 'food') to NOT IN (LOWER('fuel'), LOWER('food'))
                     value: obj.right.value.map((elem) => ({
                         type: 'function',
-                        name: 'LOWER',
+                        name: {
+                            name: [],
+                            schema: {
+                                type: 'default',
+                                value: 'LOWER',
+                            },
+                        },
                         args: {
                             type: 'expr_list',
                             value: [{ type: 'single_quote_string', value: elem.value }],
@@ -833,3 +664,54 @@ export function convertRemoveSimpleParentheses(ast) {
     }
     return { ...ast, where: node };
 }
+/**
+ * Processes an ast and converts all unary_expr nodes or direct column_refs
+ * into binary_expr form
+ */
+export function convertUnaryToBinary(ast) {
+    const newAst = { ...ast };
+    if (!newAst.where) {
+        return ast;
+    }
+    function searchAndReplace(node) {
+        if (!node) {
+            return { ...node };
+        }
+        else if (node.type === 'binary_expr') {
+            if (node.operator.toLowerCase() === 'and' ||
+                node.operator.toLowerCase() === 'or') {
+                return {
+                    ...node,
+                    left: searchAndReplace(node.left),
+                    right: searchAndReplace(node.right),
+                };
+            }
+            else {
+                return { ...node };
+            }
+        }
+        else if (node.type === 'unary_expr' && node.operator === 'NOT') {
+            // yet to run into any other operator from AI
+            return {
+                type: 'binary_expr',
+                operator: '<>',
+                left: { ...node.expr },
+                right: { type: 'bool', value: true },
+            };
+        }
+        else if (node.type === 'column_ref') {
+            // e.g. WHERE field (boolean)
+            return {
+                type: 'binary_expr',
+                operator: '=',
+                left: { ...node },
+                right: { type: 'bool', value: true },
+            };
+        }
+        else {
+            return { ...node };
+        }
+    }
+    const newWhere = searchAndReplace(newAst.where);
+    return { ...newAst, where: newWhere };
+}