@malloydata/malloy 0.0.123-dev240209030627 → 0.0.123-dev240209210150

package/dist/lang/ast/expressions/expr-func.js

@@ -226,11 +226,12 @@ class ExprFunc extends expression_def_1.ExpressionDef {
                     if (e.found === undefined) {
                         partitionField.log(`${partitionField.refString} is not defined`);
                     }
-                    else if ((0, malloy_types_1.expressionIsScalar)(e.found.typeDesc().expressionType)) {
-                        partitionByFields.push(partitionField.nameString);
+                    else if ((0, malloy_types_1.expressionIsAnalytic)(e.found.typeDesc().expressionType) ||
+                        (0, malloy_types_1.expressionIsUngroupedAggregate)(e.found.typeDesc().expressionType)) {
+                        partitionField.log('Partition expression must be scalar or aggregate');
                     }
                     else {
-                        partitionField.log('Partition expression must be scalar');
+                        partitionByFields.push(partitionField.nameString);
                     }
                 }
             }

package/dist/lang/test/expressions.spec.js

@@ -215,6 +215,25 @@ describe('expressions', () => {
           calculate: x is lag(ai) { partition_by: ai; order_by: ai }
         }
       `).toTranslate();
+        });
+        test('partition by works with scalar and aggregate', () => {
+            expect((0, test_translator_1.markSource) `
+        run: a -> {
+          group_by: ai
+          aggregate: c is count()
+          calculate: x is lag(ai) { partition_by: ai, c }
+        }
+      `).toTranslate();
+        });
+        test('partition by fails with analytic and ungrouped aggregate', () => {
+            expect((0, test_translator_1.markSource) `
+        run: a -> {
+          group_by: ai
+          aggregate: ac is all(count())
+          calculate: x is lag(ai) { partition_by: ac }
+          calculate: y is lag(ai) { partition_by: x }
+        }
+      `).translationToFailWith('Partition expression must be scalar or aggregate', 'Partition expression must be scalar or aggregate');
         });
         test('analytics order_by requires expression', () => {
             expect((0, test_translator_1.markSource) `

package/dist/malloy.d.ts

@@ -1296,13 +1296,33 @@ export declare abstract class DataWriter {
 export declare class JSONWriter extends DataWriter {
     process(data: AsyncIterableIterator<DataRecord>): Promise<void>;
 }
+/**
+ * CSV writer class that handles nested data.
+ * This writer creates CSV using a DFS traversal of the result dataset.
+ * Each trivial column value is converted to a CSV of 1x1 matrix and all the
+ * columns are merged together to create a CSV that represents 1 QueryDataRow.
+ * Since this follows DFS, each non trivial data is rendered into a NxM matrix
+ * where N is the number of rows in the nested data and M is the number of
+ * columns it has.
+ * For any row with X number of columns, we end up with X number of NxM matrices
+ * where the value of N,M pair may be different for each column.
+ * We then merge the matrices so that we end up with a larger matrix of size
+ * Max(N)xSum(M) by taking one row of csv from each matric at a time. For any
+ * matrix with N<Max(N), we add a row of empty CSV cells of size N.
+ */
 export declare class CSVWriter extends DataWriter {
     private readonly columnSeparator;
     private readonly rowSeparator;
     private readonly quoteCharacter;
     private readonly includeHeader;
+    private readonly emptyCell;
     private escape;
     private stringify;
+    private getColWeight;
+    private getHeaderRow;
+    private mergeMatrices;
+    private getChildMatrix;
+    private getRowMatrix;
     process(data: AsyncIterableIterator<DataRecord>): Promise<void>;
 }
 export {};

package/dist/malloy.js

@@ -2831,6 +2831,20 @@ class JSONWriter extends DataWriter {
     }
 }
 exports.JSONWriter = JSONWriter;
+/**
+ * CSV writer class that handles nested data.
+ * This writer creates CSV using a DFS traversal of the result dataset.
+ * Each trivial column value is converted to a CSV of 1x1 matrix and all the
+ * columns are merged together to create a CSV that represents 1 QueryDataRow.
+ * Since this follows DFS, each non trivial data is rendered into a NxM matrix
+ * where N is the number of rows in the nested data and M is the number of
+ * columns it has.
+ * For any row with X number of columns, we end up with X number of NxM matrices
+ * where the value of N,M pair may be different for each column.
+ * We then merge the matrices so that we end up with a larger matrix of size
+ * Max(N)xSum(M) by taking one row of csv from each matric at a time. For any
+ * matrix with N<Max(N), we add a row of empty CSV cells of size N.
+ */
 class CSVWriter extends DataWriter {
     constructor() {
         super(...arguments);
@@ -2838,6 +2852,7 @@ class CSVWriter extends DataWriter {
         this.rowSeparator = '\n';
         this.quoteCharacter = '"';
         this.includeHeader = true;
+        this.emptyCell = '';
     }
     escape(value) {
         const hasInnerQuote = value.includes(this.quoteCharacter);
@@ -2852,50 +2867,145 @@ class CSVWriter extends DataWriter {
         }
         return value;
     }
-    stringify(cell) {
-        if (cell.isNull()) {
-            return '';
-        }
-        else if (cell.isArray() ||
-            cell.isRecord() ||
-            cell.isBoolean() ||
-            cell.isNumber()) {
-            return JSON.stringify(cell.value);
+    // Re-using the old stringify method for sanity.
+    stringify(value) {
+        if (value === null) {
+            return this.emptyCell;
         }
-        else if (cell.isDate() || cell.isTimestamp()) {
-            return cell.value.toISOString();
+        else if (value instanceof Date) {
+            return value.toISOString();
         }
-        else if (cell.isString()) {
-            return cell.value;
+        else if (typeof value === 'boolean' || typeof value === 'number') {
+            return JSON.stringify(value);
         }
         else {
-            return `${cell.value}`;
+            return `${value}`;
+        }
+    }
+    // Extra weight to be added becase of nested tables inside the cells.
+    getColWeight(jsonVal) {
+        let firstVal = jsonVal;
+        if (Array.isArray(jsonVal)) {
+            firstVal = jsonVal[0];
+        }
+        let numKeys = 0;
+        for (const key in firstVal) {
+            numKeys = numKeys + 1;
+            const val = firstVal[key];
+            if (Array.isArray(val)) {
+                const weight = this.getColWeight(val) - 1;
+                numKeys = numKeys + weight;
+            }
+        }
+        return numKeys;
+    }
+    // Get header row along with extra empty spaces for nested children.
+    getHeaderRow(row) {
+        const csv = [];
+        let width = 0;
+        for (const key in row) {
+            csv.push(this.escape(key));
+            const val = row[key];
+            width++;
+            if (Array.isArray(val)) {
+                const numKeys = this.getColWeight(val) - 1;
+                width = width + numKeys;
+                for (let i = 0; i < numKeys; i++) {
+                    csv.push(this.emptyCell);
+                }
+            }
+        }
+        return { rows: [csv.join(this.columnSeparator)], length: 1, width: width };
+    }
+    // Merge the child matrices i.e. merge the columns into one bigger matrix i.e. CSV.
+    mergeMatrices(matrices) {
+        const maxLength = Math.max(...matrices.map(matrix => matrix.length));
+        const matrixWidth = matrices.reduce((sum, matrix) => sum + matrix.width, 0);
+        const csvMatrix = [];
+        for (let i = 0; i < maxLength; i++) {
+            const csvRow = [];
+            for (const matrix of matrices) {
+                if (i < matrix.length) {
+                    csvRow.push(matrix.rows[i]);
+                }
+                else {
+                    // Add empty cells.
+                    const emptyCells = Array(matrix.width).fill(this.emptyCell);
+                    csvRow.push(...emptyCells);
+                }
+            }
+            csvMatrix.push(csvRow.join(this.columnSeparator));
         }
+        return {
+            rows: csvMatrix,
+            length: maxLength,
+            width: matrixWidth,
+        };
+    }
+    // Gets CSV for a data cell that has nested data.
+    getChildMatrix(jsonVal) {
+        // This is not expected to happen.
+        if (!Array.isArray(jsonVal)) {
+            return {
+                rows: ['Invalid data found, value is not an array'],
+                length: 1,
+                width: 1,
+            };
+        }
+        else if (jsonVal.length === 0) {
+            return {
+                rows: [''],
+                length: 1,
+                width: 1,
+            };
+        }
+        const csvMatrix = [];
+        const header = this.getHeaderRow(jsonVal[0]);
+        // Header has 1 row.
+        csvMatrix.push(...header.rows);
+        const width = header.width;
+        let rowCount = 1;
+        for (const row of jsonVal) {
+            const rowMatrix = this.getRowMatrix(row);
+            rowCount = rowCount + rowMatrix.length;
+            csvMatrix.push(...rowMatrix.rows);
+        }
+        return { rows: csvMatrix, length: rowCount, width: width };
+    }
+    // Creates CSV content for one row of data.
+    getRowMatrix(row) {
+        const matrices = [];
+        for (const key in row) {
+            const val = row[key];
+            if (!Array.isArray(val)) {
+                const cell = {
+                    rows: [this.stringify(val)],
+                    length: 1,
+                    width: 1,
+                };
+                matrices.push(cell);
+            }
+            else {
+                const cell = this.getChildMatrix(val);
+                matrices.push(cell);
+            }
+        }
+        return this.mergeMatrices(matrices);
     }
     async process(data) {
-        let fields;
+        let headerDefined = false;
         for await (const row of data) {
-            if (fields === undefined) {
-                fields = row.field.allFields;
-                if (this.includeHeader) {
-                    for (let fieldIndex = 0; fieldIndex < fields.length; fieldIndex++) {
-                        const field = fields[fieldIndex];
-                        this.stream.write(this.escape(field.name));
-                        if (fieldIndex !== fields.length - 1) {
-                            this.stream.write(this.columnSeparator);
-                        }
-                    }
-                    this.stream.write(this.rowSeparator);
-                }
+            if (!headerDefined && this.includeHeader) {
+                const header = this.getHeaderRow(row.toObject());
+                this.stream.write(header.rows[0]);
+                this.stream.write(this.rowSeparator);
+                headerDefined = true;
             }
-            for (let fieldIndex = 0; fieldIndex < fields.length; fieldIndex++) {
-                const field = fields[fieldIndex];
-                this.stream.write(this.escape(this.stringify(row.cell(field))));
-                if (fieldIndex !== fields.length - 1) {
-                    this.stream.write(this.columnSeparator);
-                }
+            const rowCsv = this.getRowMatrix(row.toObject());
+            for (const line of rowCsv.rows) {
+                this.stream.write(line);
+                this.stream.write(this.rowSeparator);
             }
-            this.stream.write(this.rowSeparator);
         }
         this.stream.close();
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.123-dev240209030627",
+  "version": "0.0.123-dev240209210150",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",