bare-script 2.3.2 → 3.0.0

package/lib/data.js

@@ -3,8 +3,8 @@
 
 /** @module lib/data */
 
+import {valueBoolean, valueCompare, valueJSON, valueParseDatetime, valueParseNumber} from './value.js';
 import {evaluateExpression} from './runtime.js';
-import {jsonStringifySortKeys} from 'schema-markdown/lib/encode.js';
 import {parseExpression} from './parser.js';
 import {parseSchemaMarkdown} from 'schema-markdown/lib/parser.js';
 import {validateType} from 'schema-markdown/lib/schema.js';
@@ -31,7 +31,7 @@ export function parseCSV(text) {
     const rows = lines.filter((line) => !line.match(rCSVBlankLine)).map((line) => {
         const row = [];
         let linePart = line;
-        while (linePart !== '') {
+        while (linePart !== null) {
             // Quoted field?
             const mQuoted = linePart.match(rCSVQuotedField) ?? linePart.match(rCSVQuotedFieldEnd);
             if (mQuoted !== null) {
@@ -43,7 +43,7 @@ export function parseCSV(text) {
             // Non-quoted field
             const ixComma = linePart.indexOf(',');
             row.push(ixComma !== -1 ? linePart.slice(0, ixComma) : linePart);
-            linePart = (ixComma !== -1 ? linePart.slice(ixComma + 1) : '');
+            linePart = (ixComma !== -1 ? linePart.slice(ixComma + 1) : null);
         }
         return row;
     });
@@ -51,11 +51,11 @@ export function parseCSV(text) {
     // Assemble the data rows
     const result = [];
     if (rows.length >= 2) {
-        const [fields] = rows;
+        const fields = rows[0].map((field) => field.trim());
         for (let ixLine = 1; ixLine < rows.length; ixLine += 1) {
             const row = rows[ixLine];
             result.push(Object.fromEntries(fields.map(
-                (field, ixField) => [field, ixField < row.length ? row[ixField] : 'null']
+                (field, ixField) => [field, ixField < row.length ? row[ixField] : null]
             )));
         }
     }
@@ -83,15 +83,28 @@ export function validateData(data, csv = false) {
     const types = {};
     for (const row of data) {
         for (const [field, value] of Object.entries(row)) {
-            if (!(field in types)) {
-                if (typeof value === 'number') {
+            if ((types[field] ?? null) === null) {
+                if (typeof value === 'boolean') {
+                    types[field] = 'boolean';
+                } if (typeof value === 'number') {
                     types[field] = 'number';
                 } else if (value instanceof Date) {
                     types[field] = 'datetime';
-                } else if (typeof value === 'string' && (!csv || value !== 'null')) {
-                    if (parseDatetime(value) !== null) {
+                } else if (typeof value === 'string') {
+                    // If we aren't parsing CSV strings, its just a string
+                    if (!csv) {
+                        types[field] = 'string';
+
+                    // If its the null string we can't determine the type yet
+                    } else if (value === '' || value === 'null') {
+                        types[field] = null;
+
+                    // Can the string be parsed into another type?
+                    } else if (valueParseDatetime(value) !== null) {
                         types[field] = 'datetime';
-                    } else if (csv && parseNumber(value) !== null) {
+                    } else if (value === 'true' || value === 'false') {
+                        types[field] = 'boolean';
+                    } else if (valueParseNumber(value) !== null) {
                         types[field] = 'number';
                     } else {
                         types[field] = 'string';
@@ -101,13 +114,25 @@ export function validateData(data, csv = false) {
         }
     }
 
-    // Validate field values
+    // Set the type for fields with undetermined type
+    for (const [field, fieldType] of Object.entries(types)) {
+        if (fieldType === null) {
+            types[field] = 'string';
+        }
+    }
+
+    // Helper to format and raise validation errors
     const throwFieldError = (field, fieldType, fieldValue) => {
-        throw new Error(`Invalid "${field}" field value ${JSON.stringify(fieldValue)}, expected type ${fieldType}`);
+        throw new Error(`Invalid "${field}" field value ${valueJSON(fieldValue)}, expected type ${fieldType}`);
     };
+
+    // Validate field values
     for (const row of data) {
         for (const [field, value] of Object.entries(row)) {
-            const fieldType = types[field];
+            const fieldType = types[field] ?? null;
+            if (fieldType === null) {
+                continue;
+            }
 
             // Null string?
             if (csv && value === 'null') {
@@ -116,9 +141,14 @@ export function validateData(data, csv = false) {
             // Number field
             } else if (fieldType === 'number') {
                 if (csv && typeof value === 'string') {
-                    const numberValue = parseNumber(value);
-                    if (numberValue === null) {
-                        throwFieldError(field, fieldType, value);
+                    let numberValue;
+                    if (value === '') {
+                        numberValue = null;
+                    } else {
+                        numberValue = valueParseNumber(value);
+                        if (numberValue === null) {
+                            throwFieldError(field, fieldType, value);
+                        }
                     }
                     row[field] = numberValue;
                 } else if (value !== null && typeof value !== 'number') {
@@ -127,16 +157,38 @@ export function validateData(data, csv = false) {
 
             // Datetime field
             } else if (fieldType === 'datetime') {
-                if (typeof value === 'string') {
-                    const datetimeValue = parseDatetime(value);
-                    if (datetimeValue === null) {
-                        throwFieldError(field, fieldType, value);
+                if (csv && typeof value === 'string') {
+                    let datetimeValue;
+                    if (value === '') {
+                        datetimeValue = null;
+                    } else {
+                        datetimeValue = valueParseDatetime(value);
+                        if (datetimeValue === null) {
+                            throwFieldError(field, fieldType, value);
+                        }
                     }
                     row[field] = datetimeValue;
                 } else if (value !== null && !(value instanceof Date)) {
                     throwFieldError(field, fieldType, value);
                 }
 
+            // Boolean field
+            } else if (fieldType === 'boolean') {
+                if (csv && typeof value === 'string') {
+                    let booleanValue;
+                    if (value === '') {
+                        booleanValue = null;
+                    } else {
+                        booleanValue = (value === 'true' ? true : (value === 'false' ? false : null));
+                        if (booleanValue === null) {
+                            throwFieldError(field, fieldType, value);
+                        }
+                    }
+                    row[field] = booleanValue;
+                } else if (value !== null && typeof value !== 'boolean') {
+                    throwFieldError(field, fieldType, value);
+                }
+
             // String field
             } else {
                 if (value !== null && typeof value !== 'string') {
@@ -150,47 +202,23 @@ export function validateData(data, csv = false) {
 }
 
 
-function parseNumber(text) {
-    const value = Number.parseFloat(text);
-    if (Number.isNaN(value) || !Number.isFinite(value)) {
-        return null;
-    }
-    return value;
-}
-
-
-export function parseDatetime(text) {
-    const mDate = text.match(rDate);
-    if (mDate !== null) {
-        const year = Number.parseInt(mDate.groups.year, 10);
-        const month = Number.parseInt(mDate.groups.month, 10);
-        const day = Number.parseInt(mDate.groups.day, 10);
-        return new Date(year, month - 1, day);
-    } else if (rDatetime.test(text)) {
-        return new Date(text);
-    }
-    return null;
-}
-
-const rDate = /^(?<year>\d{4})-(?<month>\d{2})-(?<day>\d{2})$/;
-const rDatetime = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?(?:Z|[+-]\d{2}:\d{2})$/;
-
-
 /**
  * Join two data arrays
  *
  * @param {Object} leftData - The left data array
  * @param {Object} rightData - The left data array
- * @param {string} joinExpr - The join [expression]{@link https://craigahobbs.github.io/bare-script/language/#expressions}
- * @param {?string} [rightExpr = null] - The right join [expression]{@link https://craigahobbs.github.io/bare-script/language/#expressions}
+ * @param {string} joinExpr - The join [expression](./language/#expressions)
+ * @param {?string} [rightExpr = null] - The right join [expression](./language/#expressions)
  * @param {boolean} [isLeftJoin = false] - If true, perform a left join (always include left row)
  * @param {?Object} [variables = null] - Additional variables for expression evaluation
- * @param {?Object} [options = null] - The [script execution options]{@link module:lib/runtime~ExecuteScriptOptions}
+ * @param {?Object} [options = null] - The [script execution options]{@link module:lib/options~ExecuteScriptOptions}
  * @returns {Object[]} The joined data array
  */
 export function joinData(leftData, rightData, joinExpr, rightExpr = null, isLeftJoin = false, variables = null, options = null) {
     // Compute the map of row field name to joined row field name
     const leftNames = {};
+    const rightNamesRaw = {};
+    const rightNames = {};
     for (const row of leftData) {
         for (const fieldName of Object.keys(row)) {
             if (!(fieldName in leftNames)) {
@@ -198,24 +226,26 @@ export function joinData(leftData, rightData, joinExpr, rightExpr = null, isLeft
             }
         }
     }
-    const rightNames = {};
     for (const row of rightData) {
         for (const fieldName of Object.keys(row)) {
             if (!(fieldName in rightNames)) {
-                if (!(fieldName in leftNames)) {
-                    rightNames[fieldName] = fieldName;
-                } else {
-                    let uniqueName = fieldName;
-                    let ixUnique = 2;
-                    do {
-                        uniqueName = `${fieldName}${ixUnique}`;
-                        ixUnique += 1;
-                    } while (uniqueName in leftNames || uniqueName in rightNames);
-                    rightNames[fieldName] = uniqueName;
-                }
+                rightNamesRaw[fieldName] = fieldName;
             }
         }
     }
+    for (const fieldName of Object.keys(rightNamesRaw)) {
+        if (!(fieldName in leftNames)) {
+            rightNames[fieldName] = fieldName;
+        } else {
+            let ixUnique = 2;
+            let uniqueName = `${fieldName}${ixUnique}`;
+            while (uniqueName in leftNames || uniqueName in rightNames || uniqueName in rightNamesRaw) {
+                ixUnique += 1;
+                uniqueName = `${fieldName}${ixUnique}`;
+            }
+            rightNames[fieldName] = uniqueName;
+        }
+    }
 
     // Create the evaluation options object
     let evalOptions = options;
@@ -235,7 +265,7 @@ export function joinData(leftData, rightData, joinExpr, rightExpr = null, isLeft
     // Bucket the right rows by the right expression value
     const rightCategoryRows = {};
     for (const rightRow of rightData) {
-        const categoryKey = jsonStringifySortKeys(evaluateExpression(rightExpression, evalOptions, rightRow));
+        const categoryKey = valueJSON(evaluateExpression(rightExpression, evalOptions, rightRow));
         if (!(categoryKey in rightCategoryRows)) {
             rightCategoryRows[categoryKey] = [];
         }
@@ -245,7 +275,7 @@ export function joinData(leftData, rightData, joinExpr, rightExpr = null, isLeft
     // Join the left with the right
     const data = [];
     for (const leftRow of leftData) {
-        const categoryKey = jsonStringifySortKeys(evaluateExpression(leftExpression, evalOptions, leftRow));
+        const categoryKey = valueJSON(evaluateExpression(leftExpression, evalOptions, leftRow));
         if (categoryKey in rightCategoryRows) {
             for (const rightRow of rightCategoryRows[categoryKey]) {
                 const joinRow = {...leftRow};
@@ -270,7 +300,7 @@ export function joinData(leftData, rightData, joinExpr, rightExpr = null, isLeft
  * @param {string} fieldName - The calculated field name
  * @param {string} expr - The calculated field expression
  * @param {?Object} [variables = null] -  Additional variables for expression evaluation
- * @param {?Object} [options = null] - The [script execution options]{@link module:lib/runtime~ExecuteScriptOptions}
+ * @param {?Object} [options = null] - The [script execution options]{@link module:lib/options~ExecuteScriptOptions}
  * @returns {Object[]} The updated data array
  */
 export function addCalculatedField(data, fieldName, expr, variables = null, options = null) {
@@ -292,6 +322,7 @@ export function addCalculatedField(data, fieldName, expr, variables = null, opti
     for (const row of data) {
         row[fieldName] = evaluateExpression(calcExpr, evalOptions, row);
     }
+
     return data;
 }
 
@@ -300,9 +331,9 @@ export function addCalculatedField(data, fieldName, expr, variables = null, opti
  * Filter data rows
  *
  * @param {Object[]} data - The data array
- * @param {string} expr - The boolean filter [expression]{@link https://craigahobbs.github.io/bare-script/language/#expressions}
+ * @param {string} expr - The boolean filter [expression](./language/#expressions)
  * @param {?Object} [variables = null] -  Additional variables for expression evaluation
- * @param {?Object} [options = null] - The [script execution options]{@link module:lib/runtime~ExecuteScriptOptions}
+ * @param {?Object} [options = null] - The [script execution options]{@link module:lib/options~ExecuteScriptOptions}
  * @returns {Object[]} The filtered data array
  */
 export function filterData(data, expr, variables = null, options = null) {
@@ -324,7 +355,7 @@ export function filterData(data, expr, variables = null, options = null) {
 
     // Filter the data
     for (const row of data) {
-        if (evaluateExpression(filterExpr, evalOptions, row)) {
+        if (valueBoolean(evaluateExpression(filterExpr, evalOptions, row))) {
             result.push(row);
         }
     }
@@ -333,80 +364,16 @@ export function filterData(data, expr, variables = null, options = null) {
 }
 
 
-// The aggregation model
-export const aggregationTypes = parseSchemaMarkdown(`\
-group "Aggregation"
-
-
-# A data aggregation specification
-struct Aggregation
-
-    # The aggregation category fields
-    optional string[len > 0] categories
-
-    # The aggregation measures
-    AggregationMeasure[len > 0] measures
-
-
-# An aggregation measure specification
-struct AggregationMeasure
-
-    # The aggregation measure field
-    string field
-
-    # The aggregation function
-    AggregationFunction function
-
-    # The aggregated-measure field name
-    optional string name
-
-
-# An aggregation function
-enum AggregationFunction
-
-    # The average of the measure's values
-    average
-
-    # The count of the measure's values
-    count
-
-    # The greatest of the measure's values
-    max
-
-    # The least of the measure's values
-    min
-
-    # The standard deviation of the measure's values
-    stddev
-
-    # The sum of the measure's values
-    sum
-`);
-
-
-/**
- * Validate an aggregation model
- *
- * @param {Object} aggregation - The
- *     [aggregation model]{@link https://craigahobbs.github.io/bare-script/library/model.html#var.vName='Aggregation'}
- * @returns {Object} The validated
- *     [aggregation model]{@link https://craigahobbs.github.io/bare-script/library/model.html#var.vName='Aggregation'}
- * @throws [ValidationError]{@link https://craigahobbs.github.io/schema-markdown-js/module-lib_schema.ValidationError.html}
- */
-export function validateAggregation(aggregation) {
-    return validateType(aggregationTypes, 'Aggregation', aggregation);
-}
-
-
 /**
  * Aggregate data rows
  *
  * @param {Object[]} data - The data array
- * @param {Object} aggregation - The
- *     [aggregation model]{@link https://craigahobbs.github.io/bare-script/library/model.html#var.vName='Aggregation'}
+ * @param {Object} aggregation - The [aggregation model](./library/model.html#var.vName='Aggregation')
  * @returns {Object[]} The aggregated data array
  */
 export function aggregateData(data, aggregation) {
+    // Validate the aggregation model
+    validateType(aggregationTypes, 'Aggregation', aggregation);
     const categories = aggregation.categories ?? null;
 
     // Create the aggregate rows
@@ -417,7 +384,7 @@ export function aggregateData(data, aggregation) {
 
         // Get or create the aggregate row
         let aggregateRow;
-        const rowKey = (categoryValues !== null ? jsonStringifySortKeys(categoryValues) : '');
+        const rowKey = (categoryValues !== null ? valueJSON(categoryValues) : '');
         if (rowKey in categoryRows) {
             aggregateRow = categoryRows[rowKey];
         } else {
@@ -474,6 +441,57 @@ export function aggregateData(data, aggregation) {
 }
 
 
+// The aggregation model
+export const aggregationTypes = parseSchemaMarkdown(`\
+group "Aggregation"
+
+
+# A data aggregation specification
+struct Aggregation
+
+    # The aggregation category fields
+    optional string[len > 0] categories
+
+    # The aggregation measures
+    AggregationMeasure[len > 0] measures
+
+
+# An aggregation measure specification
+struct AggregationMeasure
+
+    # The aggregation measure field
+    string field
+
+    # The aggregation function
+    AggregationFunction function
+
+    # The aggregated-measure field name
+    optional string name
+
+
+# An aggregation function
+enum AggregationFunction
+
+    # The average of the measure's values
+    average
+
+    # The count of the measure's values
+    count
+
+    # The greatest of the measure's values
+    max
+
+    # The least of the measure's values
+    min
+
+    # The standard deviation of the measure's values
+    stddev
+
+    # The sum of the measure's values
+    sum
+`);
+
+
 /**
  * Sort data rows
  *
@@ -489,7 +507,7 @@ export function sortData(data, sorts) {
         const [field, desc = false] = sort;
         const value1 = row1[field] ?? null;
         const value2 = row2[field] ?? null;
-        const compare = compareValues(value1, value2);
+        const compare = valueCompare(value1, value2);
         return desc ? -compare : compare;
     }, 0));
 }
@@ -509,7 +527,7 @@ export function topData(data, count, categoryFields = null) {
     const categoryOrder = [];
     for (const row of data) {
         const categoryKey = categoryFields === null ? ''
-            : jsonStringifySortKeys(categoryFields.map((field) => (field in row ? row[field] : null)));
+            : valueJSON(categoryFields.map((field) => (field in row ? row[field] : null)));
         if (!(categoryKey in categoryRows)) {
             categoryRows[categoryKey] = [];
             categoryOrder.push(categoryKey);
@@ -528,24 +546,3 @@ export function topData(data, count, categoryFields = null) {
     }
     return dataTop;
 }
-
-
-/**
- * Compare two data values
- *
- * @param {*} value1 - The first value
- * @param {*} value2 - The second value
- * @returns {number} -1 if the first value is less, 1 if the first value is greater, and 0 if they are equal
- */
-export function compareValues(value1, value2) {
-    if (value1 === null) {
-        return value2 === null ? 0 : -1;
-    } else if (value2 === null) {
-        return 1;
-    } else if (value1 instanceof Date) {
-        const time1 = value1.getTime();
-        const time2 = value2.getTime();
-        return time1 < time2 ? -1 : (time1 === time2 ? 0 : 1);
-    }
-    return value1 < value2 ? -1 : (value1 === value2 ? 0 : 1);
-}