@xh/hoist 53.1.0 → 53.2.0

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## v53.2.0 - 2022-11-15
+
+### 🎁 New Features
+* New convenience methods `Store.errors`, `Store.errorCount`, and `StoreRecord.allErrors` for getting
+  easy access to validation errors in the data package.
+* A new flag `Store.validationIsComplex` which governs whether non-changed
+  uncommitted records need to be revalidated when any record in the store is changed.  This flag
+  defaults to `false`, which  should be correct for most applications.  Set to `true` for stores with
+  validations that depend on other editable record values in the store (e.g. unique constraints).
+
+### ⚙️ Technical
+* Major performance improvements to validation of records in stores.  This is a critical fix for
+  applications that do bulk insertion of hundreds of rows or greater in editable grids.
+
 ## v53.1.0 - 2022-11-03
 
 ### 🎁 New Features

package/data/Store.js

@@ -13,12 +13,15 @@ import {
     castArray,
     defaultsDeep,
     differenceBy,
+    flatMapDeep,
     isArray,
     isEmpty,
     isFunction,
     isNil,
     isString,
-    remove as lodashRemove
+    values,
+    remove as lodashRemove,
+    uniq
 } from 'lodash';
 import {Field} from './Field';
 import {parseFilter} from './filter/Utils';
@@ -60,6 +63,9 @@ export class Store extends HoistBase {
     /** @member {boolean} */
     freezeData;
 
+    /** @member {boolean} */
+    validationIsComplex;
+
     /** @member {Filter}  */
     @observable.ref filter;
 
@@ -127,6 +133,8 @@ export class Store extends HoistBase {
      *      object creation, and raw data processing when reloading reference-identical data.
      *      Should not be used if a processRawData function that depends on external state is
      *      provided, as this function will be circumvented on subsequent reloads.  Default false.
+     * @param {boolean} [c.validationIsComplex] - set to true to always validate all uncommitted
+     *      records on every change to uncommitted records (add, modify, or remove). Default false.
      * @param {Object} [c.experimental] - flags for experimental features. These features are
      *     designed for early client-access and testing, but are not yet part of the Hoist API.
      * @param {Object[]} [c.data] - source data to load.
@@ -144,6 +152,7 @@ export class Store extends HoistBase {
         freezeData = true,
         idEncodesTreePath = false,
         reuseRecords = false,
+        validationIsComplex = false,
         experimental,
         data
     }) {
@@ -161,6 +170,7 @@ export class Store extends HoistBase {
         this.freezeData = freezeData;
         this.idEncodesTreePath = idEncodesTreePath;
         this.reuseRecords = reuseRecords;
+        this.validationIsComplex = validationIsComplex;
         this.lastUpdated = Date.now();
 
         this.resetRecords();
@@ -692,6 +702,21 @@ export class Store extends HoistBase {
         return this._current.maxDepth;  // maxDepth should not be effected by filtering.
     }
 
+    /** @return {StoreErrorMap} - Map of StoreRecord IDs to StoreRecord-level error maps. */
+    get errors() {
+        return this.validator.errors;
+    }
+
+    /** @return {number} - count of all validation errors for the store. */
+    get errorCount() {
+        return this.validator.errorCount;
+    }
+
+    /** @return {string[]} - Array of all errors for this store. */
+    get allErrors() {
+        return uniq(flatMapDeep(this.errors, values));
+    }
+
     /**
      * Get a record by ID, or null if no matching record found.
      *
@@ -1007,3 +1032,11 @@ function isChildDataObject(obj) {
  * @property {Object} rawData - data for the child records to be added. Can include a `children`
  *      property that will be processed into new (grand)child records.
  */
+
+/**
+ * @typedef {Object.<string, string[]>} RecordErrorMap - map of Field names -> Field-level error lists.
+ */
+
+/**
+ * @typedef {Object.<StoreRecordId, RecordErrorMap>} StoreErrorMap - map of StoreRecord IDs -> StoreRecord-level error maps.
+ */

package/data/StoreRecord.js

@@ -5,7 +5,7 @@
  * Copyright © 2022 Extremely Heavy Industries Inc.
  */
 import {throwIf} from '@xh/hoist/utils/js';
-import {isNil} from 'lodash';
+import {isNil, flatMap} from 'lodash';
 import {ValidationState} from './validation/ValidationState';
 
 /**
@@ -152,6 +152,11 @@ export class StoreRecord {
         return this.validator?.errors ?? {};
     }
 
+    /** @return {string[]} - Array of all errors for this record. */
+    get allErrors() {
+        return flatMap(this.errors);
+    }
+
     /** @return {number} - count of all validation errors for the record. */
     get errorCount() {
         return this.validator?.errorCount ?? 0;

package/data/impl/RecordValidator.js

@@ -5,21 +5,25 @@
  * Copyright © 2022 Extremely Heavy Industries Inc.
  */
 
-import {HoistBase} from '@xh/hoist/core';
 import {ValidationState} from '@xh/hoist/data';
-import {computed, makeObservable} from '@xh/hoist/mobx';
-import {find, map, some, sumBy} from 'lodash';
-import {RecordFieldValidator} from './RecordFieldValidator';
+import {computed} from '@xh/hoist/mobx';
+import {compact, flatten, isEmpty, mapValues, values} from 'lodash';
+import {makeObservable, observable, runInAction} from 'mobx';
+import {TaskObserver} from '../../core';
 
 /**
  * Computes validation state for a StoreRecord
  * @private
  */
-export class RecordValidator extends HoistBase {
+export class RecordValidator {
 
     /** @member {StoreRecord} */
     record;
 
+    @observable.ref _fieldErrors = null;
+    _validationTask = TaskObserver.trackLast();
+    _validationRunId = 0;
+
     /** @member {StoreRecordId} */
     get id() {
         return this.record.id;
@@ -46,34 +50,28 @@ export class RecordValidator extends HoistBase {
     /** @return {RecordErrorMap} - map of field names -> field-level errors. */
     @computed.struct
     get errors() {
-        return this.getErrorMap();
+        return this._fieldErrors ?? {};
     }
 
     /** @return {number} - count of all validation errors for the record. */
     @computed
     get errorCount() {
-        return sumBy(this._validators, 'errorCount');
+        return flatten(values(this._fieldErrors)).length;
     }
 
     /** @return {boolean} - true if any fields are currently recomputing their validation state. */
     @computed
     get isPending() {
-        return some(this._validators, it => it.isPending);
+        return this._validationTask.isPending;
     }
 
-    _validators = [];
-
     /**
      * @param {Object} c - RecordValidator configuration.
      * @param {StoreRecord} c.record - record to validate
      */
     constructor({record}) {
-        super();
-        makeObservable(this);
         this.record = record;
-
-        const {fields} = this.record.store;
-        this._validators = fields.map(field => new RecordFieldValidator({record, field}));
+        makeObservable(this);
     }
 
     /**
@@ -81,36 +79,60 @@ export class RecordValidator extends HoistBase {
      * @returns {Promise<boolean>}
      */
     async validateAsync() {
-        const promises = map(this._validators, v => v.validateAsync());
-        await Promise.all(promises);
+        let runId = ++this._validationRunId,
+            fieldErrors = {},
+            {record} = this,
+            fieldsToValidate = record.store.fields.filter(it => !isEmpty(it.rules));
+
+        const promises = fieldsToValidate.flatMap(field => {
+            fieldErrors[field.name] = [];
+            return field.rules.map(async (rule) => {
+                const result = await this.evaluateRuleAsync(record, field, rule);
+                fieldErrors[field.name].push(result);
+            });
+        });
+        await Promise.all(promises).linkTo(this._validationTask);
+
+        if (runId !== this._validationRunId) return;
+        fieldErrors = mapValues(fieldErrors, it => compact(flatten(it)));
+
+        runInAction(() => this._fieldErrors = fieldErrors);
+
         return this.isValid;
     }
 
     /** @return {ValidationState} - the current validation state for the record. */
     getValidationState() {
         const VS = ValidationState,
-            states = map(this._validators, v => v.validationState);
-        if (states.includes(VS.NotValid)) return VS.NotValid;
-        if (states.includes(VS.Unknown)) return VS.Unknown;
-        return VS.Valid;
-    }
+            {_fieldErrors} = this;
 
-    /** @return {RecordErrorMap} - map of field names -> field-level errors. */
-    getErrorMap() {
-        const ret = {};
-        this._validators.forEach(v => ret[v.id] = v.errors);
-        return ret;
+        if (_fieldErrors === null) return VS.Unknown; // Before executing any rules
+
+        return (values(_fieldErrors).some(errors => !isEmpty(errors))) ?
+            VS.NotValid :
+            VS.Valid;
     }
 
-    /**
-     * @param {string} id - ID of RecordFieldValidator (should match field.name)
-     * @return {RecordFieldValidator}
-     */
-    findFieldValidator(id) {
-        return find(this._validators, {id});
+    async evaluateRuleAsync(record, field, rule) {
+        const values = record.getValues(),
+            {name, displayName} = field,
+            value = record.get(name);
+
+        if (this.ruleIsActive(record, field, rule)) {
+            const promises = rule.check.map(async (constraint) => {
+                const fieldState = {value, name, displayName, record};
+                return await constraint(fieldState, values);
+            });
+
+            const ret = await Promise.all(promises);
+            return compact(flatten(ret));
+        }
+
+        return [];
     }
-}
 
-/**
- * @typedef {Object.<string, string[]>} RecordErrorMap - map of Field names -> Field-level error lists.
- */
+    ruleIsActive(record, field, rule) {
+        const {when} = rule;
+        return !when || when(field, record.getValues());
+    }
+}

package/data/impl/StoreValidator.js

@@ -5,12 +5,14 @@
  * Copyright © 2022 Extremely Heavy Industries Inc.
  */
 
-import {XH, HoistBase} from '@xh/hoist/core';
+import {HoistBase} from '@xh/hoist/core';
 import {computed, makeObservable, observable} from '@xh/hoist/mobx';
-import {find, sumBy, map, some} from 'lodash';
+import {sumBy, chunk} from 'lodash';
+import {runInAction} from 'mobx';
+import {logDebug, findIn} from '../../utils/js';
+import {ValidationState} from '../validation/ValidationState';
 
 import {RecordValidator} from './RecordValidator';
-import {ValidationState} from '../validation/ValidationState';
 
 /**
  * Computes validation state for a Store's uncommitted Records
@@ -48,17 +50,21 @@ export class StoreValidator extends HoistBase {
     /** @return {number} - count of all validation errors for the store. */
     @computed
     get errorCount() {
-        return sumBy(this._validators, 'errorCount');
+        return sumBy(this.validators, 'errorCount');
     }
 
     /** @return {boolean} - true if any records are currently recomputing their validation state. */
     @computed
     get isPending() {
-        return some(this._validators, it => it.isPending);
+        return findIn(this._validators, it => it.isPending);
+    }
+
+    get validators() {
+        return this.mapValidators();
     }
 
-    /** @member {RecordValidator[]} */
-    @observable.ref _validators = [];
+    /** @member {Map<StoreRecordId, RecordValidator>} */
+    @observable.ref _validators = new Map();
 
     /**
      * @param {Object} c - StoreValidator configuration.
@@ -80,15 +86,14 @@ export class StoreValidator extends HoistBase {
      * @returns {Promise<boolean>}
      */
     async validateAsync() {
-        const promises = map(this._validators, v => v.validateAsync());
-        await Promise.all(promises);
+        await this.validateInChunksAsync(this.validators);
         return this.isValid;
     }
 
     /** @return {ValidationState} - the current validation state for the store. */
     getValidationState() {
         const VS = ValidationState,
-            states = map(this._validators, v => v.validationState);
+            states = this.mapValidators(v => v.validationState);
         if (states.includes(VS.NotValid)) return VS.NotValid;
         if (states.includes(VS.Unknown)) return VS.Unknown;
         return VS.Valid;
@@ -106,7 +111,7 @@ export class StoreValidator extends HoistBase {
      * @return {RecordValidator}
      */
     findRecordValidator(id) {
-        return find(this._validators, {id});
+        return this._validators.get(id);
     }
 
     //---------------------------------------
@@ -117,12 +122,46 @@ export class StoreValidator extends HoistBase {
     }
 
     async syncValidatorsAsync() {
-        XH.safeDestroy(this._validators);
-        this._validators = this.uncommittedRecords.map(record => new RecordValidator({record}));
-        return this.validateAsync();
+        const isComplex = this.store.validationIsComplex,
+            currValidators = this._validators,
+            newValidators = new Map(),
+            toValidate = [];
+
+        this.uncommittedRecords.forEach(record => {
+            const {id} = record;
+
+            // Re-use existing validators to preserve validation state and avoid churn.
+            let validator = currValidators.get(id);
+
+            // 1) If exists validator for an unchanged record, no need to validate
+            if (!isComplex && validator?.record == record) {
+                newValidators.set(id, validator);
+                return;
+            }
+
+            // 2) Otherwise create/update the validator, and trigger validation
+            if (!validator) {
+                validator = new RecordValidator({record});
+            } else {
+                validator.record = record;
+            }
+            newValidators.set(id, validator);
+            toValidate.push(validator);
+        });
+
+        await this.validateInChunksAsync(toValidate);
+        runInAction(() => this._validators = newValidators);
     }
-}
 
-/**
- * @typedef {Object.<StoreRecordId, RecordErrorMap>} StoreErrorMap - map of StoreRecord IDs -> StoreRecord-level error maps.
- */
+    async validateInChunksAsync(validators) {
+        logDebug(`Validating ${validators.length} records`, this);
+        const validateChunks = chunk(validators, 100);
+        for (let chunk of validateChunks) {
+            await Promise.all(chunk.map(v => v.validateAsync()));
+        }
+    }
+
+    mapValidators(fn = undefined) {
+        return Array.from(this._validators.values(), fn);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xh/hoist",
-  "version": "53.1.0",
+  "version": "53.2.0",
   "description": "Hoist add-on for building and deploying React Applications.",
   "repository": "github:xh/hoist-react",
   "homepage": "https://xh.io",

package/data/impl/RecordFieldValidator.js

@@ -1,150 +0,0 @@
-/*
- * This file belongs to Hoist, an application development toolkit
- * developed by Extremely Heavy Industries (www.xh.io | info@xh.io)
- *
- * Copyright © 2022 Extremely Heavy Industries Inc.
- */
-
-import {HoistBase, managed, TaskObserver} from '@xh/hoist/core';
-import {ValidationState} from '@xh/hoist/data';
-import {computed, makeObservable, observable, runInAction} from '@xh/hoist/mobx';
-import {compact, flatten, isEmpty, isNil} from 'lodash';
-
-/**
- * Computes validation state for a Field on a StoreRecord instance
- * @private
- */
-export class RecordFieldValidator extends HoistBase {
-
-    /** @member {StoreRecord} */
-    record;
-
-    /** @member {Field} */
-    field;
-
-    /** @return {string} */
-    get id() {
-        return this.field.name;
-    }
-
-    /** @return {Rule[]} */
-    get rules() {
-        return this.field.rules;
-    }
-
-    /** @return {boolean} - true if the field is confirmed to be Valid. */
-    @computed
-    get isValid() {
-        return this.validationState === ValidationState.Valid;
-    }
-
-    /** @return {boolean} - true if the field is confirmed to be NotValid. */
-    @computed
-    get isNotValid() {
-        return this.validationState === ValidationState.NotValid;
-    }
-
-    /** @return {ValidationState} - the current validation state of the field. */
-    @computed
-    get validationState() {
-        return this.getValidationState();
-    }
-
-    /** @return {string[]} - all validation errors for the field. */
-    @computed.struct
-    get errors() {
-        return this.getErrorList();
-    }
-
-    /** @return {number} - count of all validation errors for the field. */
-    @computed
-    get errorCount() {
-        return this.errors.length;
-    }
-
-    /** @return {boolean} - true if any rules are currently recomputing their validation state. */
-    get isPending() {
-        return this._validationTask.isPending;
-    }
-
-    // An array with the result of evaluating each rule. Each element will be array of strings
-    // containing any validation errors for the rule. If validation for the rule has not
-    // completed will contain null
-    @observable _errors;
-
-    @managed _validationTask = TaskObserver.trackLast();
-    _validationRunId = 0;
-
-    /**
-     * @param {Object} c - RecordFieldValidator configuration.
-     * @param {StoreRecord} c.record - record to validate
-     * @param {Field} c.field - Field to validate
-     */
-    constructor({record, field}) {
-        super();
-        makeObservable(this);
-        this.record = record;
-        this.field = field;
-        this._errors = this.rules.map(() => null);
-    }
-
-    /**
-     * Recompute validations for the record field and return true if valid.
-     * @returns {Promise<boolean>}
-     */
-    async validateAsync() {
-        await this.evaluateAsync().linkTo(this._validationTask);
-        return this.isValid;
-    }
-
-    /** @return {ValidationState} - the current validation state for the record field. */
-    getValidationState() {
-        const VS = ValidationState,
-            {_errors} = this;
-        if (_errors.some(e => !isEmpty(e))) return VS.NotValid;
-        if (_errors.some(e => isNil(e))) return VS.Unknown;
-        return VS.Valid;
-    }
-
-    /** @member {string[]} - all validation errors for this field. */
-    getErrorList() {
-        return compact(flatten(this._errors));
-    }
-
-    //---------------------------------------
-    // Implementation
-    //---------------------------------------
-    async evaluateAsync() {
-        const runId = ++this._validationRunId;
-        const promises = this.rules.map(async (rule, idx) => {
-            const result = await this.evaluateRuleAsync(rule);
-            if (runId === this._validationRunId) {
-                runInAction(() => this._errors[idx] = result);
-            }
-        });
-        await Promise.all(promises).linkTo(this._validationTask);
-    }
-
-    async evaluateRuleAsync(rule) {
-        const {record, field} = this;
-        if (this.ruleIsActive(rule)) {
-            const promises = rule.check.map(async (constraint) => {
-                const {name, displayName} = field,
-                    value = record.get(name),
-                    fieldState = {value, name, displayName, record};
-
-                return await constraint(fieldState, record.getValues());
-            });
-
-            const ret = await Promise.all(promises);
-            return compact(flatten(ret));
-        }
-        return [];
-    }
-
-    ruleIsActive(rule) {
-        const {record, field} = this,
-            {when} = rule;
-        return !when || when(field, record.getValues());
-    }
-}