forms-angular 0.12.0-beta.33 → 0.12.0-beta.330

package/dist/server/data_form.js

@@ -1,21 +1,43 @@
-'use strict';
-Object.defineProperty(exports, "__esModule", { value: true });
-// This part of forms-angular borrows _very_ heavily from https://github.com/Alexandre-Strzelewicz/angular-bridge
+import { Types } from "mongoose";
+// This part of forms-angular borrows from https://github.com/Alexandre-Strzelewicz/angular-bridge
 // (now https://github.com/Unitech/angular-bridge
-var _ = require('lodash');
-var util = require('util');
-var extend = require('node.extend');
-var async = require('async');
-var debug = false;
+import _ from "lodash";
+import util from "util";
+import extend from "node.extend";
+import async from "async";
+let debug = false;
 function logTheAPICalls(req, res, next) {
-    void (res);
-    console.log('API     : ' + req.method + ' ' + req.url + '  [ ' + JSON.stringify(req.body) + ' ]');
+    void res;
+    console.log("API     : " +
+        req.method +
+        " " +
+        req.url +
+        "  [ " +
+        JSON.stringify(req.body) +
+        " ]");
     next();
 }
+var entityMap = {
+    "&": "&",
+    "<": "&lt;",
+    ">": "&gt;",
+    '"': "&quot;",
+    "'": "&#39;",
+    "/": "&#x2F;",
+    "`": "&#x60;",
+    "=": "&#x3D;",
+};
+function escapeHtml(string) {
+    return String(string).replace(/[&<>"'`=\/]/g, function fromEntityMap(s) {
+        return entityMap[s];
+    });
+}
 function processArgs(options, array) {
     if (options.authentication) {
-        var authArray = _.isArray(options.authentication) ? options.authentication : [options.authentication];
-        for (var i = authArray.length - 1; i >= 0; i--) {
+        let authArray = _.isArray(options.authentication)
+            ? options.authentication
+            : [options.authentication];
+        for (let i = authArray.length - 1; i >= 0; i--) {
             array.splice(1, 0, authArray[i]);
         }
     }
@@ -25,248 +47,501 @@ function processArgs(options, array) {
     array[0] = options.urlPrefix + array[0];
     return array;
 }
-var DataForm = function (mongoose, app, options) {
-    this.app = app;
-    app.locals.formsAngular = app.locals.formsAngular || [];
-    app.locals.formsAngular.push(this);
-    this.mongoose = mongoose;
-    mongoose.set('debug', debug);
-    mongoose.Promise = global.Promise;
-    this.options = _.extend({
-        urlPrefix: '/api/'
-    }, options || {});
-    this.resources = [];
-    this.searchFunc = async.forEach;
-    this.registerRoutes();
-    this.app.get.apply(this.app, processArgs(this.options, ['search', this.searchAll()]));
-    for (var pluginName in this.options.plugins) {
-        var pluginObj = this.options.plugins[pluginName];
-        this[pluginName] = pluginObj.plugin(this, processArgs, pluginObj.options);
+export class FormsAngular {
+    constructor(mongoose, app, options) {
+        this.mongoose = mongoose;
+        this.app = app;
+        app.locals.formsAngular = app.locals.formsAngular || [];
+        app.locals.formsAngular.push(this);
+        mongoose.set("debug", debug);
+        mongoose.Promise = global.Promise;
+        this.options = _.extend({
+            urlPrefix: "/api/",
+        }, options || {});
+        this.resources = [];
+        this.searchFunc = async.forEach;
+        // Do the plugin routes first
+        for (let pluginName in this.options.plugins) {
+            if (this.options.plugins.hasOwnProperty(pluginName)) {
+                let pluginObj = this.options.plugins[pluginName];
+                this.options.plugins[pluginName] = Object.assign(this.options.plugins[pluginName], pluginObj.plugin(this, processArgs, pluginObj.options));
+            }
+        }
+        const search = "search/", schema = "schema/", report = "report/", resourceName = ":resourceName", id = "/:id", formName = "/:formName", newClarifier = "/new";
+        this.app.get.apply(this.app, processArgs(this.options, ["models", this.models()]));
+        this.app.get.apply(this.app, processArgs(this.options, [search + resourceName, this.search()]));
+        this.app.get.apply(this.app, processArgs(this.options, [schema + resourceName, this.schema()]));
+        this.app.get.apply(this.app, processArgs(this.options, [
+            schema + resourceName + formName,
+            this.schema(),
+        ]));
+        this.app.get.apply(this.app, processArgs(this.options, [report + resourceName, this.report()]));
+        this.app.get.apply(this.app, processArgs(this.options, [
+            report + resourceName + "/:reportName",
+            this.report(),
+        ]));
+        this.app.get.apply(this.app, processArgs(this.options, [resourceName, this.collectionGet()]));
+        // return the List attributes for all records.  two endpoints that go through the same handler so permissions
+        // can be applied differently for the two use cases.  /listAll is intended for listing records on a page;
+        // /picklistAll for listing them in a <select> or similar - used by record-handler's setUpLookupOptions() method, for cases
+        // where there's a lookup that doesn't use the fngajax option
+        this.app.get.apply(this.app, processArgs(this.options, [
+            resourceName + "/listAll",
+            this.entityListAll(),
+        ]));
+        this.app.get.apply(this.app, processArgs(this.options, [
+            resourceName + "/picklistAll",
+            this.entityListAll(),
+        ]));
+        // return the List attributes for a record - used by fng-ui-select
+        this.app.get.apply(this.app, processArgs(this.options, [
+            resourceName + id + "/list",
+            this.entityList(),
+        ]));
+        // 2x get, with and without formName
+        this.app.get.apply(this.app, processArgs(this.options, [resourceName + id, this.entityGet()]));
+        this.app.get.apply(this.app, processArgs(this.options, [
+            resourceName + formName + id,
+            this.entityGet(),
+        ])); // We don't use the form name, but it can optionally be included so it can be referenced by the permissions check
+        // 3x post (for creating a new record), with and without formName, and in the case of without, with or without /new (which isn't needed if there's no formName)
+        this.app.post.apply(this.app, processArgs(this.options, [resourceName, this.collectionPost()]));
+        this.app.post.apply(this.app, processArgs(this.options, [
+            resourceName + newClarifier,
+            this.collectionPost(),
+        ]));
+        this.app.post.apply(this.app, processArgs(this.options, [
+            resourceName + formName + newClarifier,
+            this.collectionPost(),
+        ]));
+        // 2x post and 2x put (for saving modifications to existing record), with and without formName
+        // (You can POST or PUT to update data)
+        this.app.post.apply(this.app, processArgs(this.options, [resourceName + id, this.entityPut()]));
+        this.app.post.apply(this.app, processArgs(this.options, [
+            resourceName + formName + id,
+            this.entityPut(),
+        ]));
+        this.app.put.apply(this.app, processArgs(this.options, [resourceName + id, this.entityPut()]));
+        this.app.put.apply(this.app, processArgs(this.options, [
+            resourceName + formName + id,
+            this.entityPut(),
+        ]));
+        // 2x delete, with and without formName
+        this.app.delete.apply(this.app, processArgs(this.options, [resourceName + id, this.entityDelete()]));
+        this.app.delete.apply(this.app, processArgs(this.options, [
+            resourceName + formName + id,
+            this.entityDelete(),
+        ]));
+        this.app.get.apply(this.app, processArgs(this.options, ["search", this.searchAll()]));
     }
-};
-module.exports = exports = DataForm;
-DataForm.prototype.extractTimestampFromMongoID = function (record) {
-    var timestamp = record.toString().substring(0, 8);
-    return new Date(parseInt(timestamp, 16) * 1000);
-};
-DataForm.prototype.getListFields = function (resource, doc, cb) {
-    function getFirstMatchingField(keyList, type) {
-        for (var i = 0; i < keyList.length; i++) {
-            var fieldDetails = resource.model.schema['tree'][keyList[i]];
-            if (fieldDetails.type && (!type || fieldDetails.type.name === type) && keyList[i] !== '_id') {
+    getFirstMatchingField(resource, doc, keyList, type) {
+        for (let i = 0; i < keyList.length; i++) {
+            let fieldDetails = resource.model.schema["tree"][keyList[i]];
+            if (fieldDetails.type &&
+                (!type || fieldDetails.type.name === type) &&
+                keyList[i] !== "_id") {
                 resource.options.listFields = [{ field: keyList[i] }];
-                return doc[keyList[i]];
+                return doc ? doc[keyList[i]] : keyList[i];
             }
         }
     }
-    var that = this;
-    var display = '';
-    var listFields = resource.options.listFields;
-    if (listFields) {
-        async.map(listFields, function (aField, cbm) {
-            if (typeof doc[aField.field] !== 'undefined') {
-                if (aField.params) {
-                    if (aField.params.ref) {
-                        var fieldOptions = resource.model.schema['paths'][aField.field].options;
-                        if (typeof fieldOptions.ref === 'string') {
-                            var lookupResource_1 = that.getResource(fieldOptions.ref);
-                            if (lookupResource_1) {
-                                var hiddenFields = that.generateHiddenFields(lookupResource_1, false);
-                                hiddenFields.__v = 0;
-                                lookupResource_1.model.findOne({ _id: doc[aField.field] }).select(hiddenFields).exec(function (err, doc2) {
-                                    if (err) {
+    getListFields(resource, doc, cb) {
+        const that = this;
+        let display = "";
+        let listFields = resource.options.listFields;
+        if (listFields) {
+            async.map(listFields, function (aField, cbm) {
+                if (typeof doc[aField.field] !== "undefined") {
+                    if (aField.params) {
+                        if (aField.params.ref) {
+                            let fieldOptions = resource.model.schema["paths"][aField.field].options;
+                            if (typeof fieldOptions.ref === "string") {
+                                let lookupResource = that.getResource(fieldOptions.ref);
+                                if (lookupResource) {
+                                    let hiddenFields = that.generateHiddenFields(lookupResource, false);
+                                    lookupResource.model
+                                        .findOne({ _id: doc[aField.field] })
+                                        .select(hiddenFields)
+                                        .exec()
+                                        .then((doc2) => {
+                                        that.getListFields(lookupResource, doc2, cbm);
+                                    })
+                                        .catch((err) => {
                                         cbm(err);
-                                    }
-                                    else {
-                                        that.getListFields(lookupResource_1, doc2, cbm);
-                                    }
-                                });
+                                    });
+                                }
+                            }
+                            else {
+                                throw new Error("No support for ref type " + aField.params.ref.type);
+                            }
+                        }
+                        else if (aField.params.params === "timestamp") {
+                            let date = that.extractTimestampFromMongoID(doc[aField.field]);
+                            cbm(null, date.toLocaleDateString() + " " + date.toLocaleTimeString());
+                        }
+                        else if (!aField.params.params) {
+                            throw new Error(`Missing idIsList params for resource ${resource.resourceName}: ${JSON.stringify(aField.params)}`);
+                        }
+                        else if (typeof doc[aField.params.params] === "function") {
+                            const resultOrPromise = doc[aField.params.params]();
+                            if (typeof resultOrPromise.then === "function") {
+                                resultOrPromise.then((result) => cbm(null, result));
+                            }
+                            else {
+                                cbm(null, resultOrPromise);
                             }
                         }
                         else {
-                            throw new Error('No support for ref type ' + aField.params.ref.type);
+                            throw new Error(`No support for idIsList params for resource ${resource.resourceName}: ${JSON.stringify(aField.params)}`);
                         }
                     }
-                    else if (aField.params.params === 'timestamp') {
-                        var date = this.extractTimestampFromMongoID(doc[aField.field]);
-                        cbm(null, date.toLocaleDateString() + ' ' + date.toLocaleTimeString());
+                    else {
+                        cbm(null, doc[aField.field]);
                     }
                 }
                 else {
-                    cbm(null, doc[aField.field]);
+                    cbm(null, "");
                 }
-            }
-            else {
-                cbm(null, '');
-            }
-        }, function (err, results) {
-            if (err) {
-                cb(err);
-            }
-            else {
-                if (results) {
-                    cb(err, results.join(' ').trim());
+            }, function (err, results) {
+                if (err) {
+                    cb(err);
                 }
                 else {
-                    console.log('No results ' + listFields);
+                    if (results) {
+                        cb(err, results.join(" ").trim());
+                    }
+                    else {
+                        console.log("No results " + listFields);
+                    }
                 }
-            }
-        });
-    }
-    else {
-        var keyList = Object.keys(resource.model.schema['tree']);
-        // No list field specified - use the first String field,
-        display = getFirstMatchingField(keyList, 'String') ||
-            // and if there aren't any then just take the first field
-            getFirstMatchingField(keyList);
-        cb(null, display.trim());
-    }
-};
-/**
- * Registers all REST routes with the provided `app` object.
- */
-DataForm.prototype.registerRoutes = function () {
-    var search = 'search/', schema = 'schema/', report = 'report/', resourceName = ':resourceName', id = '/:id';
-    this.app.get.apply(this.app, processArgs(this.options, ['models', this.models()]));
-    this.app.get.apply(this.app, processArgs(this.options, [search + resourceName, this.search()]));
-    this.app.get.apply(this.app, processArgs(this.options, [schema + resourceName, this.schema()]));
-    this.app.get.apply(this.app, processArgs(this.options, [schema + resourceName + '/:formName', this.schema()]));
-    this.app.get.apply(this.app, processArgs(this.options, [report + resourceName, this.report()]));
-    this.app.get.apply(this.app, processArgs(this.options, [report + resourceName + '/:reportName', this.report()]));
-    this.app.get.apply(this.app, processArgs(this.options, [resourceName, this.collectionGet()]));
-    this.app.post.apply(this.app, processArgs(this.options, [resourceName, this.collectionPost()]));
-    this.app.get.apply(this.app, processArgs(this.options, [resourceName + id, this.entityGet()]));
-    this.app.post.apply(this.app, processArgs(this.options, [resourceName + id, this.entityPut()])); // You can POST or PUT to update data
-    this.app.put.apply(this.app, processArgs(this.options, [resourceName + id, this.entityPut()]));
-    this.app.delete.apply(this.app, processArgs(this.options, [resourceName + id, this.entityDelete()]));
-    // return the List attributes for a record - used by select2
-    this.app.get.apply(this.app, processArgs(this.options, [resourceName + id + '/list', this.entityList()]));
-};
-DataForm.prototype.newResource = function (model, options) {
-    options = options || {};
-    options.suppressDeprecatedMessage = true;
-    var passModel = model;
-    if (typeof model !== 'function') {
-        passModel = model.model;
-    }
-    this.addResource(passModel.modelName, passModel, options);
-};
-//    Add a resource, specifying the model and any options.
-//    Models may include their own options, which means they can be passed through from the model file
-DataForm.prototype.addResource = function (resourceName, model, options) {
-    var resource = {
-        resourceName: resourceName,
-        options: options || {}
-    };
-    if (!resource.options.suppressDeprecatedMessage) {
-        console.log('addResource is deprecated - see https://github.com/forms-angular/forms-angular/issues/39');
-    }
-    if (typeof model === 'function') {
-        resource.model = model;
+            });
+        }
+        else {
+            const keyList = Object.keys(resource.model.schema["tree"]);
+            // No list field specified - use the first String field,
+            display =
+                this.getFirstMatchingField(resource, doc, keyList, "String") ||
+                    // and if there aren't any then just take the first field
+                    this.getFirstMatchingField(resource, doc, keyList);
+            cb(null, display.trim());
+        }
     }
-    else {
-        resource.model = model.model;
-        for (var prop in model) {
-            if (model.hasOwnProperty(prop) && prop !== 'model') {
-                resource.options[prop] = model[prop];
+    // generate a Mongo projection that can be used to restrict a query to return only those fields from the given
+    // resource that are identified as "list" fields (i.e., ones that should appear whenever records of that type are
+    // displayed in a list)
+    generateListFieldProjection(resource) {
+        const projection = {};
+        const listFields = resource.options?.listFields;
+        // resource.options.listFields will identify all of the fields from resource that have a value for .list.
+        // generally, that value will be "true", identifying the corresponding field as one which should be
+        // included whenever records of that type appear in a list.
+        // occasionally, it will instead be "{ ref: true }"", which means something entirely different -
+        // this means that the field requires a lookup translation before it can be displayed on a form.
+        // for our purposes, we're interested in only the first of these two cases, so we'll ignore anything where
+        // field.params.ref has a truthy value
+        if (listFields) {
+            for (const field of listFields) {
+                if (!field.params?.ref) {
+                    projection[field.field] = 1;
+                }
             }
         }
+        else {
+            const keyList = Object.keys(resource.model.schema["tree"]);
+            const firstField = 
+            // No list field specified - use the first String field,
+            this.getFirstMatchingField(resource, undefined, keyList, "String") ||
+                // and if there aren't any then just take the first field
+                this.getFirstMatchingField(resource, undefined, keyList);
+            projection[firstField] = 1;
+        }
+        return projection;
     }
-    extend(resource.options, this.preprocess(resource, resource.model.schema['paths'], null));
-    if (resource.options.searchImportance) {
-        this.searchFunc = async.forEachSeries;
+    newResource(model, options) {
+        options = options || {};
+        options.suppressDeprecatedMessage = true;
+        let passModel = model;
+        if (typeof model !== "function") {
+            passModel = model.model;
+        }
+        this.addResource(passModel.modelName, passModel, options);
     }
-    if (this.searchFunc === async.forEachSeries) {
-        this.resources.splice(_.sortedIndexBy(this.resources, resource, function (obj) {
-            return obj.options.searchImportance || 99;
-        }), 0, resource);
+    //    Add a resource, specifying the model and any options.
+    //    Models may include their own options, which means they can be passed through from the model file
+    addResource(resourceName, model, options) {
+        let resource = {
+            resourceName: resourceName,
+            resourceNameLower: resourceName.toLowerCase(),
+            options: options || {},
+        };
+        if (!resource.options.suppressDeprecatedMessage) {
+            console.log("addResource is deprecated - see https://github.com/forms-angular/forms-angular/issues/39");
+        }
+        // Check all the synonyms are lower case
+        resource.options.synonyms?.forEach((s) => {
+            s.name = s.name.toLowerCase();
+        });
+        if (typeof model === "function") {
+            resource.model = model;
+        }
+        else {
+            resource.model = model.model;
+            for (const prop in model) {
+                if (model.hasOwnProperty(prop) && prop !== "model") {
+                    resource.options[prop] = model[prop];
+                }
+            }
+        }
+        extend(resource.options, this.preprocess(resource, resource.model.schema.paths));
+        resource.options.searchFields = [];
+        // commenting this out, as we used to do this in a place where the type of resource.model.schema was any,
+        // so it was allowed, despite the fact that _indexes is not a known property of a mongoose schema.
+        // changing it to indexes does compile, but this might not be what was intended
+        //
+        // for (let j = 0; j < resource.model.schema._indexes.length; j++) {
+        //     let attributes = resource.model.schema._indexes[j][0];
+        //     let field = Object.keys(attributes)[0];
+        //     if (resource.options.searchFields.indexOf(field) === -1) {
+        //         resource.options.searchFields.push(field);
+        //     }
+        // }
+        function addSearchFields(schema, pathSoFar) {
+            for (let path in schema.paths) {
+                if (path !== "_id" && schema.paths.hasOwnProperty(path)) {
+                    const qualifiedPath = pathSoFar ? pathSoFar + "." + path : path;
+                    if (schema.paths[path].options.index &&
+                        !schema.paths[path].options.noSearch) {
+                        if (resource.options.searchFields.indexOf(qualifiedPath) === -1) {
+                            resource.options.searchFields.push(qualifiedPath);
+                        }
+                    }
+                    else if (schema.paths[path].schema) {
+                        addSearchFields(schema.paths[path].schema, qualifiedPath);
+                    }
+                }
+            }
+        }
+        addSearchFields(resource.model.schema, "");
+        if (resource.options.searchImportance) {
+            this.searchFunc = async.forEachSeries;
+        }
+        if (this.searchFunc === async.forEachSeries) {
+            this.resources.splice(_.sortedIndexBy(this.resources, resource, function (obj) {
+                return obj.options.searchImportance || 99;
+            }), 0, resource);
+        }
+        else {
+            this.resources.push(resource);
+        }
     }
-    else {
-        this.resources.push(resource);
+    getResource(name) {
+        return _.find(this.resources, function (resource) {
+            return (resource.resourceName === name || resource.options.resourceName === name);
+        });
     }
-};
-DataForm.prototype.getResource = function (name) {
-    return _.find(this.resources, function (resource) {
-        return resource.resourceName === name;
-    });
-};
-DataForm.prototype.getResourceFromCollection = function (name) {
-    return _.find(this.resources, function (resource) {
-        return resource.model.collection.collectionName === name;
-    });
-};
-DataForm.prototype.internalSearch = function (req, resourcesToSearch, includeResourceInResults, limit, callback) {
-    if (typeof req.query === 'undefined') {
-        req.query = {};
+    getResourceFromCollection(name) {
+        return _.find(this.resources, function (resource) {
+            return resource.model.collection.collectionName === name;
+        });
     }
-    var searches = [], resourceCount = resourcesToSearch.length, searchFor = req.query.q || '', filter = req.query.f;
-    function translate(string, array, context) {
-        if (array) {
-            var translation = _.find(array, function (fromTo) {
-                return fromTo.from === string && (!fromTo.context || fromTo.context === context);
+    // Using the given (already-populated) AmbiguousRecordStore, generate text suitable for
+    // disambiguation of each ambiguous record, and pass that to the given disambiguateItemCallback
+    // so our caller can decorate the ambiguous record in whatever way it deems appropriate.
+    //
+    // The ambiguousRecordStore provided to this function (generated either by a call to
+    // buildSingleResourceAmbiguousRecordStore() or buildMultiResourceAmbiguousRecordStore()) will
+    // already be grouping records by the resource that should be used to disambiguate them, with
+    // the name of that resource being the primary index property of the store.
+    //
+    // The disambiguation text will be the concatenation (space-seperated) of the list fields for
+    // the doc from that resource whose _id matches the value of record[disambiguationField].
+    //
+    // allRecords should include all of the ambiguous records (also held by AmbiguousRecordStore)
+    // as well as those found not to be ambiguous.  The final act of this function will be to delete
+    // the disambiguation field from those records - it is only going to be there for the purpose
+    // of disambiguation, and should not be returned by our caller once disambiguation is complete.
+    //
+    // The scary-looking templating used here ensures that the objects in allRecords (and also
+    // ambiguousRecordStore) include an (optional) string property with the name identified by
+    // disambiguationField.  For the avoidance of doubt, "prop" here could be anything - "foo in f"
+    // would achieve the same result.
+    disambiguate(allRecords, ambiguousRecordStore, disambiguationField, disambiguateItemCallback, completionCallback) {
+        const that = this;
+        async.map(Object.keys(ambiguousRecordStore), function (resourceName, cbm) {
+            const resource = that.getResource(resourceName);
+            const projection = that.generateListFieldProjection(resource);
+            resource.model
+                .find({
+                _id: {
+                    $in: ambiguousRecordStore[resourceName].map((sr) => sr[disambiguationField]),
+                },
+            })
+                .select(projection)
+                .lean()
+                .then((disambiguationRecs) => {
+                for (const ambiguousResult of ambiguousRecordStore[resourceName]) {
+                    const disambiguator = disambiguationRecs.find((d) => d._id.toString() ===
+                        ambiguousResult[disambiguationField].toString());
+                    if (disambiguator) {
+                        let suffix = "";
+                        for (const listField in projection) {
+                            if (disambiguator[listField]) {
+                                if (suffix) {
+                                    suffix += " ";
+                                }
+                                suffix += disambiguator[listField];
+                            }
+                        }
+                        if (suffix) {
+                            disambiguateItemCallback(ambiguousResult, suffix);
+                        }
+                    }
+                }
+                cbm(null);
+            })
+                .catch((err) => {
+                cbm(err);
             });
-            if (translation) {
-                string = translation.to;
+        }, (err) => {
+            for (const record of allRecords) {
+                delete record[disambiguationField];
             }
-        }
-        return string;
+            completionCallback(err);
+        });
     }
-    // return a string that determines the sort order of the resultObject
-    function calcResultValue(obj) {
-        function padLeft(score, reqLength, str) {
-            if (str === void 0) { str = '0'; }
-            return new Array(1 + reqLength - String(score).length).join(str) + score;
+    internalSearch(req, resourcesToSearch, includeResourceInResults, limit, callback) {
+        if (typeof req.query === "undefined") {
+            req.query = {};
         }
-        var sortString = '';
-        sortString += padLeft(obj.addHits || 9, 1);
-        sortString += padLeft(obj.searchImportance || 99, 2);
-        sortString += padLeft(obj.weighting || 9999, 4);
-        sortString += obj.text;
-        return sortString;
-    }
-    if (filter) {
-        filter = JSON.parse(filter);
-    }
-    for (var i = 0; i < resourceCount; i++) {
-        var resource = resourcesToSearch[i];
-        if (resourceCount === 1 || resource.options.searchImportance !== false) {
-            var schema = resource.model.schema;
-            var indexedFields = [];
-            for (var j = 0; j < schema._indexes.length; j++) {
-                var attributes = schema._indexes[j][0];
-                var field = Object.keys(attributes)[0];
-                if (indexedFields.indexOf(field) === -1) {
-                    indexedFields.push(field);
-                }
-            }
-            for (var path in schema.paths) {
-                if (path !== '_id' && schema.paths.hasOwnProperty(path)) {
-                    if (schema.paths[path]._index && !schema.paths[path].options.noSearch) {
-                        if (indexedFields.indexOf(path) === -1) {
-                            indexedFields.push(path);
-                        }
+        const timestamps = {
+            sentAt: req.query.sentAt,
+            startedAt: new Date().valueOf(),
+            completedAt: undefined,
+        };
+        let searches = [], resourceCount = resourcesToSearch.length, searchFor = req.query.q || "", filter = req.query.f;
+        function translate(string, array, context) {
+            if (array) {
+                let translation = _.find(array, function (fromTo) {
+                    return (fromTo.from === string &&
+                        (!fromTo.context || fromTo.context === context));
+                });
+                if (translation) {
+                    string = translation.to;
+                }
+            }
+            return string;
+        }
+        // return a string that determines the sort order of the resultObject
+        function calcResultValue(obj) {
+            function padLeft(score, reqLength, str = "0") {
+                return (new Array(1 + reqLength - String(score).length).join(str) + score);
+            }
+            let sortString = "";
+            sortString += padLeft(obj.addHits || 9, 1);
+            sortString += padLeft(obj.searchImportance || 99, 2);
+            sortString += padLeft(obj.weighting || 9999, 4);
+            sortString += obj.text;
+            return sortString;
+        }
+        if (filter) {
+            filter = JSON.parse(filter);
+        }
+        // See if we are narrowing down the resources
+        let collectionName;
+        let collectionNameLower;
+        let colonPos = searchFor.indexOf(":");
+        switch (colonPos) {
+            case -1:
+                // Original behaviour = do nothing different
+                break;
+            case 0:
+                // "Special search" - yet to be implemented
+                break;
+            default:
+                collectionName = searchFor.slice(0, colonPos);
+                collectionNameLower = collectionName.toLowerCase();
+                searchFor = searchFor.slice(colonPos + 1, 999).trim();
+                if (searchFor === "") {
+                    searchFor = "?";
+                }
+                break;
+        }
+        for (let i = 0; i < resourceCount; i++) {
+            let resource = resourcesToSearch[i];
+            if (resourceCount === 1 ||
+                (resource.options.searchImportance !== false &&
+                    (!collectionName ||
+                        collectionNameLower === resource.resourceNameLower ||
+                        resource.options?.synonyms?.find((s) => s.name === collectionNameLower)))) {
+                let searchFields = resource.options.searchFields;
+                if (searchFields.length === 0) {
+                    console.log("ERROR: Searching on a collection with no indexes " +
+                        resource.resourceName);
+                }
+                let synonymObj = resource.options?.synonyms?.find((s) => s.name.toLowerCase() === collectionNameLower);
+                const synonymFilter = synonymObj?.filter;
+                for (let m = 0; m < searchFields.length; m++) {
+                    let searchObj = {
+                        resource: resource,
+                        field: searchFields[m],
+                    };
+                    if (synonymFilter) {
+                        searchObj.filter = synonymFilter;
                     }
+                    searches.push(searchObj);
                 }
             }
-            if (indexedFields.length === 0) {
-                console.log('ERROR: Searching on a collection with no indexes ' + resource.resourceName);
+        }
+        const that = this;
+        let results = [];
+        let moreCount = 0;
+        let searchCriteria;
+        let searchStrings;
+        let multiMatchPossible = false;
+        if (searchFor === "?") {
+            // interpret this as a wildcard (so there is no way to search for ?
+            searchCriteria = null;
+        }
+        else {
+            // Support for searching anywhere in a field by starting with *
+            let startAnchor = "^";
+            if (searchFor.slice(0, 1) === "*") {
+                startAnchor = "";
+                searchFor = searchFor.slice(1);
             }
-            for (var m = 0; m < indexedFields.length; m++) {
-                searches.push({ resource: resource, field: indexedFields[m] });
+            // THe snippet to escape the special characters comes from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions
+            searchFor = searchFor.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&");
+            multiMatchPossible = searchFor.includes(" ");
+            if (multiMatchPossible) {
+                searchStrings = searchFor.split(" ");
             }
+            let modifiedSearchStr = multiMatchPossible
+                ? searchStrings.join("|")
+                : searchFor;
+            searchFor = searchFor.toLowerCase(); // For later case-insensitive comparison
+            // Removed the logic that preserved spaces when collection was specified because Louise asked me to.
+            searchCriteria = {
+                $regex: `${startAnchor}(${modifiedSearchStr})`,
+                $options: "i",
+            };
         }
-    }
-    var that = this;
-    var results = [], moreCount = 0, searchCriteria, multiMatchPossible = searchFor.includes(' '), modifiedSearchStr = multiMatchPossible ? searchFor.split(' ').join('|') : searchFor;
-    // Removed the logic that preserved spaces when collection was specified because Louise asked me to.
-    searchCriteria = { $regex: '^(' + modifiedSearchStr + ')', $options: 'i' };
-    var handleSearchResultsFromIndex = function (err, docs, item, cb) {
-        if (!err && docs && docs.length > 0) {
-            async.map(docs, function (aDoc, cbdoc) {
-                // Do we already have them in the list?
-                var thisId = aDoc._id.toString(), resultObject, resultPos;
+        let handleSearchResultsFromIndex = function (err, docs, item, cb) {
+            function handleSingleSearchResult(aDoc, cbdoc) {
+                let thisId = aDoc._id.toString(), resultObject, resultPos;
                 function handleResultsInList() {
-                    resultObject.searchImportance = item.resource.options.searchImportance || 99;
+                    if (multiMatchPossible) {
+                        resultObject.matched = resultObject.matched || [];
+                        // record the index of string that matched, so we don't count it against another field
+                        for (let i = 0; i < searchStrings.length; i++) {
+                            if (aDoc[item.field]?.toLowerCase().indexOf(searchStrings[i]) === 0) {
+                                resultObject.matched.push(i);
+                                break;
+                            }
+                        }
+                    }
+                    resultObject.resourceCollection = item.resource.resourceName;
+                    resultObject.searchImportance =
+                        item.resource.options.searchImportance || 99;
                     if (item.resource.options.localisationData) {
                         resultObject.resource = translate(resultObject.resource, item.resource.options.localisationData, "resource");
                         resultObject.resourceText = translate(resultObject.resourceText, item.resource.options.localisationData, "resourceText");
@@ -275,32 +550,58 @@ DataForm.prototype.internalSearch = function (req, resourcesToSearch, includeRes
                     results.splice(_.sortedIndexBy(results, resultObject, calcResultValue), 0, resultObject);
                     cbdoc(null);
                 }
+                // Do we already have them in the list?
                 for (resultPos = results.length - 1; resultPos >= 0; resultPos--) {
-                    if (results[resultPos].id.toString() === thisId) {
+                    // check for matching id and resource
+                    if (results[resultPos].id.toString() === thisId && results[resultPos].resourceCollection === item.resource.resourceName) {
                         break;
                     }
                 }
                 if (resultPos >= 0) {
-                    resultObject = {};
-                    extend(resultObject, results[resultPos]);
+                    resultObject = Object.assign({}, results[resultPos]);
                     // If they have already matched then improve their weighting
-                    // TODO: if the search string is B F currently Benjamin Barker scores same as Benjamin Franklin)
                     if (multiMatchPossible) {
-                        resultObject.addHits = Math.max((resultObject.addHits || 9) - 1, 1);
+                        // record the index of string that matched, so we don't count it against another field
+                        for (let i = 0; i < searchStrings.length; i++) {
+                            if (!resultObject.matched.includes(i) &&
+                                aDoc[item.field]?.toLowerCase().indexOf(searchStrings[i]) === 0) {
+                                resultObject.matched.push(i);
+                                resultObject.addHits = Math.max((resultObject.addHits || 9) - 1, 0);
+                                // remove it from current position
+                                results.splice(resultPos, 1);
+                                // and re-insert where appropriate
+                                results.splice(_.sortedIndexBy(results, resultObject, calcResultValue), 0, resultObject);
+                                break;
+                            }
+                        }
                     }
-                    // remove it from current position
-                    results.splice(resultPos, 1);
-                    // and re-insert where appropriate
-                    results.splice(_.sortedIndexBy(results, resultObject, calcResultValue), 0, resultObject);
                     cbdoc(null);
                 }
                 else {
                     // Otherwise add them new...
+                    let addHits;
+                    if (multiMatchPossible) {
+                        // If they match the whole search phrase in one index they get smaller addHits (so they sort higher)
+                        if (aDoc[item.field]?.toLowerCase().indexOf(searchFor) === 0) {
+                            addHits = 7;
+                        }
+                    }
+                    let disambiguationId;
+                    const opts = item.resource.options;
+                    const disambiguationResource = opts.disambiguation?.resource;
+                    if (disambiguationResource) {
+                        disambiguationId = aDoc[opts.disambiguation.field]?.toString();
+                    }
                     // Use special listings format if defined
-                    var specialListingFormat = item.resource.options.searchResultFormat;
+                    let specialListingFormat = item.resource.options.searchResultFormat;
                     if (specialListingFormat) {
-                        resultObject = specialListingFormat.apply(aDoc);
-                        handleResultsInList();
+                        specialListingFormat.apply(aDoc, [req]).then((resultObj) => {
+                            resultObject = resultObj;
+                            resultObject.addHits = addHits;
+                            resultObject.disambiguationResource = disambiguationResource;
+                            resultObject.disambiguationId = disambiguationId;
+                            handleResultsInList();
+                        });
                     }
                     else {
                         that.getListFields(item.resource, aDoc, function (err, description) {
@@ -311,430 +612,783 @@ DataForm.prototype.internalSearch = function (req, resourcesToSearch, includeRes
                                 resultObject = {
                                     id: aDoc._id,
                                     weighting: 9999,
-                                    text: description
+                                    addHits,
+                                    disambiguationResource,
+                                    disambiguationId,
+                                    text: description,
                                 };
                                 if (resourceCount > 1 || includeResourceInResults) {
-                                    resultObject.resource = resultObject.resourceText = item.resource.resourceName;
+                                    resultObject.resource = resultObject.resourceText =
+                                        item.resource.resourceName;
                                 }
                                 handleResultsInList();
                             }
                         });
                     }
                 }
-            }, function (err) {
+            }
+            if (!err && docs && docs.length > 0) {
+                async.map(docs, handleSingleSearchResult, cb);
+            }
+            else {
                 cb(err);
-            });
-        }
-        else {
-            cb(err);
-        }
-    };
-    this.searchFunc(searches, function (item, cb) {
-        var searchDoc = {};
-        if (filter) {
-            that.hackVariables(filter);
-            extend(searchDoc, filter);
-            if (filter[item.field]) {
-                delete searchDoc[item.field];
-                var obj1 = {}, obj2 = {};
-                obj1[item.field] = filter[item.field];
-                obj2[item.field] = searchCriteria;
-                searchDoc['$and'] = [obj1, obj2];
+            }
+        };
+        this.searchFunc(searches, function (item, cb) {
+            let searchDoc = {};
+            let searchFilter = filter || item.filter;
+            if (searchFilter) {
+                that.hackVariables(searchFilter);
+                extend(searchDoc, searchFilter);
+                if (searchFilter[item.field]) {
+                    delete searchDoc[item.field];
+                    let obj1 = {}, obj2 = {};
+                    obj1[item.field] = searchFilter[item.field];
+                    obj2[item.field] = searchCriteria;
+                    searchDoc["$and"] = [obj1, obj2];
+                }
+                else {
+                    if (searchCriteria) {
+                        searchDoc[item.field] = searchCriteria;
+                    }
+                }
             }
             else {
-                searchDoc[item.field] = searchCriteria;
+                if (searchCriteria) {
+                    searchDoc[item.field] = searchCriteria;
+                }
             }
-        }
-        else {
-            searchDoc[item.field] = searchCriteria;
-        }
-        // The +60 in the next line is an arbitrary safety zone for situations where items that match the string
-        // in more than one index get filtered out.
-        // TODO : Figure out a better way to deal with this
-        if (item.resource.options.searchFunc) {
-            item.resource.options.searchFunc(item.resource, req, null, searchDoc, item.resource.options.searchOrder, limit + 60, null, function (err, docs) {
-                handleSearchResultsFromIndex(err, docs, item, cb);
-            });
-        }
-        else {
-            that.filteredFind(item.resource, req, null, searchDoc, item.resource.options.searchOrder, limit + 60, null, function (err, docs) {
-                handleSearchResultsFromIndex(err, docs, item, cb);
-            });
-        }
-    }, function (err) {
-        if (err) {
-            callback(err);
-        }
-        else {
-            // Strip weighting from the results
-            results = _.map(results, function (aResult) {
-                delete aResult.weighting;
-                return aResult;
-            });
-            if (results.length > limit) {
-                moreCount += results.length - limit;
-                results.splice(limit);
+            /*
+                    The +200 below line is an (imperfect) arbitrary safety zone for situations where items that match the string in more than one index get filtered out.
+                    An example where it fails is searching for "e c" which fails to get a old record Emily Carpenter in a big dataset sorted by date last accessed as they
+                    are not returned within the first 200 in forenames so don't get the additional hit score and languish outside the visible results, though those visible
+                    results end up containing people who only match either c or e (but have been accessed much more recently).
+    
+                    Increasing the number would be a short term fix at the cost of slowing down the search.
+                     */
+            // TODO : Figure out a better way to deal with this
+            if (item.resource.options.searchFunc) {
+                item.resource.options.searchFunc(item.resource, req, null, searchDoc, item.resource.options.searchOrder, limit ? limit + 200 : 0, null, function (err, docs) {
+                    handleSearchResultsFromIndex(err, docs, item, cb);
+                });
             }
-            callback(null, { results: results, moreCount: moreCount });
-        }
-    });
-};
-DataForm.prototype.wrapInternalSearch = function (req, res, resourcesToSearch, includeResourceInResults, limit) {
-    this.internalSearch(req, resourcesToSearch, includeResourceInResults, limit, function (err, resultsObject) {
-        if (err) {
-            res.status(400, err);
-        }
-        else {
-            res.send(resultsObject);
-        }
-    });
-};
-DataForm.prototype.search = function () {
-    return _.bind(function (req, res, next) {
-        if (!(req.resource = this.getResource(req.params.resourceName))) {
-            return next();
-        }
-        this.wrapInternalSearch(req, res, [req.resource], false, 10);
-    }, this);
-};
-DataForm.prototype.searchAll = function () {
-    return _.bind(function (req, res) {
-        this.wrapInternalSearch(req, res, this.resources, true, 10);
-    }, this);
-};
-DataForm.prototype.models = function () {
-    var that = this;
-    return function (req, res) {
-        //    TODO: Make this less wasteful - we only need to send the resourceNames of the resources
-        // Check for optional modelFilter and call it with the request and current list.  Otherwise just return the list.
-        res.send(that.options.modelFilter ? that.options.modelFilter.call(null, req, that.resources) : that.resources);
-    };
-};
-DataForm.prototype.renderError = function (err, redirectUrl, req, res) {
-    if (typeof err === 'string') {
-        res.send(err);
-    }
-    else {
-        res.send(err.message);
-    }
-};
-DataForm.prototype.redirect = function (address, req, res) {
-    res.send(address);
-};
-DataForm.prototype.applySchemaSubset = function (vanilla, schema) {
-    var outPath;
-    if (schema) {
-        outPath = {};
-        for (var fld in schema) {
-            if (schema.hasOwnProperty(fld)) {
-                if (!vanilla[fld]) {
-                    throw new Error('No such field as ' + fld + '.  Is it part of a sub-doc? If so you need the bit before the period.');
-                }
-                outPath[fld] = vanilla[fld];
-                if (vanilla[fld].schema) {
-                    outPath[fld].schema = this.applySchemaSubset(outPath[fld].schema, schema[fld].schema);
-                }
-                outPath[fld].options = outPath[fld].options || {};
-                for (var override in schema[fld]) {
-                    if (schema[fld].hasOwnProperty(override)) {
-                        if (!outPath[fld].options.form) {
-                            outPath[fld].options.form = {};
+            else {
+                that.filteredFind(item.resource, req, null, searchDoc, null, item.resource.options.searchOrder, limit ? limit + 200 : 0, null, function (err, docs) {
+                    handleSearchResultsFromIndex(err, docs, item, cb);
+                });
+            }
+        }, function (err) {
+            if (err) {
+                callback(err);
+            }
+            else {
+                // Strip weighting from the results
+                results = _.map(results, function (aResult) {
+                    delete aResult.weighting;
+                    return aResult;
+                });
+                if (limit && results.length > limit) {
+                    moreCount += results.length - limit;
+                    results.splice(limit);
+                }
+                that.disambiguate(results, that.buildMultiResourceAmbiguousRecordStore(results, ["text"], "disambiguationResource"), "disambiguationId", (item, disambiguationText) => {
+                    item.text += ` (${disambiguationText})`;
+                }, (err) => {
+                    if (err) {
+                        callback(err);
+                    }
+                    else {
+                        // the disambiguate() call will have deleted the disambiguationIds but we're responsible for
+                        // the disambiguationResources, which we shouldn't be returning to the client
+                        for (const result of results) {
+                            delete result.disambiguationResource;
                         }
-                        outPath[fld].options.form[override] = schema[fld][override];
+                        timestamps.completedAt = new Date().valueOf();
+                        callback(null, { results, moreCount, timestamps });
                     }
-                }
+                });
             }
-        }
+        });
+    }
+    wrapInternalSearch(req, res, resourcesToSearch, includeResourceInResults, limit) {
+        this.internalSearch(req, resourcesToSearch, includeResourceInResults, limit, function (err, resultsObject) {
+            if (err) {
+                res.status(400, err);
+            }
+            else {
+                res.send(resultsObject);
+            }
+        });
     }
-    else {
-        outPath = vanilla;
+    search() {
+        return _.bind(function (req, res, next) {
+            if (!(req.resource = this.getResource(req.params.resourceName))) {
+                return next();
+            }
+            this.wrapInternalSearch(req, res, [req.resource], false, 0);
+        }, this);
     }
-    return outPath;
-};
-DataForm.prototype.preprocess = function (resource, paths, formSchema) {
-    var outPath = {}, hiddenFields = [], listFields = [];
-    if (resource && resource.options && resource.options.idIsList) {
-        paths['_id'].options = paths['_id'].options || {};
-        paths['_id'].options.list = resource.options.idIsList;
+    searchAll() {
+        return _.bind(function (req, res) {
+            this.wrapInternalSearch(req, res, this.resources, true, 10);
+        }, this);
     }
-    for (var element in paths) {
-        if (paths.hasOwnProperty(element) && element !== '__v') {
-            // check for schemas
-            if (paths[element].schema) {
-                var subSchemaInfo = this.preprocess(null, paths[element].schema.paths);
-                outPath[element] = { schema: subSchemaInfo.paths };
-                if (paths[element].options.form) {
-                    outPath[element].options = { form: extend(true, {}, paths[element].options.form) };
-                }
+    models() {
+        const that = this;
+        return function (req, res) {
+            // Check for optional modelFilter and call it with the request and current list.  Otherwise just return the list.
+            let resources = that.options.modelFilter
+                ? that.options.modelFilter.call(null, req, that.resources)
+                : that.resources;
+            if (req.query?.resourceNamesOnly) {
+                resources = resources.map((r) => r.resourceName);
             }
-            else {
-                // check for arrays
-                var realType = paths[element].caster ? paths[element].caster : paths[element];
-                if (!realType.instance) {
-                    if (realType.options.type) {
-                        var type = realType.options.type(), typeType = typeof type;
-                        if (typeType === 'string') {
-                            realType.instance = (!isNaN(Date.parse(type))) ? 'Date' : 'String';
+            res.send(resources);
+        };
+    }
+    renderError(err, redirectUrl, req, res) {
+        res.statusMessage = err?.message || err;
+        res.status(400).end(err?.message || err);
+    }
+    redirect(address, req, res) {
+        res.send(address);
+    }
+    applySchemaSubset(vanilla, schema) {
+        let outPath;
+        if (schema) {
+            outPath = {};
+            for (let fld in schema) {
+                if (schema.hasOwnProperty(fld)) {
+                    if (vanilla[fld]) {
+                        outPath[fld] = _.cloneDeep(vanilla[fld]);
+                        if (vanilla[fld].schema) {
+                            outPath[fld].schema = this.applySchemaSubset(outPath[fld].schema, schema[fld].schema);
+                        }
+                    }
+                    else {
+                        if (fld.slice(0, 8) === "_bespoke") {
+                            outPath[fld] = {
+                                path: fld,
+                                instance: schema[fld]._type,
+                            };
                         }
                         else {
-                            realType.instance = typeType;
+                            throw new Error("No such field as " +
+                                fld +
+                                ".  Is it part of a sub-doc? If so you need the bit before the period.");
                         }
                     }
-                }
-                outPath[element] = extend(true, {}, paths[element]);
-                if (paths[element].options.secure) {
-                    hiddenFields.push(element);
-                }
-                if (paths[element].options.match) {
-                    outPath[element].options.match = paths[element].options.match.source;
-                }
-                var schemaListInfo = paths[element].options.list;
-                if (schemaListInfo) {
-                    var listFieldInfo = { field: element };
-                    if (typeof schemaListInfo === 'object' && Object.keys(schemaListInfo).length > 0) {
-                        listFieldInfo.params = schemaListInfo;
+                    outPath[fld].options = outPath[fld].options || {};
+                    for (const override in schema[fld]) {
+                        if (schema[fld].hasOwnProperty(override)) {
+                            if (override.slice(0, 1) !== "_") {
+                                if (schema[fld].hasOwnProperty(override)) {
+                                    if (!outPath[fld].options.form) {
+                                        outPath[fld].options.form = {};
+                                    }
+                                    outPath[fld].options.form[override] = schema[fld][override];
+                                }
+                            }
+                        }
                     }
-                    listFields.push(listFieldInfo);
                 }
             }
         }
-    }
-    outPath = this.applySchemaSubset(outPath, formSchema);
-    var returnObj = { paths: outPath };
-    if (hiddenFields.length > 0) {
-        returnObj.hide = hiddenFields;
-    }
-    if (listFields.length > 0) {
-        returnObj.listFields = listFields;
-    }
-    return returnObj;
-};
-DataForm.prototype.schema = function () {
-    return _.bind(function (req, res) {
-        if (!(req.resource = this.getResource(req.params.resourceName))) {
-            return res.status(404).end();
-        }
-        var formSchema = null;
-        if (req.params.formName) {
-            formSchema = req.resource.model.schema.statics['form'](req.params.formName);
-        }
-        var paths = this.preprocess(req.resource, req.resource.model.schema.paths, formSchema).paths;
-        res.send(paths);
-    }, this);
-};
-DataForm.prototype.report = function () {
-    return _.bind(function (req, res, next) {
-        if (!(req.resource = this.getResource(req.params.resourceName))) {
-            return next();
-        }
-        var self = this;
-        if (typeof req.query === 'undefined') {
-            req.query = {};
+        else {
+            outPath = vanilla;
         }
-        var reportSchema;
-        if (req.params.reportName) {
-            reportSchema = req.resource.model.schema.statics['report'](req.params.reportName, req);
+        return outPath;
+    }
+    preprocess(resource, paths, formName, formSchema) {
+        function processInternalObject(obj) {
+            return Object.keys(obj).reduce((acc, cur) => {
+                const curType = typeof obj[cur];
+                if ((!["$", "_"].includes(cur.charAt(0)) ||
+                    [
+                        "$in",
+                        "$nin",
+                        "$eq",
+                        "$ne",
+                        "$gt",
+                        "$gte",
+                        "$lt",
+                        "$lte",
+                        "$regex",
+                        "$or",
+                        "$exists",
+                    ].includes(cur)) &&
+                    curType !== "function") {
+                    const val = obj[cur];
+                    if (val) {
+                        if (Array.isArray(val)) {
+                            if (val.length > 0) {
+                                acc[cur] = val;
+                            }
+                        }
+                        else if (curType === "object" &&
+                            !(val instanceof Date) &&
+                            !(val instanceof RegExp)) {
+                            acc[cur] = processInternalObject(obj[cur]);
+                        }
+                        else {
+                            acc[cur] = obj[cur];
+                        }
+                    }
+                }
+                return acc;
+            }, {});
         }
-        else if (req.query.r) {
-            switch (req.query.r[0]) {
-                case '[':
-                    reportSchema = { pipeline: JSON.parse(req.query.r) };
-                    break;
-                case '{':
-                    reportSchema = JSON.parse(req.query.r);
-                    break;
-                default:
-                    return self.renderError(new Error('Invalid "r" parameter'), null, req, res, next);
-            }
+        let outPath = {}, hiddenFields = [], listFields = [];
+        if (resource &&
+            !resource.options?.doNotCacheSchema &&
+            resource.preprocessed &&
+            resource.preprocessed[formName || "__default"]) {
+            return resource.preprocessed[formName || "__default"].paths;
         }
         else {
-            var fields = {};
-            for (var key in req.resource.model.schema.paths) {
-                if (req.resource.model.schema.paths.hasOwnProperty(key)) {
-                    if (key !== '__v' && !req.resource.model.schema.paths[key].options.secure) {
-                        if (key.indexOf('.') === -1) {
-                            fields[key] = 1;
+            if (resource && resource.options && resource.options.idIsList) {
+                paths["_id"].options = paths["_id"].options || {};
+                paths["_id"].options.list = resource.options.idIsList;
+            }
+            for (let element in paths) {
+                if (paths.hasOwnProperty(element) && element !== "__v") {
+                    // check for schemas
+                    if (paths[element].schema) {
+                        let subSchemaInfo = this.preprocess(null, paths[element].schema.paths);
+                        outPath[element] = { schema: subSchemaInfo.paths };
+                        if (paths[element].options.form) {
+                            outPath[element].options = {
+                                form: extend(true, {}, paths[element].options.form),
+                            };
+                        }
+                        // this provides support for entire nested schemas that wish to remain hidden
+                        if (paths[element].options.secure) {
+                            hiddenFields.push(element);
+                        }
+                        // to support hiding individual properties of nested schema would require us
+                        // to do something with subSchemaInfo.hide here
+                    }
+                    else {
+                        // check for arrays
+                        let realType = paths[element].caster
+                            ? paths[element].caster
+                            : paths[element];
+                        if (!realType.instance) {
+                            if (realType.options.type) {
+                                let type = realType.options.type(), typeType = typeof type;
+                                if (typeType === "string") {
+                                    realType.instance = !isNaN(Date.parse(type))
+                                        ? "Date"
+                                        : "String";
+                                }
+                                else {
+                                    realType.instance = typeType;
+                                }
+                            }
+                        }
+                        outPath[element] = processInternalObject(paths[element]);
+                        if (paths[element].options.secure) {
+                            hiddenFields.push(element);
+                        }
+                        if (paths[element].options.match) {
+                            outPath[element].options.match =
+                                paths[element].options.match.source ||
+                                    paths[element].options.match;
+                        }
+                        let schemaListInfo = paths[element].options.list;
+                        if (schemaListInfo) {
+                            let listFieldInfo = { field: element };
+                            if (typeof schemaListInfo === "object" &&
+                                Object.keys(schemaListInfo).length > 0) {
+                                listFieldInfo.params = schemaListInfo;
+                            }
+                            listFields.push(listFieldInfo);
                         }
                     }
                 }
             }
-            reportSchema = {
-                pipeline: [
-                    { $project: fields }
-                ], drilldown: req.params.resourceName + '/|_id|/edit'
-            };
-        }
-        // Replace parameters in pipeline
-        var schemaCopy = {};
-        extend(schemaCopy, reportSchema);
-        schemaCopy.params = schemaCopy.params || [];
-        self.reportInternal(req, req.resource, schemaCopy, function (err, result) {
-            if (err) {
-                self.renderError(err, null, req, res, next);
+            outPath = this.applySchemaSubset(outPath, formSchema);
+            let returnObj = { paths: outPath };
+            if (hiddenFields.length > 0) {
+                returnObj.hide = hiddenFields;
             }
-            else {
-                res.send(result);
+            if (listFields.length > 0) {
+                returnObj.listFields = listFields;
             }
-        });
-    }, this);
-};
-DataForm.prototype.hackVariablesInPipeline = function (runPipeline) {
-    for (var pipelineSection = 0; pipelineSection < runPipeline.length; pipelineSection++) {
-        if (runPipeline[pipelineSection]['$match']) {
-            this.hackVariables(runPipeline[pipelineSection]['$match']);
+            if (resource && !resource.options?.doNotCacheSchema) {
+                resource.preprocessed = resource.preprocessed || {};
+                resource.preprocessed[formName || "__default"] = returnObj;
+            }
+            return returnObj;
         }
     }
-};
-DataForm.prototype.hackVariables = function (obj) {
-    // Replace variables that cannot be serialised / deserialised.  Bit of a hack, but needs must...
-    // Anything formatted 1800-01-01T00:00:00.000Z or 1800-01-01T00:00:00.000+0000 is converted to a Date
-    // Only handles the cases I need for now
-    // TODO: handle arrays etc
-    for (var prop in obj) {
-        if (obj.hasOwnProperty(prop)) {
-            if (typeof obj[prop] === 'string') {
-                var dateTest = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3})(Z|[+ -]\d{4})$/.exec(obj[prop]);
-                if (dateTest) {
-                    obj[prop] = new Date(dateTest[1] + 'Z');
-                }
-                else {
-                    var objectIdTest = /^([0-9a-fA-F]{24})$/.exec(obj[prop]);
-                    if (objectIdTest) {
-                        obj[prop] = new this.mongoose.Types.ObjectId(objectIdTest[1]);
-                    }
-                }
+    schema() {
+        return _.bind(function (req, res) {
+            if (!(req.resource = this.getResource(req.params.resourceName))) {
+                return res.status(404).end();
             }
-            else if (_.isObject(obj[prop])) {
-                this.hackVariables(obj[prop]);
+            let formSchema = null;
+            const formName = req.params.formName;
+            if (req.resource.preprocessed?.[formName || "__default"]) {
+                res.send(req.resource.preprocessed[formName || "__default"].paths);
             }
-        }
-    }
-};
-DataForm.prototype.reportInternal = function (req, resource, schema, callback) {
-    var runPipelineStr;
-    var runPipelineObj;
-    var self = this;
-    if (typeof req.query === 'undefined') {
-        req.query = {};
-    }
-    self.doFindFunc(req, resource, function (err, queryObj) {
-        if (err) {
-            return 'There was a problem with the findFunc for model';
-        }
-        else {
-            runPipelineStr = JSON.stringify(schema.pipeline);
-            for (var param in req.query) {
-                if (req.query[param]) {
-                    if (param !== 'r') { // we don't want to copy the whole report schema (again!)
-                        schema.params[param].value = req.query[param];
+            else {
+                if (formName) {
+                    try {
+                        formSchema = req.resource.model.schema.statics["form"](escapeHtml(formName), req);
+                    }
+                    catch (e) {
+                        return res.status(404).send(e.message);
                     }
                 }
+                let paths = this.preprocess(req.resource, req.resource.model.schema.paths, formName, formSchema).paths;
+                res.send(paths);
             }
-            // Replace parameters with the value
-            if (runPipelineStr) {
-                runPipelineStr = runPipelineStr.replace(/\"\(.+?\)\"/g, function (match) {
-                    var sparam = schema.params[match.slice(2, -2)];
-                    if (sparam.type === 'number') {
-                        return sparam.value;
-                    }
-                    else if (_.isObject(sparam.value)) {
-                        return JSON.stringify(sparam.value);
-                    }
-                    else if (sparam.value[0] === '{') {
-                        return sparam.value;
-                    }
-                    else {
-                        return '"' + sparam.value + '"';
-                    }
-                });
+        }, this);
+    }
+    report() {
+        return _.bind(async function (req, res, next) {
+            if (!(req.resource = this.getResource(req.params.resourceName))) {
+                return next();
+            }
+            const self = this;
+            req._isReport = true; // Can be used to modify findFunc behaviour
+            if (typeof req.query === "undefined") {
+                req.query = {};
+            }
+            let reportSchema;
+            if (req.params.reportName) {
+                try {
+                    reportSchema = await req.resource.model.schema.statics["report"](req.params.reportName, req);
+                }
+                catch (e) {
+                    res.send({ success: false, error: e.message || e });
+                }
+            }
+            else if (req.query.r) {
+                switch (req.query.r[0]) {
+                    case "[":
+                        reportSchema = { pipeline: JSON.parse(req.query.r) };
+                        break;
+                    case "{":
+                        reportSchema = JSON.parse(req.query.r);
+                        break;
+                    default:
+                        return self.renderError(new Error('Invalid "r" parameter'), null, req, res);
+                }
             }
-            // Don't send the 'secure' fields
-            var hiddenFields = self.generateHiddenFields(resource, false);
-            for (var hiddenField in hiddenFields) {
-                if (hiddenFields.hasOwnProperty(hiddenField)) {
-                    if (runPipelineStr.indexOf(hiddenField) !== -1) {
-                        return callback('You cannot access ' + hiddenField);
+            else {
+                let fields = {};
+                for (let key in req.resource.model.schema.paths) {
+                    if (req.resource.model.schema.paths.hasOwnProperty(key)) {
+                        if (key !== "__v" &&
+                            !req.resource.model.schema.paths[key].options.secure) {
+                            if (key.indexOf(".") === -1) {
+                                fields[key] = 1;
+                            }
+                        }
                     }
                 }
+                reportSchema = {
+                    pipeline: [{ $project: fields }],
+                    drilldown: req.params.resourceName + "/|_id|/edit",
+                };
             }
-            runPipelineObj = JSON.parse(runPipelineStr);
-            if (!_.isArray(runPipelineObj)) {
-                runPipelineObj = [runPipelineObj];
+            // Replace parameters in pipeline
+            let schemaCopy = {};
+            extend(schemaCopy, reportSchema);
+            schemaCopy.params = schemaCopy.params || [];
+            self.reportInternal(req, req.resource, schemaCopy, function (err, result) {
+                if (err) {
+                    res.send({ success: false, error: err.message || err });
+                }
+                else {
+                    res.send(result);
+                }
+            });
+        }, this);
+    }
+    hackVariablesInPipeline(runPipeline) {
+        for (let pipelineSection = 0; pipelineSection < runPipeline.length; pipelineSection++) {
+            let sectionType = Object.keys(runPipeline[pipelineSection])[0];
+            if (["$match", "$group", "$project"].includes(sectionType)) {
+                this.hackVariables(runPipeline[pipelineSection][sectionType]);
             }
-            self.hackVariablesInPipeline(runPipelineObj);
-            // Add the findFunc query to the pipeline
-            if (queryObj) {
-                runPipelineObj.unshift({ $match: queryObj });
+        }
+    }
+    hackVariables(obj) {
+        // Replace variables that cannot be serialised / deserialised.  Bit of a hack, but needs must...
+        // Anything formatted 1800-01-01T00:00:00.000Z or 1800-01-01T00:00:00.000+0000 is converted to a Date
+        // Only handles the cases I need for now
+        // TODO: handle arrays etc
+        for (const prop in obj) {
+            if (obj.hasOwnProperty(prop)) {
+                if (typeof obj[prop] === "string") {
+                    const dateTest = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{3})?)(Z|[+ -]\d{2}:?\d{2})$/.exec(obj[prop]);
+                    if (dateTest) {
+                        obj[prop] = new Date(dateTest[1] + "Z");
+                    }
+                    else if (prop !== "$regex") {
+                        const objectIdTest = /^([0-9a-fA-F]{24})$/.exec(obj[prop]);
+                        if (objectIdTest) {
+                            obj[prop] = new Types.ObjectId(objectIdTest[1]);
+                        }
+                    }
+                }
+                else if (_.isObject(obj[prop])) {
+                    this.hackVariables(obj[prop]);
+                }
             }
-            var toDo = {
-                runAggregation: function (cb) {
-                    resource.model.aggregate(runPipelineObj, cb);
+        }
+    }
+    async sanitisePipeline(aggregationParam, hiddenFields, findFuncQry, req) {
+        let that = this;
+        let array = Array.isArray(aggregationParam)
+            ? aggregationParam
+            : [aggregationParam];
+        let retVal = [];
+        let doneHiddenFields = false;
+        if (findFuncQry) {
+            retVal.unshift({ $match: findFuncQry });
+        }
+        async function sanitiseLookupPipeline(stage) {
+            // can't think of a way to exploit this (if you add a findFunc) but it is the responsibility of the user
+            // to ensure that the pipeline returns fields used by the findFunc
+            console.log(`In some scenarios $lookups that use pipelines may not provide all the fields used by the 'findFunc' of a collection.
+              If you get no results returned this might be the explanation.`);
+            /* Sanitise the pipeline we are doing a lookup with, removing hidden fields from that collection */
+            const lookupCollectionName = stage.$lookup.from;
+            const lookupResource = that.getResourceFromCollection(lookupCollectionName);
+            let lookupHiddenLookupFields = {};
+            if (lookupResource) {
+                if (lookupResource.options?.hide?.length > 0) {
+                    lookupHiddenLookupFields = that.generateHiddenFields(lookupResource, false);
                 }
-            };
-            var translations_1 = []; // array of form {ref:'lookupname',translations:[{value:xx, display:'  '}]}
-            // if we need to do any column translations add the function to the tasks list
-            if (schema.columnTranslations) {
-                toDo.applyTranslations = ['runAggregation', function (results, cb) {
-                        function doATranslate(column, theTranslation) {
-                            results['runAggregation'].forEach(function (resultRow) {
-                                var valToTranslate = resultRow[column.field];
-                                valToTranslate = (valToTranslate ? valToTranslate.toString() : '');
-                                var thisTranslation = _.find(theTranslation.translations, function (option) {
-                                    return valToTranslate === option.value.toString();
-                                });
-                                resultRow[column.field] = thisTranslation ? thisTranslation.display : ' * Missing columnTranslation * ';
+            }
+            stage.$lookup.pipeline = await that.sanitisePipeline(stage.$lookup.pipeline, lookupHiddenLookupFields, findFuncQry, req);
+        }
+        for (let pipelineSection = 0; pipelineSection < array.length; pipelineSection++) {
+            let stage = array[pipelineSection];
+            let keys = Object.keys(stage);
+            if (keys.length !== 1) {
+                throw new Error("Invalid pipeline instruction");
+            }
+            switch (keys[0]) {
+                case "$project":
+                case "$addFields":
+                case "$count":
+                case "$facet":
+                case "$group":
+                case "$unset":
+                case "$limit":
+                case "$replaceRoot":
+                case "$replaceWith":
+                case "$sort":
+                case "$unwind":
+                    // We don't care about these - they are all (as far as we know) safe
+                    break;
+                case "$unionWith":
+                    /*
+                              Sanitise the pipeline we are doing a union with, removing hidden fields from that collection
+                           */
+                    if (!stage.$unionWith.coll) {
+                        stage.$unionWith = { coll: stage.$unionWith, pipeline: [] };
+                    }
+                    const unionCollectionName = stage.$unionWith.coll;
+                    const unionResource = that.getResourceFromCollection(unionCollectionName);
+                    let unionHiddenLookupFields = {};
+                    if (unionResource) {
+                        if (unionResource.options?.hide?.length > 0) {
+                            unionHiddenLookupFields = this.generateHiddenFields(unionResource, false);
+                        }
+                    }
+                    stage.$unionWith.pipeline = await that.sanitisePipeline(stage.$unionWith.pipeline, unionHiddenLookupFields, findFuncQry, req);
+                    break;
+                case "$match":
+                    this.hackVariables(array[pipelineSection]["$match"]);
+                    retVal.push(array[pipelineSection]);
+                    if (!doneHiddenFields &&
+                        Object.keys(hiddenFields) &&
+                        Object.keys(hiddenFields).length > 0) {
+                        // We can now project out the hidden fields (we wait for the $match to make sure we don't break
+                        // a select that uses a hidden field
+                        retVal.push({ $project: hiddenFields });
+                        doneHiddenFields = true;
+                    }
+                    stage = null;
+                    break;
+                case "$lookup":
+                case "$graphLookup":
+                    let needFindFunc = true;
+                    if (keys[0] === "$lookup") {
+                        let lookupProps = Object.keys(stage.$lookup);
+                        // First deal with simple $lookups with a single join field equality
+                        if ((lookupProps.length === 4 || lookupProps.length === 5) &&
+                            lookupProps.includes("from") &&
+                            lookupProps.includes("localField") &&
+                            lookupProps.includes("foreignField") &&
+                            lookupProps.includes("as")) {
+                            // If we are doing a lookup using an _id (so not fishing) we don't need to do the findFunc (see tkt #12399)
+                            if (lookupProps.length === 4) {
+                                if (stage.$lookup.foreignField === "_id" || stage.$lookup.localField === "_id") {
+                                    needFindFunc = false;
+                                }
+                            }
+                            else if (lookupProps.length === 5 && lookupProps.includes("pipeline")) {
+                                await sanitiseLookupPipeline.call(this, stage);
+                            }
+                            else {
+                                throw new Error("Unsupported $lookup format");
+                            }
+                        }
+                        else if (lookupProps.length === 4 &&
+                            lookupProps.includes("from") &&
+                            lookupProps.includes("let") &&
+                            lookupProps.includes("pipeline") &&
+                            lookupProps.includes("as")) {
+                            await sanitiseLookupPipeline.call(this, stage);
+                        }
+                        else {
+                            throw new Error(`No support for $lookup of with properties ${lookupProps.join(', ')}`);
+                        }
+                    }
+                    // hide any hiddenfields in the lookup collection
+                    const collectionName = stage[keys[0]].from;
+                    const lookupField = stage[keys[0]].as;
+                    if ((collectionName + lookupField).indexOf("$") !== -1) {
+                        throw new Error('No support for lookups where the "from" or "as" is anything other than a simple string');
+                    }
+                    const resource = that.getResourceFromCollection(collectionName);
+                    if (resource) {
+                        retVal.push(stage);
+                        stage = null;
+                        if (resource.options?.hide?.length > 0) {
+                            const hiddenLookupFields = this.generateHiddenFields(resource, false);
+                            let hiddenFieldsObj = {};
+                            Object.keys(hiddenLookupFields).forEach((hf) => {
+                                hiddenFieldsObj[`${lookupField}.${hf}`] = false;
                             });
+                            retVal.push({ $project: hiddenFieldsObj });
                         }
-                        schema.columnTranslations.forEach(function (columnTranslation) {
-                            if (columnTranslation.translations) {
-                                doATranslate(columnTranslation, columnTranslation);
+                        // Now we need to make sure that we restrict the lookup to documents we have access to (or can provide the _id of)
+                        if (needFindFunc && resource.options.findFunc) {
+                            let allowNulls = false;
+                            // If the next stage is an $unwind
+                            let nextStageIsUnwind = false;
+                            if (array.length >= pipelineSection) {
+                                const nextStage = array[pipelineSection + 1];
+                                let nextKeys = Object.keys(nextStage);
+                                if (nextKeys.length !== 1) {
+                                    throw new Error("Invalid pipeline instruction");
+                                }
+                                if (nextKeys[0] === "$unwind") {
+                                    if (nextStage["$unwind"] === "$" + lookupField) {
+                                        nextStageIsUnwind = true;
+                                    }
+                                    if (nextStage["$unwind"] &&
+                                        nextStage["$unwind"].path === "$" + lookupField) {
+                                        nextStageIsUnwind = true;
+                                        if (nextStage["$unwind"].preserveNullAndEmptyArrays) {
+                                            allowNulls = true;
+                                        }
+                                    }
+                                }
                             }
-                            if (columnTranslation.ref) {
-                                var theTranslation = _.find(translations_1, function (translation) {
-                                    return (translation.ref === columnTranslation.ref);
-                                });
-                                if (theTranslation) {
-                                    doATranslate(columnTranslation, theTranslation);
+                            if (!nextStageIsUnwind) {
+                                throw new Error("No support for $lookup where the next stage is not an $unwind and the resources has a findFunc");
+                            }
+                            // Push the $unwind, add our own findFunc, and increment the pipelineStage counter
+                            retVal.push(array[pipelineSection + 1]);
+                            let lookedUpFindQry = await this.doFindFuncPromise(req, resource);
+                            if (lookedUpFindQry) {
+                                // Now we need to put the lookup base into the criteria
+                                for (const prop in lookedUpFindQry) {
+                                    if (lookedUpFindQry.hasOwnProperty(prop)) {
+                                        lookedUpFindQry[`${lookupField}.${prop}`] =
+                                            lookedUpFindQry[prop];
+                                        delete lookedUpFindQry[prop];
+                                    }
+                                }
+                                if (allowNulls) {
+                                    lookedUpFindQry = {
+                                        $or: [
+                                            lookedUpFindQry,
+                                            { [lookupField]: { $exists: false } },
+                                        ],
+                                    };
+                                }
+                                retVal.push({ $match: lookedUpFindQry });
+                            }
+                            pipelineSection++;
+                        }
+                    }
+                    break;
+                default:
+                    // anything else is either known to be dangerous, not yet needed or we don't know what it is
+                    throw new Error("Unsupported pipeline instruction " + keys[0]);
+            }
+            if (stage) {
+                retVal.push(stage);
+            }
+        }
+        if (!doneHiddenFields &&
+            Object.keys(hiddenFields) &&
+            Object.keys(hiddenFields).length > 0) {
+            // If there was no $match we still need to hide the hidden fields
+            retVal.unshift({ $project: hiddenFields });
+        }
+        return retVal;
+    }
+    reportInternal(req, resource, schema, callback) {
+        let runPipelineStr;
+        let runPipelineObj;
+        let self = this;
+        if (typeof req.query === "undefined") {
+            req.query = {};
+        }
+        self.doFindFunc(req, resource, function (err, queryObj) {
+            if (err) {
+                return "There was a problem with the findFunc for model";
+            }
+            else {
+                runPipelineStr = JSON.stringify(schema.pipeline);
+                for (let param in req.query) {
+                    if (param !== "noinput" && req.query.hasOwnProperty(param)) {
+                        if (req.query[param]) {
+                            if (param !== "r") {
+                                // we don't want to copy the whole report schema (again!)
+                                if (schema.params[param] !== undefined) {
+                                    schema.params[param].value = req.query[param];
                                 }
                                 else {
-                                    cb('Invalid ref property of ' + columnTranslation.ref + ' in columnTranslations ' + columnTranslation.field);
+                                    return callback(`No such parameter as ${param} - try one of ${Object.keys(schema.params).join()}`);
                                 }
                             }
-                        });
-                        cb(null, null);
-                    }];
-                var callFuncs = false;
-                for (var i = 0; i < schema.columnTranslations.length; i++) {
-                    var thisColumnTranslation = schema.columnTranslations[i];
-                    if (thisColumnTranslation.field) {
-                        // if any of the column translations are adhoc funcs, set up the tasks to perform them
-                        if (thisColumnTranslation.fn) {
-                            callFuncs = true;
                         }
-                        // if this column translation is a "ref", set up the tasks to look up the values and populate the translations
-                        if (thisColumnTranslation.ref) {
-                            var lookup = self.getResource(thisColumnTranslation.ref);
-                            if (lookup) {
-                                if (!toDo[thisColumnTranslation.ref]) {
-                                    var getFunc = function (ref) {
-                                        var lookup = ref;
-                                        return function (cb) {
-                                            var translateObject = { ref: lookup.resourceName, translations: [] };
-                                            translations_1.push(translateObject);
-                                            lookup.model.find({}, {}, { lean: true }, function (err, findResults) {
-                                                if (err) {
-                                                    cb(err);
-                                                }
-                                                else {
-                                                    // TODO - this ref func can probably be done away with now that list fields can have ref
-                                                    var j_1 = 0;
-                                                    async.whilst(function () {
-                                                        return j_1 < findResults.length;
+                    }
+                }
+                // Replace parameters with the value
+                if (runPipelineStr) {
+                    runPipelineStr = runPipelineStr.replace(/"\(.+?\)"/g, function (match) {
+                        let sparam = schema.params[match.slice(2, -2)];
+                        if (sparam !== undefined) {
+                            if (sparam.type === "number") {
+                                return sparam.value;
+                            }
+                            else if (_.isObject(sparam.value)) {
+                                return JSON.stringify(sparam.value);
+                            }
+                            else if (sparam.value[0] === "{") {
+                                return sparam.value;
+                            }
+                            else {
+                                return '"' + sparam.value + '"';
+                            }
+                        }
+                        else {
+                            return callback(`No such parameter as ${match.slice(2, -2)} - try one of ${Object.keys(schema.params).join()}`);
+                        }
+                    });
+                }
+                runPipelineObj = JSON.parse(runPipelineStr);
+                self.hackVariablesInPipeline(runPipelineObj);
+                let hiddenFields = self.generateHiddenFields(resource, false);
+                let toDo = {
+                    runAggregation: function (cb) {
+                        self
+                            .sanitisePipeline(runPipelineObj, hiddenFields, queryObj, req)
+                            .then((runPipelineObj) => {
+                            resource.model
+                                .aggregate(runPipelineObj)
+                                .then((results) => {
+                                cb(null, results);
+                            })
+                                .catch((err) => {
+                                cb(err);
+                            });
+                        })
+                            .catch((err) => {
+                            throw new Error("Error in sanitisePipeline " + err);
+                        });
+                    },
+                };
+                let translations = []; // array of form {ref:'lookupname',translations:[{value:xx, display:'  '}]}
+                // if we need to do any column translations add the function to the tasks list
+                if (schema.columnTranslations) {
+                    toDo.applyTranslations = [
+                        "runAggregation",
+                        function (results, cb) {
+                            function doATranslate(column, theTranslation) {
+                                results["runAggregation"].forEach(function (resultRow) {
+                                    let valToTranslate = resultRow[column.field];
+                                    valToTranslate = valToTranslate
+                                        ? valToTranslate.toString()
+                                        : "";
+                                    let thisTranslation = _.find(theTranslation.translations, function (option) {
+                                        return valToTranslate === option.value.toString();
+                                    });
+                                    resultRow[column.field] = thisTranslation
+                                        ? thisTranslation.display
+                                        : " * Missing columnTranslation * ";
+                                });
+                            }
+                            schema.columnTranslations.forEach(function (columnTranslation) {
+                                if (columnTranslation.translations) {
+                                    doATranslate(columnTranslation, columnTranslation);
+                                }
+                                if (columnTranslation.ref) {
+                                    let theTranslation = _.find(translations, function (translation) {
+                                        return translation.ref === columnTranslation.ref;
+                                    });
+                                    if (theTranslation) {
+                                        doATranslate(columnTranslation, theTranslation);
+                                    }
+                                    else {
+                                        cb("Invalid ref property of " +
+                                            columnTranslation.ref +
+                                            " in columnTranslations " +
+                                            columnTranslation.field);
+                                    }
+                                }
+                            });
+                            cb(null, null);
+                        },
+                    ];
+                    let callFuncs = false;
+                    for (let i = 0; i < schema.columnTranslations.length; i++) {
+                        let thisColumnTranslation = schema.columnTranslations[i];
+                        if (thisColumnTranslation.field) {
+                            // if any of the column translations are adhoc funcs, set up the tasks to perform them
+                            if (thisColumnTranslation.fn) {
+                                callFuncs = true;
+                            }
+                            // if this column translation is a "ref", set up the tasks to look up the values and populate the translations
+                            if (thisColumnTranslation.ref) {
+                                let lookup = self.getResource(thisColumnTranslation.ref);
+                                if (lookup) {
+                                    if (!toDo[thisColumnTranslation.ref]) {
+                                        let getFunc = function (ref) {
+                                            let lookup = ref;
+                                            return function (cb) {
+                                                let translateObject = {
+                                                    ref: lookup.resourceName,
+                                                    translations: [],
+                                                };
+                                                translations.push(translateObject);
+                                                lookup.model
+                                                    .find({}, {})
+                                                    .lean()
+                                                    .exec()
+                                                    .then((findResults) => {
+                                                    let j = 0;
+                                                    async.whilst(function (cbtest) {
+                                                        cbtest(null, j < findResults.length);
                                                     }, function (cbres) {
-                                                        var theResult = findResults[j_1];
-                                                        translateObject.translations[j_1] = translateObject.translations[j_1] || {};
-                                                        var theTranslation = translateObject.translations[j_1];
-                                                        j_1++;
+                                                        let theResult = findResults[j];
+                                                        translateObject.translations[j] =
+                                                            translateObject.translations[j] || {};
+                                                        let theTranslation = translateObject.translations[j];
+                                                        j++;
                                                         self.getListFields(lookup, theResult, function (err, description) {
                                                             if (err) {
                                                                 cbres(err);
@@ -746,59 +1400,74 @@ DataForm.prototype.reportInternal = function (req, resource, schema, callback) {
                                                             }
                                                         });
                                                     }, cb);
-                                                }
-                                            });
+                                                })
+                                                    .catch((err) => {
+                                                    cb(err);
+                                                });
+                                            };
                                         };
-                                    };
-                                    toDo[thisColumnTranslation.ref] = getFunc(lookup);
-                                    toDo.applyTranslations.unshift(thisColumnTranslation.ref); // Make sure we populate lookup before doing translation
+                                        toDo[thisColumnTranslation.ref] = getFunc(lookup);
+                                        toDo.applyTranslations.unshift(thisColumnTranslation.ref); // Make sure we populate lookup before doing translation
+                                    }
+                                }
+                                else {
+                                    return callback("Invalid ref property of " +
+                                        thisColumnTranslation.ref +
+                                        " in columnTranslations " +
+                                        thisColumnTranslation.field);
                                 }
                             }
-                            else {
-                                return callback('Invalid ref property of ' + thisColumnTranslation.ref + ' in columnTranslations ' + thisColumnTranslation.field);
+                            if (!thisColumnTranslation.translations &&
+                                !thisColumnTranslation.ref &&
+                                !thisColumnTranslation.fn) {
+                                return callback("A column translation needs a ref, fn or a translations property - " +
+                                    thisColumnTranslation.field +
+                                    " has neither");
                             }
                         }
-                        if (!thisColumnTranslation.translations && !thisColumnTranslation.ref && !thisColumnTranslation.fn) {
-                            return callback('A column translation needs a ref, fn or a translations property - ' + thisColumnTranslation.field + ' has neither');
+                        else {
+                            return callback("A column translation needs a field property");
                         }
                     }
-                    else {
-                        return callback('A column translation needs a field property');
-                    }
-                }
-                if (callFuncs) {
-                    toDo['callFunctions'] = ['runAggregation', function (results, cb) {
-                            async.each(results.runAggregation, function (row, cb) {
-                                for (var i = 0; i < schema.columnTranslations.length; i++) {
-                                    var thisColumnTranslation = schema.columnTranslations[i];
-                                    if (thisColumnTranslation.fn) {
-                                        thisColumnTranslation.fn(row, cb);
+                    if (callFuncs) {
+                        toDo["callFunctions"] = [
+                            "runAggregation",
+                            function (results, cb) {
+                                async.each(results.runAggregation, function (row, cb) {
+                                    for (let i = 0; i < schema.columnTranslations.length; i++) {
+                                        let thisColumnTranslation = schema.columnTranslations[i];
+                                        if (thisColumnTranslation.fn) {
+                                            thisColumnTranslation.fn(row, cb);
+                                        }
                                     }
-                                }
-                            }, function () {
-                                cb(null);
-                            });
-                        }];
-                    toDo.applyTranslations.unshift('callFunctions'); // Make sure we do function before translating its result
+                                }, function () {
+                                    cb(null);
+                                });
+                            },
+                        ];
+                        toDo.applyTranslations.unshift("callFunctions"); // Make sure we do function before translating its result
+                    }
                 }
+                async.auto(toDo, function (err, results) {
+                    if (err) {
+                        callback(err);
+                    }
+                    else {
+                        callback(null, {
+                            success: true,
+                            schema: schema,
+                            report: results.runAggregation,
+                            paramsUsed: schema.params,
+                        });
+                    }
+                });
             }
-            async.auto(toDo, function (err, results) {
-                if (err) {
-                    callback(err);
-                }
-                else {
-                    // TODO: Could loop through schema.params and just send back the values
-                    callback(null, { success: true, schema: schema, report: results.runAggregation, paramsUsed: schema.params });
-                }
-            });
-        }
-    });
-};
-DataForm.prototype.saveAndRespond = function (req, res, hiddenFields) {
-    function internalSave(doc) {
-        doc.save(function (err, doc2) {
-            if (err) {
-                var err2 = { status: 'err' };
+        });
+    }
+    saveAndRespond(req, res, hiddenFields) {
+        function internalSave(doc) {
+            function decorateError(err) {
+                let err2 = { status: "err" };
                 if (!err.errors) {
                     err2.message = err.message;
                 }
@@ -806,18 +1475,21 @@ DataForm.prototype.saveAndRespond = function (req, res, hiddenFields) {
                     extend(err2, err);
                 }
                 if (debug) {
-                    console.log('Error saving record: ' + JSON.stringify(err2));
+                    console.log("Error saving record: " + JSON.stringify(err2));
                 }
                 res.status(400).send(err2);
             }
-            else {
-                doc2 = doc2.toObject();
-                for (var hiddenField in hiddenFields) {
-                    if (hiddenFields.hasOwnProperty(hiddenField) && hiddenFields[hiddenField]) {
-                        var parts = hiddenField.split('.');
-                        var lastPart = parts.length - 1;
-                        var target = doc2;
-                        for (var i = 0; i < lastPart; i++) {
+            doc
+                .save()
+                .then((saved) => {
+                saved = saved.toObject();
+                for (const hiddenField in hiddenFields) {
+                    if (hiddenFields.hasOwnProperty(hiddenField) &&
+                        hiddenFields[hiddenField]) {
+                        let parts = hiddenField.split(".");
+                        let lastPart = parts.length - 1;
+                        let target = saved;
+                        for (let i = 0; i < lastPart; i++) {
                             if (target.hasOwnProperty(parts[i])) {
                                 target = target[parts[i]];
                             }
@@ -827,360 +1499,741 @@ DataForm.prototype.saveAndRespond = function (req, res, hiddenFields) {
                         }
                     }
                 }
-                res.send(doc2);
-            }
-        });
-    }
-    var doc = req.doc;
-    if (typeof req.resource.options.onSave === 'function') {
-        req.resource.options.onSave(doc, req, function (err) {
-            if (err) {
-                throw err;
-            }
-            internalSave(doc);
-        });
-    }
-    else {
-        internalSave(doc);
-    }
-};
-/**
- * All entities REST functions have to go through this first.
- */
-DataForm.prototype.processCollection = function (req) {
-    req.resource = this.getResource(req.params.resourceName);
-};
-/**
- * Renders a view with the list of docs, which may be modified by query parameters
- */
-DataForm.prototype.collectionGet = function () {
-    return _.bind(function (req, res, next) {
-        this.processCollection(req);
-        if (!req.resource) {
-            return next();
-        }
-        if (typeof req.query === 'undefined') {
-            req.query = {};
-        }
-        try {
-            var aggregationParam = req.query.a ? JSON.parse(req.query.a) : null;
-            var findParam = req.query.f ? JSON.parse(req.query.f) : {};
-            var limitParam = req.query.l ? JSON.parse(req.query.l) : 0;
-            var skipParam = req.query.s ? JSON.parse(req.query.s) : 0;
-            var orderParam = req.query.o ? JSON.parse(req.query.o) : req.resource.options.listOrder;
-            // Dates in aggregation must be Dates
-            if (aggregationParam) {
-                this.hackVariablesInPipeline(aggregationParam);
-            }
-            var self_1 = this;
-            this.filteredFind(req.resource, req, aggregationParam, findParam, orderParam, limitParam, skipParam, function (err, docs) {
-                if (err) {
-                    return self_1.renderError(err, null, req, res, next);
+                if (doc.__toClient) {
+                    /* Use this to pass anything back to the client that is not saved in the record
+                                Possible use case - sent a message saying that since this data has changed in a
+                                particular way the user should consider doing something else
+                                 */
+                    saved.__toClient = doc.__toClient;
+                }
+                res.send(saved);
+            })
+                .catch((err) => {
+                if (req.resource.options.onSaveError) {
+                    req.resource.options.onSaveError(err, req, res)
+                        .then(() => {
+                        decorateError(err);
+                    });
                 }
                 else {
-                    res.send(docs);
+                    decorateError(err);
                 }
             });
         }
-        catch (e) {
-            res.send(e);
-        }
-    }, this);
-};
-DataForm.prototype.doFindFunc = function (req, resource, cb) {
-    if (resource.options.findFunc) {
-        resource.options.findFunc(req, cb);
-    }
-    else {
-        cb(null);
-    }
-};
-DataForm.prototype.filteredFind = function (resource, req, aggregationParam, findParam, sortOrder, limit, skip, callback) {
-    var that = this;
-    var hiddenFields = this.generateHiddenFields(resource, false);
-    var stashAggregationResults;
-    function doAggregation(cb) {
-        if (aggregationParam) {
-            resource.model.aggregate(aggregationParam, function (err, aggregationResults) {
+        let doc = req.doc;
+        if (typeof req.resource.options.onSave === "function") {
+            req.resource.options.onSave(doc, req, function (err) {
                 if (err) {
                     throw err;
                 }
-                else {
-                    stashAggregationResults = aggregationResults;
-                    cb(_.map(aggregationResults, function (obj) {
-                        return obj._id;
-                    }));
-                }
+                internalSave(doc);
             });
         }
         else {
-            cb([]);
+            internalSave(doc);
         }
     }
-    that.doFindFunc(req, resource, function (err, queryObj) {
-        if (err) {
-            callback(err);
-        }
-        else {
-            if (queryObj && aggregationParam) {
-                var match = aggregationParam.find(function (o) { return o.hasOwnProperty('$match'); });
-                if (match) {
-                    _.extend(match.$match, queryObj);
-                }
-                else {
-                    aggregationParam.unshift({ $match: queryObj });
+    /**
+     * All entities REST functions have to go through this first.
+     */
+    processCollection(req) {
+        req.resource = this.getResource(req.params.resourceName);
+    }
+    /**
+     * Renders a view with the list of docs, which may be modified by query parameters
+     */
+    collectionGet() {
+        return _.bind(function (req, res, next) {
+            this.processCollection(req);
+            if (!req.resource) {
+                return next();
+            }
+            if (typeof req.query === "undefined") {
+                req.query = {};
+            }
+            try {
+                const aggregationParam = req.query.a ? JSON.parse(req.query.a) : null;
+                const findParam = req.query.f ? JSON.parse(req.query.f) : {};
+                const projectParam = req.query.p ? JSON.parse(req.query.p) : {};
+                const limitParam = req.query.l ? JSON.parse(req.query.l) : 0;
+                const skipParam = req.query.s ? JSON.parse(req.query.s) : 0;
+                const orderParam = req.query.o
+                    ? JSON.parse(req.query.o)
+                    : req.resource.options.listOrder;
+                // Dates in aggregation must be Dates
+                if (aggregationParam) {
+                    this.hackVariablesInPipeline(aggregationParam);
                 }
+                const self = this;
+                this.filteredFind(req.resource, req, aggregationParam, findParam, projectParam, orderParam, limitParam, skipParam, function (err, docs) {
+                    if (err) {
+                        return self.renderError(err, null, req, res);
+                    }
+                    else {
+                        res.send(docs);
+                    }
+                });
+            }
+            catch (e) {
+                res.status(400).send(e.message);
+            }
+        }, this);
+    }
+    generateProjection(hiddenFields, projectParam) {
+        let type;
+        function setSelectType(typeChar, checkChar) {
+            if (type === checkChar) {
+                throw new Error("Cannot mix include and exclude fields in select");
             }
-            doAggregation(function (idArray) {
-                if (aggregationParam && idArray.length === 0) {
-                    callback(null, []);
+            else {
+                type = typeChar;
+            }
+        }
+        let retVal = hiddenFields;
+        if (projectParam) {
+            let projection = Object.keys(projectParam);
+            if (projection.length > 0) {
+                projection.forEach((p) => {
+                    if (projectParam[p] === 0) {
+                        setSelectType("E", "I");
+                    }
+                    else if (projectParam[p] === 1) {
+                        setSelectType("I", "E");
+                    }
+                    else {
+                        throw new Error("Invalid projection: " + projectParam);
+                    }
+                });
+                if (type && type === "E") {
+                    // We are excluding fields - can just merge with hiddenFields
+                    Object.assign(retVal, projectParam, hiddenFields);
                 }
                 else {
-                    var query = resource.model.find(queryObj);
-                    if (idArray.length > 0) {
-                        query = query.where('_id').in(idArray);
-                    }
-                    if (findParam) {
-                        query = query.find(findParam);
-                    }
-                    query = query.select(hiddenFields);
-                    if (limit) {
-                        query = query.limit(limit);
-                    }
-                    if (skip) {
-                        query = query.skip(skip);
-                    }
-                    if (sortOrder) {
-                        query = query.sort(sortOrder);
-                    }
-                    query.exec(function (err, docs) {
-                        if (!err && stashAggregationResults) {
-                            docs.forEach(function (obj) {
-                                // Add any fields from the aggregation results whose field name starts __ to the mongoose Document
-                                var aggObj = stashAggregationResults.find(function (a) { return a._id.toString() === obj._id.toString(); });
-                                Object.keys(aggObj).forEach(function (k) {
-                                    if (k.slice(0, 2) === '__') {
-                                        obj[k] = aggObj[k];
-                                    }
-                                });
-                            });
+                    // We are selecting fields - make sure none are hidden
+                    retVal = projectParam;
+                    for (let h in hiddenFields) {
+                        if (hiddenFields.hasOwnProperty(h)) {
+                            delete retVal[h];
                         }
-                        callback(err, docs);
-                    });
+                    }
                 }
-            });
-        }
-    });
-};
-DataForm.prototype.collectionPost = function () {
-    return _.bind(function (req, res, next) {
-        this.processCollection(req);
-        if (!req.resource) {
-            next();
-            return;
-        }
-        if (!req.body) {
-            throw new Error('Nothing submitted.');
+            }
         }
-        var cleansedBody = this.cleanseRequest(req);
-        req.doc = new req.resource.model(cleansedBody);
-        this.saveAndRespond(req, res);
-    }, this);
-};
-/**
- * Generate an object of fields to not expose
- **/
-DataForm.prototype.generateHiddenFields = function (resource, state) {
-    var hiddenFields = {};
-    if (resource.options['hide'] !== undefined) {
-        resource.options.hide.forEach(function (dt) {
-            hiddenFields[dt] = state;
-        });
+        return retVal;
     }
-    return hiddenFields;
-};
-/** Sec issue
- * Cleanse incoming data to avoid overwrite and POST request forgery
- * (name may seem weird but it was in French, so it is some small improvement!)
- */
-DataForm.prototype.cleanseRequest = function (req) {
-    var reqData = req.body, resource = req.resource;
-    delete reqData.__v; // Don't mess with Mongoose internal field (https://github.com/LearnBoost/mongoose/issues/1933)
-    if (typeof resource.options['hide'] === 'undefined') {
-        return reqData;
-    }
-    var hiddenFields = resource.options.hide;
-    _.each(reqData, function (num, key) {
-        _.each(hiddenFields, function (fi) {
-            if (fi === key) {
-                delete reqData[key];
-            }
-        });
-    });
-    return reqData;
-};
-DataForm.prototype.generateQueryForEntity = function (req, resource, id, cb) {
-    var hiddenFields = this.generateHiddenFields(resource, false);
-    hiddenFields.__v = 0;
-    this.doFindFunc(req, resource, function (err, queryObj) {
-        if (err) {
-            cb(err);
+    doFindFunc(req, resource, cb) {
+        // filter out records the user has no access to unless we are just asking for list attributes
+        if (resource.options.findFunc &&
+            req?.route?.path !== "/api/:resourceName/:id/list") {
+            resource.options.findFunc(req, cb);
         }
         else {
-            var idSel = { _id: id };
-            var crit = queryObj ? extend(queryObj, idSel) : idSel;
-            cb(null, resource.model.findOne(crit).select(hiddenFields));
+            cb(null);
         }
-    });
-};
-/*
- * Entity request goes there first
- * It retrieves the resource
- */
-DataForm.prototype.processEntity = function (req, res, next) {
-    if (!(req.resource = this.getResource(req.params.resourceName))) {
-        next();
-        return;
     }
-    this.generateQueryForEntity(req, req.resource, req.params.id, function (err, query) {
-        if (err) {
-            return res.send({
-                success: false,
-                err: util.inspect(err)
-            });
-        }
-        else {
-            query.exec(function (err, doc) {
+    async doFindFuncPromise(req, resource) {
+        return new Promise((resolve, reject) => {
+            this.doFindFunc(req, resource, (err, queryObj) => {
                 if (err) {
-                    return res.send({
-                        success: false,
-                        err: util.inspect(err)
-                    });
+                    reject(err);
                 }
-                else if (doc == null) {
-                    return res.send({
-                        success: false,
-                        err: 'Record not found'
-                    });
+                else {
+                    resolve(queryObj);
                 }
-                req.doc = doc;
-                next();
             });
-        }
-    });
-};
-/**
- * Gets a single entity
- *
- * @return {Function} The function to use as route
- */
-DataForm.prototype.entityGet = function () {
-    return _.bind(function (req, res, next) {
-        this.processEntity(req, res, function () {
-            if (!req.resource) {
-                return next();
-            }
-            if (req.resource.options.onAccess) {
-                req.resource.options.onAccess(req, function () {
-                    return res.send(req.doc);
+        });
+    }
+    async filteredFind(resource, req, aggregationParam, findParam, projectParam, sortOrder, limit, skip, callback) {
+        const that = this;
+        let hiddenFields = this.generateHiddenFields(resource, false);
+        let stashAggregationResults;
+        async function doAggregation(queryObj, cb) {
+            if (aggregationParam) {
+                aggregationParam = await that.sanitisePipeline(aggregationParam, hiddenFields, queryObj, req);
+                resource.model
+                    .aggregate(aggregationParam)
+                    .then((aggregationResults) => {
+                    stashAggregationResults = aggregationResults;
+                    cb(_.map(aggregationResults, function (obj) {
+                        return obj._id;
+                    }));
+                })
+                    .catch((err) => {
+                    throw err;
                 });
             }
             else {
-                return res.send(req.doc);
+                cb([]);
             }
-        });
-    }, this);
-};
-DataForm.prototype.replaceHiddenFields = function (record, data) {
-    var self = this;
-    if (record) {
-        record._replacingHiddenFields = true;
-        _.each(data, function (value, name) {
-            if (_.isObject(value)) {
-                self.replaceHiddenFields(record[name], value);
+        }
+        that.doFindFunc(req, resource, function (err, queryObj) {
+            if (err) {
+                callback(err);
             }
             else {
-                record[name] = value;
+                doAggregation(queryObj, function (idArray) {
+                    if (aggregationParam && idArray.length === 0) {
+                        callback(null, []);
+                    }
+                    else {
+                        let query = resource.model.find(queryObj);
+                        if (idArray.length > 0) {
+                            query = query.where("_id").in(idArray);
+                        }
+                        if (findParam) {
+                            query = query.find(findParam);
+                        }
+                        query = query.select(that.generateProjection(hiddenFields, projectParam));
+                        if (limit) {
+                            query = query.limit(limit);
+                        }
+                        if (skip) {
+                            query = query.skip(skip);
+                        }
+                        if (sortOrder) {
+                            query = query.sort(sortOrder);
+                        }
+                        query
+                            .exec()
+                            .then((docs) => {
+                            if (stashAggregationResults) {
+                                for (const obj of docs) {
+                                    // Add any fields from the aggregation results whose field name starts __ to the mongoose Document
+                                    let aggObj = stashAggregationResults.find((a) => a._id.toString() === obj._id.toString());
+                                    for (const k of Object.keys(aggObj).filter((k) => k.startsWith("__"))) {
+                                        obj[k] = aggObj[k];
+                                    }
+                                }
+                            }
+                            callback(null, docs);
+                        })
+                            .catch((err) => {
+                            callback(err);
+                        });
+                    }
+                });
             }
         });
-        delete record._replacingHiddenFields;
     }
-};
-DataForm.prototype.entityPut = function () {
-    return _.bind(function (req, res, next) {
-        var that = this;
-        this.processEntity(req, res, function () {
+    collectionPost() {
+        return _.bind(function (req, res, next) {
+            this.processCollection(req);
             if (!req.resource) {
                 next();
                 return;
             }
             if (!req.body) {
-                throw new Error('Nothing submitted.');
+                throw new Error("Nothing submitted.");
             }
-            var cleansedBody = that.cleanseRequest(req);
-            // Merge
-            _.each(cleansedBody, function (value, name) {
-                req.doc[name] = (value === '') ? undefined : value;
+            let cleansedBody = this.cleanseRequest(req);
+            req.doc = new req.resource.model(cleansedBody);
+            this.saveAndRespond(req, res);
+        }, this);
+    }
+    /**
+     * Generate an object of fields to not expose
+     **/
+    generateHiddenFields(resource, state) {
+        let hiddenFields = {};
+        if (resource.options["hide"] !== undefined) {
+            resource.options.hide.forEach(function (dt) {
+                hiddenFields[dt] = state;
             });
-            if (req.resource.options.hide !== undefined) {
-                var hiddenFields_1 = that.generateHiddenFields(req.resource, true);
-                hiddenFields_1._id = false;
-                req.resource.model.findById(req.doc._id, hiddenFields_1, { lean: true }, function (err, data) {
-                    that.replaceHiddenFields(req.doc, data);
-                    that.saveAndRespond(req, res, hiddenFields_1);
+        }
+        return hiddenFields;
+    }
+    /** Sec issue
+     * Cleanse incoming data to avoid overwrite and POST request forgery
+     * (name may seem weird but it was in French, so it is some small improvement!)
+     */
+    cleanseRequest(req) {
+        let reqData = req.body, resource = req.resource;
+        if (typeof resource.options["hide"] === "undefined") {
+            return reqData;
+        }
+        let hiddenFields = resource.options.hide;
+        _.each(reqData, function (num, key) {
+            _.each(hiddenFields, function (fi) {
+                if (fi === key) {
+                    delete reqData[key];
+                }
+            });
+        });
+        return reqData;
+    }
+    generateQueryForEntity(req, resource, id, cb) {
+        let that = this;
+        let hiddenFields = this.generateHiddenFields(resource, false);
+        that.doFindFunc(req, resource, function (err, queryObj) {
+            if (err) {
+                cb(err);
+            }
+            else {
+                const idSel = { _id: id };
+                let crit;
+                if (queryObj) {
+                    if (queryObj._id) {
+                        crit = { $and: [idSel, { _id: queryObj._id }] };
+                        delete queryObj._id;
+                        if (Object.keys(queryObj).length > 0) {
+                            crit = extend(crit, queryObj);
+                        }
+                    }
+                    else {
+                        crit = extend(queryObj, idSel);
+                    }
+                }
+                else {
+                    crit = idSel;
+                }
+                cb(null, resource.model
+                    .findOne(crit)
+                    .select(that.generateProjection(hiddenFields, req.query?.p)));
+            }
+        });
+    }
+    /*
+     * Entity request goes here first
+     * It retrieves the resource
+     */
+    processEntity(req, res, next) {
+        if (!(req.resource = this.getResource(req.params.resourceName))) {
+            return next();
+        }
+        this.generateQueryForEntity(req, req.resource, req.params.id, function (err, query) {
+            if (err) {
+                return res.status(500).send({
+                    success: false,
+                    err: util.inspect(err),
                 });
             }
             else {
-                that.saveAndRespond(req, res);
+                query
+                    .exec()
+                    .then((doc) => {
+                    if (doc) {
+                        req.doc = doc;
+                        return next();
+                    }
+                    else {
+                        return res.status(404).send({
+                            success: false,
+                            err: "Record not found",
+                        });
+                    }
+                })
+                    .catch((err) => {
+                    return res.status(400).send({
+                        success: false,
+                        err: util.inspect(err),
+                    });
+                });
             }
         });
-    }, this);
-};
-DataForm.prototype.entityDelete = function () {
-    return _.bind(function (req, res, next) {
-        function internalRemove(doc) {
-            doc.remove(function (err) {
-                if (err) {
-                    return res.send({ success: false });
+    }
+    /**
+     * Gets a single entity
+     *
+     * @return {Function} The function to use as route
+     */
+    entityGet() {
+        return _.bind(function (req, res, next) {
+            this.processEntity(req, res, function () {
+                if (!req.resource) {
+                    return next();
+                }
+                if (req.resource.options.onAccess) {
+                    req.resource.options.onAccess(req, function () {
+                        return res.status(200).send(req.doc);
+                    });
+                }
+                else {
+                    return res.status(200).send(req.doc);
                 }
-                return res.send({ success: true });
             });
+        }, this);
+    }
+    replaceHiddenFields(record, data) {
+        const self = this;
+        if (record) {
+            record._replacingHiddenFields = true;
+            _.each(data, function (value, name) {
+                if (_.isObject(value) && !Array.isArray(value)) {
+                    self.replaceHiddenFields(record[name], value);
+                }
+                else if (!record[name]) {
+                    record[name] = value;
+                }
+            });
+            delete record._replacingHiddenFields;
         }
-        this.processEntity(req, res, function () {
-            if (!req.resource) {
-                next();
-                return;
+    }
+    entityPut() {
+        return _.bind(function (req, res, next) {
+            const that = this;
+            this.processEntity(req, res, function () {
+                if (!req.resource) {
+                    next();
+                    return;
+                }
+                if (!req.body) {
+                    throw new Error("Nothing submitted.");
+                }
+                let cleansedBody = that.cleanseRequest(req);
+                // Merge
+                for (let prop in cleansedBody) {
+                    if (cleansedBody.hasOwnProperty(prop)) {
+                        req.doc.set(prop, cleansedBody[prop] === "" ? undefined : cleansedBody[prop]);
+                    }
+                }
+                if (req.resource.options.hide !== undefined) {
+                    let hiddenFields = that.generateHiddenFields(req.resource, true);
+                    hiddenFields._id = false;
+                    req.resource.model
+                        .findById(req.doc._id, hiddenFields)
+                        .lean()
+                        .exec()
+                        .then((data) => {
+                        that.replaceHiddenFields(req.doc, data);
+                        that.saveAndRespond(req, res, hiddenFields);
+                    })
+                        .catch((err) => {
+                        throw err; // not sure if this is the right thing to do - didn't have any error-handing here in earlier version
+                    });
+                }
+                else {
+                    that.saveAndRespond(req, res);
+                }
+            });
+        }, this);
+    }
+    generateDependencyList(resource) {
+        if (resource.options.dependents === undefined) {
+            let that = this;
+            resource.options.dependents = that.resources.reduce(function (acc, r) {
+                function searchPaths(schema, prefix) {
+                    let fldList = [];
+                    for (let fld in schema.paths) {
+                        if (schema.paths.hasOwnProperty(fld)) {
+                            const parts = fld.split(".");
+                            let schemaType = schema.tree;
+                            while (parts.length > 0) {
+                                schemaType = schemaType[parts.shift()];
+                            }
+                            if (schemaType.type) {
+                                if (schemaType.type.name === "ObjectId" &&
+                                    schemaType.ref === resource.resourceName) {
+                                    fldList.push(prefix + fld);
+                                }
+                                else if (_.isArray(schemaType.type)) {
+                                    schemaType.type.forEach(function (t) {
+                                        searchPaths(t, prefix + fld + ".");
+                                    });
+                                }
+                            }
+                        }
+                    }
+                    if (fldList.length > 0) {
+                        acc.push({ resource: r, keys: fldList });
+                    }
+                }
+                searchPaths(r.model.schema, "");
+                return acc;
+            }, []);
+            for (let pluginName in that.options.plugins) {
+                let thisPlugin = that.options.plugins[pluginName];
+                if (thisPlugin.dependencyChecks &&
+                    thisPlugin.dependencyChecks[resource.resourceName]) {
+                    resource.options.dependents = resource.options.dependents.concat(thisPlugin.dependencyChecks[resource.resourceName]);
+                }
             }
-            var doc = req.doc;
-            if (typeof req.resource.options.onRemove === 'function') {
-                req.resource.options.onRemove(doc, req, function (err) {
-                    if (err) {
-                        throw err;
+        }
+    }
+    async getDependencies(resource, id) {
+        this.generateDependencyList(resource);
+        let promises = [];
+        let foreignKeyList = [];
+        resource.options.dependents.forEach((collection) => {
+            collection.keys.forEach((key) => {
+                promises.push({
+                    p: collection.resource.model
+                        .find({ [key]: id })
+                        .limit(1)
+                        .exec(),
+                    collection,
+                    key,
+                });
+            });
+        });
+        return Promise.all(promises.map((p) => p.p)).then((results) => {
+            results.forEach((r, i) => {
+                if (r.length > 0) {
+                    foreignKeyList.push({
+                        resourceName: promises[i].collection.resource.resourceName,
+                        key: promises[i].key,
+                        id: r[0]._id,
+                    });
+                }
+            });
+            return foreignKeyList;
+        });
+    }
+    entityDelete() {
+        let that = this;
+        return _.bind(async function (req, res, next) {
+            async function removeDoc(doc, resource) {
+                switch (resource.options.handleRemove) {
+                    case "allow":
+                        // old behaviour - no attempt to maintain data integrity
+                        return doc.deleteOne();
+                    case "cascade":
+                        res.status(400).send('"cascade" option not yet supported');
+                        break;
+                    default:
+                        return that
+                            .getDependencies(resource, doc._id)
+                            .then((dependencies) => {
+                            if (dependencies.length > 0) {
+                                throw new ForeignKeyError(resource.resourceName, dependencies);
+                            }
+                            return doc.deleteOne();
+                        });
+                }
+            }
+            async function runDeletion(doc, resource) {
+                return new Promise((resolve) => {
+                    if (resource.options.onRemove) {
+                        resource.options.onRemove(doc, req, async function (err) {
+                            if (err) {
+                                throw err;
+                            }
+                            resolve(removeDoc(doc, resource));
+                        });
+                    }
+                    else {
+                        resolve(removeDoc(doc, resource));
                     }
-                    internalRemove(doc);
                 });
             }
+            this.processEntity(req, res, async function () {
+                if (!req.resource) {
+                    next();
+                    return;
+                }
+                let doc = req.doc;
+                try {
+                    void (await runDeletion(doc, req.resource));
+                    res.status(200).send();
+                }
+                catch (e) {
+                    if (e instanceof ForeignKeyError) {
+                        res.status(400).send(e.message);
+                    }
+                    else {
+                        res.status(500).send(e.message);
+                    }
+                }
+            });
+        }, this);
+    }
+    entityList() {
+        return _.bind(function (req, res, next) {
+            const that = this;
+            this.processEntity(req, res, function () {
+                if (!req.resource) {
+                    return next();
+                }
+                const returnRawParam = req.query?.returnRaw
+                    ? !!JSON.parse(req.query.returnRaw)
+                    : false;
+                if (returnRawParam) {
+                    const result = { _id: req.doc._id };
+                    for (const field of req.resource.options.listFields) {
+                        result[field.field] = req.doc[field.field];
+                    }
+                    return res.send(result);
+                }
+                else {
+                    that.getListFields(req.resource, req.doc, function (err, display) {
+                        if (err) {
+                            return res.status(500).send(err);
+                        }
+                        else {
+                            return res.send({ list: display });
+                        }
+                    });
+                }
+            });
+        }, this);
+    }
+    // To disambiguate the contents of items - assumed to be the results of a single resource lookup or search -
+    // pass the result of this function as the second argument to the disambiguate() function.
+    // equalityProps should identify the property(s) of the items that must ALL be equal for two items to
+    // be considered ambiguous.
+    // disambiguationResourceName should identify the resource whose list field(s) should be used to generate
+    // the disambiguation text for ambiguous results later.
+    buildSingleResourceAmbiguousRecordStore(items, equalityProps, disambiguationResourceName) {
+        const ambiguousResults = [];
+        for (let i = 0; i < items.length - 1; i++) {
+            for (let j = i + 1; j < items.length; j++) {
+                if (!equalityProps.some((p) => items[i][p] !== items[j][p])) {
+                    if (!ambiguousResults.includes(items[i])) {
+                        ambiguousResults.push(items[i]);
+                    }
+                    if (!ambiguousResults.includes(items[j])) {
+                        ambiguousResults.push(items[j]);
+                    }
+                }
+            }
+        }
+        return { [disambiguationResourceName]: ambiguousResults };
+    }
+    // An alternative to buildSingleResourceAmbiguousRecordStore() for use when disambiguating the results of a
+    // multi-resource lookup or search.  In this case, all items need to include the name of the resource that
+    // will be used (when necessary) to yield their disambiguation text later.  The property of items that holds
+    // that resource name should be identified by the disambiguationResourceNameProp parameter.
+    // The scary-looking templating used here ensures that the items really do all have an (optional) string
+    // property with the name identified by disambiguationResourceNameProp.  For the avoidance of doubt, "prop"
+    // here could be anything - "foo in f" would achieve the same result.
+    buildMultiResourceAmbiguousRecordStore(items, equalityProps, disambiguationResourceNameProp) {
+        const store = {};
+        for (let i = 0; i < items.length - 1; i++) {
+            for (let j = i + 1; j < items.length; j++) {
+                if (items[i][disambiguationResourceNameProp] &&
+                    items[i][disambiguationResourceNameProp] ===
+                        items[j][disambiguationResourceNameProp] &&
+                    !equalityProps.some((p) => items[i][p] !== items[j][p])) {
+                    if (!store[items[i][disambiguationResourceNameProp]]) {
+                        store[items[i][disambiguationResourceNameProp]] = [];
+                    }
+                    if (!store[items[i][disambiguationResourceNameProp]].includes(items[i])) {
+                        store[items[i][disambiguationResourceNameProp]].push(items[i]);
+                    }
+                    if (!store[items[i][disambiguationResourceNameProp]].includes(items[j])) {
+                        store[items[i][disambiguationResourceNameProp]].push(items[j]);
+                    }
+                }
+            }
+        }
+        return store;
+    }
+    // return just the id and list fields for all of the records from req.resource.
+    // list fields are those whose schema entry has a value for the "list" attribute (except where this is { ref: true })
+    // if the resource has no explicit list fields identified, the first string field will be returned.  if the resource
+    // doesn't have any string fields either, the first (non-id) field will be returned.
+    // usually, we will respond with an array of ILookupItem objects, where the .text property of those objects is the concatenation
+    // of all of the document's list fields (space-seperated).
+    // to request the documents without this transformation applied, include "c=true" in the query string.
+    // the query string can also be used to filter and order the response, by providing values for "f" (find), "l" (limit),
+    // "s" (skip) and/or "o" (order).
+    // results will be disambiguated if req.resource includes disambiguation parameters in its resource options.
+    // where c=true, the disambiguation will be added as a suffix to the .text property of the returned ILookupItem objects.
+    // otherwise, if the resource has just one list field, the disambiguation will be appended to the values of that field, and if
+    // it has multiple list fields, it will be returned as an additional property of the returned (untransformed) objects
+    internalEntityListAll(req, callback) {
+        const projection = this.generateListFieldProjection(req.resource);
+        const listFields = Object.keys(projection);
+        const aggregationParam = req.query.a ? JSON.parse(req.query.a) : null;
+        const findParam = req.query.f ? JSON.parse(req.query.f) : {};
+        const limitParam = req.query.l ? JSON.parse(req.query.l) : 0;
+        const skipParam = req.query.s ? JSON.parse(req.query.s) : 0;
+        const orderParam = req.query.o
+            ? JSON.parse(req.query.o)
+            : req.resource.options.listOrder;
+        const concatenateParam = req.query.c ? JSON.parse(req.query.c) : true;
+        const resOpts = req.resource.options;
+        let disambiguationField;
+        let disambiguationResourceName;
+        if (resOpts?.disambiguation) {
+            disambiguationField = resOpts.disambiguation.field;
+            if (disambiguationField) {
+                projection[disambiguationField] = 1;
+                disambiguationResourceName = resOpts.disambiguation.resource;
+            }
+        }
+        const that = this;
+        this.filteredFind(req.resource, req, aggregationParam, findParam, projection, orderParam, limitParam, skipParam, function (err, docs) {
+            if (err) {
+                return callback(err);
+            }
             else {
-                internalRemove(doc);
+                docs = docs.map((d) => d.toObject());
+                if (concatenateParam) {
+                    const transformed = docs.map((doc) => {
+                        let text = "";
+                        for (const field of listFields) {
+                            if (doc[field]) {
+                                if (text !== "") {
+                                    text += " ";
+                                }
+                                text += doc[field];
+                            }
+                        }
+                        const disambiguationId = disambiguationField
+                            ? doc[disambiguationField]
+                            : undefined;
+                        return { id: doc._id, text, disambiguationId };
+                    });
+                    if (disambiguationResourceName) {
+                        that.disambiguate(transformed, that.buildSingleResourceAmbiguousRecordStore(transformed, ["text"], disambiguationResourceName), "disambiguationId", (item, disambiguationText) => {
+                            item.text += ` (${disambiguationText})`;
+                        }, (err) => {
+                            callback(err, transformed);
+                        });
+                    }
+                    else {
+                        return callback(null, transformed);
+                    }
+                }
+                else {
+                    if (disambiguationResourceName) {
+                        that.disambiguate(docs, that.buildSingleResourceAmbiguousRecordStore(docs, listFields, disambiguationResourceName), disambiguationField, (item, disambiguationText) => {
+                            if (listFields.length === 1) {
+                                item[listFields[0]] += ` (${disambiguationText})`;
+                            }
+                            else {
+                                // store the text against hard-coded property name "disambiguation", rather than (say) using
+                                // item[disambiguationResourceName], because if disambiguationResourceName === disambiguationField,
+                                // that value would end up being deleted again when the this.disambiguate() call (which we have
+                                // been called from) does its final tidy-up and deletes [disambiguationField] from all of the items in docs
+                                item.disambiguation = disambiguationText;
+                            }
+                        }, (err) => {
+                            callback(err, docs);
+                        });
+                    }
+                    else {
+                        return callback(null, docs);
+                    }
+                }
             }
         });
-    }, this);
-};
-DataForm.prototype.entityList = function () {
-    return _.bind(function (req, res, next) {
-        var that = this;
-        this.processEntity(req, res, function () {
-            if (!req.resource) {
+    }
+    entityListAll() {
+        return _.bind(function (req, res, next) {
+            if (!(req.resource = this.getResource(req.params.resourceName))) {
                 return next();
             }
-            that.getListFields(req.resource, req.doc, function (err, display) {
+            this.internalEntityListAll(req, function (err, resultsObject) {
                 if (err) {
-                    return res.status(500).send(err);
+                    res.status(400, err);
                 }
                 else {
-                    return res.send({ list: display });
+                    res.send(resultsObject);
                 }
             });
-        });
-    }, this);
-};
+        }, this);
+    }
+    extractTimestampFromMongoID(record) {
+        let timestamp = record.toString().substring(0, 8);
+        return new Date(parseInt(timestamp, 16) * 1000);
+    }
+}
+class ForeignKeyError extends global.Error {
+    constructor(resourceName, foreignKeys) {
+        super(`Cannot delete this ${resourceName}, as it is: ${foreignKeys.map((d) => ` the ${d.key} on ${d.resourceName} ${d.id}`).join("; ")}`);
+        this.name = "ForeignKeyError";
+        this.stack = new global.Error("").stack;
+    }
+}
+export default FormsAngular;