grnsight 4.0.0 → 6.0.0

package/server/controllers/additional-sheet-parser.js

@@ -1,87 +1,324 @@
-// Parses "optimization_paramters," expression data sheets, and 2-column sheets
+// Parses "optimization_paramters" and 2-column sheets
 // from GRNmap input or output workbook
 
-var TWO_COL_SHEET_NAMES = [
+var constants = require(__dirname + "/workbook-constants");
+
+const getSheetHeader = (sheetName, column, row) => {
+    if (row === 0) {
+        if (sheetName === "production_rates" || sheetName === "optimized_production_rates") {
+            return "production_rate";
+        } else if (sheetName === "degradation_rates") {
+            return "degradation_rate";
+        } else if (sheetName === "threshold_b" || sheetName === "optimized_threshold_b") {
+            return "threshold_b";
+        } else if (sheetName === "optimization_parameters") {
+            return (column === 0) ? "optimization_parameter" : "value";
+        } else if (sheetName === "optimization_diagnostics") {
+            return (column === 0) ? "Parameter" : "Value";
+        }
+    }
+};
+
+const optimizationParametersTypeKey = {
+    alpha: "number", "kk_max": "number", MaxIter: "number", TolFun: "number", MaxFunEval: "number",
+    TolX: "number", "production_function": "string", "L_curve": "number", "estimate_params": "number",
+    "make_graphs": "number", "fix_P": "number", "fix_b": "number", "expression_timepoints": "object",
+    Strain: "object", species: "string", "taxon_id": "number", "simulation_timepoints": "object", "b_or_tau" : "number"
+
+};
+
+const optimizationDiagnosticsParameters = ["LSE", "Penalty", "min LSE", "iteration count"];
+
+const optimizationParametersObectKey = {
+    "expression_timepoints": "number", Strain: "string", "simulation_timepoints": "number"
+};
+
+const addWarning = (workbook, message) => {
+    let warningsCount;
+    if (!Object.keys(workbook).includes("warnings")) {
+        warningsCount = 0;
+        workbook.warnings = [];
+    } else {
+        warningsCount = workbook.warnings.length;
+    }
+    const MAX_WARNINGS = 75;
+    if (warningsCount < MAX_WARNINGS) {
+        workbook.warnings.push(message);
+    } else {
+        workbook.errors.push(constants.errors.warningsCountError);
+        return false;
+    }
+};
+
+const addError = (output, message) => {
+    const errorsCount = output.errors.length;
+    const MAX_ERRORS = 20;
+    if (errorsCount < MAX_ERRORS) {
+        output.errors.push(message);
+    } else {
+        output.errors.push(constants.errors.errorsCountError);
+        return false;
+    }
+};
+
+const TWO_COL_SHEET_NAMES = [
     "production_rates",
     "degradation_rates",
     "threshold_b",
     "optimized_production_rates",
-    "optimized_threshold_b"];
-
-var EXPRESSION_SHEET_SUFFIXES = ["_expression", "_optimized_expression", "_sigmas"];
-
-var isExpressionSheet = function (sheetName) {
-    return EXPRESSION_SHEET_SUFFIXES.some(function (suffix) {
-        return sheetName.includes(suffix);
-    });
-};
+    "optimized_threshold_b"
+];
 
-var fillArray = function (value, array, length) { // mutator
-    while (array.length < length) {
-        array.push(value);
+const validGeneName = (output, sheetName, gene, row) => {
+    var maxGeneLength = 12;
+    var regex = /[^a-z0-9\_\-]/gi;
+    if (typeof gene !== "string") {
+        addError(output, constants.errors.invalidGeneTypeError(sheetName, gene, row));
+        return false;
+    } else if (gene.length > maxGeneLength) {
+        addError(output, constants.errors.invalidGeneLengthError(sheetName, gene, row));
+        return false;
+    } else if (gene.match(regex) !== null) {
+        addError(output, constants.errors.specialCharacterError(sheetName, gene, row));
+        return false;
     }
-    return array;
+    return true;
 };
+// Optimization Parameters Parser
+const parseMetaDataSheet = (sheet) => {
+    let meta = {
+        data: {},
+        errors: [],
+        warnings: []
+    };
+    let paramType;
+    if (sheet.data[0][0] === undefined) {
+        addError(meta, constants.errors.missingColumnHeaderError(sheet.name, constants.numbersToLetters[0],
+            getSheetHeader(sheet.name, 0, 0)));
+    } else if (sheet.data[0][0] !== getSheetHeader(sheet.name, 0, 0)) {
+        addError(meta, constants.errors.incorrectColumnHeaderError(sheet.name, constants.numbersToLetters[0],
+            getSheetHeader(sheet.name, 0, 0)));
+    }
+    if (sheet.data[0][1] === undefined) {
+        addError(meta, constants.errors.missingColumnHeaderError(sheet.name, constants.numbersToLetters[1],
+            getSheetHeader(sheet.name, 1, 0)));
+    } else if (sheet.data[0][1] !== getSheetHeader(sheet.name, 1, 0)) {
+        addError(meta, constants.errors.incorrectColumnHeaderError(sheet.name, constants.numbersToLetters[1],
+            getSheetHeader(sheet.name, 1, 0)));
+    }
 
-var parseMetaDataSheet = function (sheet) {
-    var meta = {};
     sheet.data.forEach(function (element, index) {
         if (index !== 0) {
-            var value = element.slice(1);
+            const value = element.slice(1);
             // Extract element from array if array contains only 1 value
-            meta[element[0]] = value.length > 1 ? value : value[0];
+            meta.data[element[0]] = value.length > 1 ? value : value[0];
         }
     });
+    for (let key in meta.data) {
+        paramType = optimizationParametersTypeKey[key];
+        if (paramType === "object") {
+            paramType = `list of ${optimizationParametersObectKey[key]}s`;
+        }
+        if (meta.data[key] === undefined) {
+            addWarning(meta, constants.warnings.unknownOptimizationParameter(sheet.name, key));
+        } else if (typeof meta.data[key] !== optimizationParametersTypeKey[key]) {
+            if (optimizationParametersTypeKey[key] !== "object" ||
+                typeof meta.data[key] !== optimizationParametersObectKey[key]) {
+                addWarning(meta, constants.warnings.invalidOptimizationParameter(sheet.name, key, paramType));
+            }
+        } else if (optimizationParametersTypeKey[key] === "object") {
+            for (let val of meta.data[key]) {
+                if (typeof val !== optimizationParametersObectKey[key]) {
+                    // throw error once per object. Makes sure that errors list is not flooded
+                    addWarning(meta, constants.warnings.invalidOptimizationParameter(sheet.name, key, paramType));
+                    break;
+                }
+            }
+        }
+    }
     return meta;
 };
 
-var parseTwoColumnSheet = function (sheet) {
-    var data = {};
-    sheet.data.forEach(function (element, index) {
-        if (index !== 0) {
-            data[element[0]] = element[1];
+const parseOptimizationDiagnosticsSheet = (sheet) => {
+    let output = {
+        data: {
+            Parameters: {},
+            MSE: {
+                "column-headers": [],
+                Genes: {}
+            },
+        },
+        errors: [],
+        warnings: [],
+    };
+    let currentParameter;
+    let currentValue;
+    let currentGene;
+    let currentMSE = [];
+    // Check Headers
+    if (sheet.data[0].length > 1) {
+        if (sheet.data[0][0] !== getSheetHeader(sheet.name, 0, 0)) {
+            addError(output, constants.errors.incorrectColumnHeaderError(sheet.name, constants.numbersToLetters[0],
+                getSheetHeader(sheet.name, 0, 0)));
         }
-    });
-    return data;
+        if (sheet.data[0][1] !== getSheetHeader(sheet.name, 1, 0)) {
+            addError(output, constants.errors.incorrectColumnHeaderError(sheet.name, constants.numbersToLetters[1],
+                getSheetHeader(sheet.name, 1, 0)));
+        }
+    } else {
+        // seems a bit sus, but we'll see if this works properly during testing :\
+        for (let col = 1; col >= sheet.data[0].length; col--) {
+            addError(output, constants.errors.missingColumnHeaderError(sheet.name, constants.numbersToLetters[col],
+                getSheetHeader(sheet.name, col, 0)));
+        }
+    }
+    // Check Parameter Section
+    let row = 1;
+    // a missing row is the indicator to move onto the MSE
+    while (sheet.data[row].length > 0) {
+        currentParameter = sheet.data[row][0];
+        currentValue = sheet.data[row][1];
+        if (currentParameter === undefined || currentParameter.replace(/\s+/g, "") === "") {
+            if (currentValue === undefined || currentValue.replace(/\s+/g, "") === "") {
+                // if there is no parameter or value assume that its time to move on
+                row++;
+                break;
+            }
+        }
+        if (sheet.data[row].length > 2) {
+            addWarning(output, constants.warnings.extraneousDataWarning(sheet.name, row + 1));
+        }
+        if (! optimizationDiagnosticsParameters.includes(currentParameter)) {
+            if (currentParameter === "Gene") {
+                row--;
+                break;
+            }
+            addWarning(output, constants.warnings.unknownOptimizationDiagnosticsParameter(sheet.name,
+                currentParameter));
+        } else if (typeof currentValue !== "number") {
+            addWarning(output, constants.warnings.invalidOptimizationDiagnosticsValue(sheet.name, currentParameter));
+        } else {
+            output.data.Parameters[currentParameter] = currentValue;
+        }
+        row++;
+    }
+    // Skip until Gene section
+    while (sheet.data[row] !== undefined && sheet.data[row].length < 1) {
+        row++;
+    }
+    // Check Gene section MSE's
+    if (sheet.data[row].length > 1) {
+        if (sheet.data[row][0] !== "Gene") {
+            addWarning(output, constants.warnings.incorrectMSEGeneHeaderWarning(sheet.name, row + 1));
+        }
+        for (let col = 1; col < sheet.data[row].length; col++) {
+            if (!sheet.data[row][col].includes("MSE")) {
+                addWarning(output, constants.warnings.incorrectMSEHeaderWarning(sheet.name, sheet.data[row][col],
+                    row + 1, constants.numbersToLetters[col]));
+            }
+            // we still push the header (even tho it's sus) because the gene MSE's are
+            // dependent on the order of the column headers
+            output.data.MSE["column-headers"].push(sheet.data[row][col]);
+        }
+        row++;
+        // on to the actual genes
+        while (row < sheet.data.length) {
+            if (sheet.data[row].length > output.data.MSE["column-headers"].length + 1) {
+                addWarning(output, constants.warnings.extraneousDataWarning(sheet.name, row + 1));
+            }
+            currentGene = sheet.data[row][0];
+            // if it's a valid gene set the key = MSE value
+            if (validGeneName(output, sheet.name, currentGene, row)) {
+                for (let col = 1; col <= output.data.MSE["column-headers"].length; col++) {
+                    if (typeof sheet.data[row][col] === "number") {
+                        currentMSE.push(sheet.data[row][col]);
+                    } else if (sheet.data[row][col] === undefined) {
+                        addWarning(output, constants.warnings.missingMSEDataWarning(sheet.name, row + 1,
+                            constants.numbersToLetters[col]));
+                    } else {
+                        addWarning(output, constants.warnings.invalidMSEDataWarning(sheet.name, row + 1,
+                            constants.numbersToLetters[col]));
+                    }
+                }
+                output.data.MSE.Genes[currentGene] = currentMSE;
+                currentMSE = [];
+            }
+            row++;
+        }
+    }
+    return output;
 };
 
-var parseExpressionSheet = function (sheet) {
-    var expressionData = {};
-    var geneData = {};
-    expressionData["time_points"] = sheet.data[0].slice(1);
-    var numberOfDataPoints = expressionData["time_points"].length;
-    sheet.data.forEach(function (sheet) {
-        var geneName = sheet[0];
-        if (geneName) {
-            var rowData = sheet.slice(1);
-            // Sometimes, missing data is at the end of the row. In this case, pad the
-            // array with nulls
-            if (rowData.length < numberOfDataPoints) {
-                fillArray(null, rowData, numberOfDataPoints);
+const parseTwoColumnSheet = (sheet) => {
+    let output = {
+        data : {},
+        errors: [],
+        warnings: [],
+    };
+
+    let currentGene;
+    let currentValue;
+
+    // check to see if the genes are strings and the values are numbers
+
+    for (let row = 0; row < sheet.data.length; row++) {
+        if (sheet.data[row].length > 2) {
+            addWarning(output, constants.warnings.extraneousDataWarning(sheet.name, row + 1));
+        }
+        if (row === 0) {
+            if (sheet.data[row].length > 0) {
+                if (sheet.data[row][0] !== "id") {
+                    addError(output, constants.errors.idLabelError(sheet.name));
+                }
+            }
+            if (sheet.data[row].length > 1) {
+                if (sheet.data[row][1] !== getSheetHeader(sheet.name, 1, row)) {
+                    addError(output, constants.errors.incorrectColumnHeaderError(sheet.name,
+                        constants.numbersToLetters[1],
+                        getSheetHeader(sheet.name, 1, row)));
+                }
+            } else {
+                addError(output, constants.errors.missingColumnHeaderError(sheet.name, constants.numbersToLetters[1],
+                    getSheetHeader(sheet.name, 1, row)));
+            }
+        } else {
+            currentGene = sheet.data[row][0];
+            currentValue = sheet.data[row][1];
+
+            if (validGeneName(output, sheet.name, currentGene, row + 1)) {
+                if (typeof currentValue === "number") {
+                    output.data[currentGene] = currentValue;
+                } else {
+                    addError(output, constants.errors.invalidValueError(sheet.name, currentValue, row + 1,
+                        getSheetHeader(sheet.name, 1, row)));
+                }
             }
-            geneData[geneName] = rowData;
         }
-    });
-    expressionData["data"] = geneData;
-    return expressionData;
+    }
+
+    return output;
 };
 
-module.exports = function (workbook) {
-    var output = {
-        expression: {}, // expression data
-        meta: {},
-        test: {} // 2-column data
+module.exports = function (workbookFile) {
+    let output = {
+        meta: {
+            data: {},
+            errors: [],
+            warnings: []
+        }, // optimization_parameters only
+        test: {}, // 2-column data
+        meta2: {} // optimation_diagnostics only //temporary until where it goes is decided
     };
-    workbook.forEach(function (sheet) {
+    workbookFile.forEach(function (sheet) {
         if (sheet.name === "optimization_parameters") {
-            output["meta"] = parseMetaDataSheet(sheet);
-        // Parse 2-column sheets
+            output.meta = parseMetaDataSheet(sheet);
+            // above line creates an object from the optimization paramerters sheet
+            // these are part of the "meta" property
         } else if (TWO_COL_SHEET_NAMES.includes(sheet.name)) {
             output["test"][sheet.name] = parseTwoColumnSheet(sheet);
-        // Parse expression sheets
-        } else if (isExpressionSheet(sheet.name)) {
-            output["expression"][sheet.name] = parseExpressionSheet(sheet);
+        } else if (sheet.name === "optimization_diagnostics") {
+            output.meta2 = parseOptimizationDiagnosticsSheet(sheet);
         }
     });
     return output;
-};
+};

package/server/controllers/constants.js

@@ -11,48 +11,15 @@ module.exports = {
         EDGES_WITHOUT_WEIGHTS: {
             warningCode: "EDGES_WITHOUT_WEIGHTS",
             errorDescription: "GRNsight has detected that one or more edges in your network are missing numerical" +
-            " weight values. Because the algorithm GRNsight uses for determining the arrowhead type and the color and" +
-            " thickness of the edges requires numerical weight values, your graph will display as an unweighted graph" +
-            " with black edges and pointed arrowheads. If you want to display the network as a weighted graph, please" +
-            " modify your input file to include weight values for all edges."
+                " weight values. Because the algorithm GRNsight uses for determining the arrowhead type and the" +
+                " color and  thickness of the edges requires numerical weight values, your graph will display as" +
+                " an unweighted graph with black edges and pointed arrowheads. If you want to display the network" +
+                " as a weighted graph, please modify your input file to include weight values for all edges."
         },
 
         EDGE_DEFAULT_NOT_DIRECTED: {
             warningCode: "EDGE_DEFAULT_NOT_DIRECTED",
             errorDescription: "GRNsight interprets the graph as directed unconditionally."
         },
-
-        SIF_FORMAT_WARNING: {
-            warningCode: "SIF_FORMAT_WARNING",
-            errorDescription: " GRNsight has detected that there are no tabs in your file. The GRNsight specification" +
-            " for SIF files states that data must be delimited by tabs. Please review your data. This warning may" +
-            " suggest that your SIF file has comma separated data or contains no data at all. Additionally, valid" +
-            " networks which consist of single source nodes may also trigger this warning."
-        }
     },
-
-    errors: {
-        SIF_UNWEIGHTED_RELATIONSHIP_TYPE_ERROR: {
-            errorCode: "SIF_UNWEIGHTED_RELATIONSHIP_TYPE_ERROR",
-            possibleCause: "The SIF importer detects an unweighted network with an unsupported relationship type.",
-            suggestedFix: "SIF files accepted by GRNsight must use 'pd' as the text string for the relationship type" +
-            " in unweighted networks. Please review the SIF input documentation. Additionally, this error may be have" +
-            " been caused by missing data in your file, which caused the importer to incorrectly interpret a source" +
-            " or target as the relationship."
-        },
-
-        SIF_MISSING_DATA_ERROR: {
-            errorCode: "SIF_MISSING_DATA_ERROR",
-            possibleCause: "GRNsight has detected that your SIF file contains missing data. ",
-            suggestedFix: "Please review the data. In a SIF file, each entry must have a source node, relationship" +
-            " type, and at least one target node, separated by tabs. An entry with a single source node is also" +
-            " allowed."
-        },
-
-        SIF_STRAY_DATA_ERROR: {
-            errorCode: "SIF_STRAY_DATA_ERROR",
-            possibleCause: "GRNsight has detected stray data and/or extraneous blank rows in your SIF file. ",
-            suggestedFix: "Please review the data and delete extraneous data from the file."
-        }
-    }
 };

package/server/controllers/custom-workbook-controller.js

@@ -0,0 +1,66 @@
+/* eslint-disable camelcase */
+var helpers = require(__dirname + "/helpers");
+
+var processCustomWorkbook = function (path, res, app, workbook) {
+    helpers.attachCorsHeader(res, app);
+    helpers.attachFileHeaders(res, path);
+    return workbook.errors.length === 0
+        ? // If all looks well, return the workbook with an all clear
+        res.json(workbook)
+        : // If all does not look well, return the workbook with an error 400
+        res.status(400).json(workbook);
+};
+
+const createCustomWorkbook = (genesString, linksString) => {
+    let genes = genesString.split(",").map(gene => {
+        return {name: gene};
+    });
+    let links = linksString.split(",").map( link => {
+        link = link.split("->");
+        return {
+            source: parseInt(link[0]),
+            target: parseInt(link[1]),
+            value:1,
+            type:"arrowhead",
+            stroke: "black"
+        };
+    });
+    let positiveWeights = Array(links.length).fill(1);
+    return  {
+        genes,
+        links,
+        errors: [],
+        warnings: [],
+        positiveWeights,
+        negativeWeights: [],
+        sheetType: "unweighted",
+        network: {
+            genes,
+            links,
+            errors: [],
+            warnings: [],
+            positiveWeights,
+        },
+        meta: {
+            data: {
+                species: "Saccharomyces cerevisiae",
+                taxon_id: 559292
+            }
+        },
+        test: {
+        },
+        expression: {
+        }
+    };
+};
+
+module.exports = function (app) {
+    if (app) {
+
+        // Load Custom Workbook
+        app.get("/upload-custom-workbook", function (req, res) {
+            let workbook = createCustomWorkbook(req.query.genes, req.query.links);
+            return processCustomWorkbook(req.query.name, res, app, workbook);
+        });
+    }
+};