grnsight 5.1.0 → 6.0.4

package/gdb/postmaster.opts

@@ -0,0 +1 @@
+/usr/local/Cellar/postgresql@14/14.5_3/bin/postgres "-D" "gdb"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "grnsight",
-  "version": "5.1.0",
+  "version": "6.0.4",
   "description": "Web app and service for visualizing models of gene regulatory networks",
   "directories": {
     "test": "test"
@@ -11,7 +11,7 @@
     "cors": "2.8.1",
     "cytoscape": "2.7.14",
     "d3-v4-grid": "2.0.1",
-    "dotenv": "^8.2.0",
+    "dotenv": "8.2.0",
     "errorhandler": "1.5.0",
     "express": "4.16.0",
     "fs": "0.0.1-security",
@@ -25,31 +25,31 @@
     "node-xlsx": "0.7.2",
     "nodemon": "1.19.1",
     "parallelshell": "3.0.1",
-    "pg": "^8.0.0",
-    "pg-hstore": "^2.3.3",
-    "pug": "^3.0.2",
-    "querystring": "^0.2.0",
+    "pg": "8.0.0",
+    "pg-hstore": "2.3.3",
+    "pug": "3.0.2",
+    "querystring": "0.2.0",
     "save-svg-as-png": "1.4.14",
-    "sequelize": "^5.21.6",
+    "sequelize": "5.21.6",
     "serve-static": "1.13.1",
     "should": "11.2.0",
-    "stylus": "^0.54.5",
+    "stylus": "0.54.5",
     "supertest": "3.0.0",
-    "url": "^0.11.0",
+    "url": "0.11.0",
     "webpack": "4.0.0",
     "xml2js": "0.4.17",
     "xmlbuilder": "8.2.2"
   },
   "devDependencies": {
-    "browser-env": "^3.3.0",
+    "browser-env": "3.3.0",
     "chai": "4.1.2",
     "coveralls": "2.13.1",
     "eslint": "3.19.0",
     "isomorphic-fetch": "2.2.1",
     "istanbul": "0.4.5",
-    "jquery": "^3.6.0",
+    "jquery": "3.6.0",
     "jquery-extend": "2.0.3",
-    "jsdom": "11.11.0",
+    "jsdom": "13.2.0",
     "markdown-pdf": "8.0.0",
     "mocha": "2.5.3",
     "mocha-lcov-reporter": "1.3.0",

package/server/app.js

@@ -33,7 +33,9 @@ require(__dirname + "/controllers/export-controller")(app);
 require(__dirname + "/controllers/import-controller")(app);
 require(__dirname + "/controllers/ga-controller")(app);
 require(__dirname + "/controllers/api-controllers")(app);
-require(__dirname + "/controllers/database-controller")(app);
+require(__dirname + "/controllers/expression-database-controller")(app);
+require(__dirname + "/controllers/network-database-controller")(app);
+require(__dirname + "/controllers/custom-workbook-controller")(app);
 
 // Don"t start the server if this app is run as a child process.
 if (!module.parent) {

package/server/controllers/additional-sheet-parser.js

@@ -306,7 +306,7 @@ module.exports = function (workbookFile) {
             errors: [],
             warnings: []
         }, // optimization_parameters only
-        test: {}, // 2-column data
+        twoColumnSheets: {}, // 2-column data
         meta2: {} // optimation_diagnostics only //temporary until where it goes is decided
     };
     workbookFile.forEach(function (sheet) {
@@ -315,7 +315,7 @@ module.exports = function (workbookFile) {
             // above line creates an object from the optimization paramerters sheet
             // these are part of the "meta" property
         } else if (TWO_COL_SHEET_NAMES.includes(sheet.name)) {
-            output["test"][sheet.name] = parseTwoColumnSheet(sheet);
+            output.twoColumnSheets[sheet.name] = parseTwoColumnSheet(sheet);
         } else if (sheet.name === "optimization_diagnostics") {
             output.meta2 = parseOptimizationDiagnosticsSheet(sheet);
         }

package/server/controllers/custom-workbook-controller.js

@@ -0,0 +1,67 @@
+/* eslint-disable camelcase */
+var helpers = require(__dirname + "/helpers");
+
+var processCustomWorkbook = function (path, res, app, workbook) {
+    helpers.attachCorsHeader(res, app);
+    helpers.attachFileHeaders(res, path);
+    return workbook.errors.length === 0
+        ? // If all looks well, return the workbook with an all clear
+        res.json(workbook)
+        : // If all does not look well, return the workbook with an error 400
+        res.status(400).json(workbook);
+};
+
+const createCustomWorkbook = (genesString, linksString) => {
+    let genes = genesString.split(",").map(gene => {
+        return {name: gene};
+    });
+    let links = linksString === "" ? [] : linksString.split(",").map( link => {
+        link = link.split("->");
+        return {
+            source: parseInt(link[0]),
+            target: parseInt(link[1]),
+            value:1,
+            type:"arrowhead",
+            stroke: "black"
+        };
+    });
+    let positiveWeights = Array(links.length).fill(1);
+    return  {
+        genes,
+        links,
+        errors: [],
+        warnings: [],
+        positiveWeights,
+        negativeWeights: [],
+        sheetType: "unweighted",
+        network: {
+            genes,
+            links,
+            errors: [],
+            warnings: [],
+            positiveWeights,
+        },
+        meta: {
+            data: {
+                species: "Saccharomyces cerevisiae",
+                taxon_id: 559292
+            }
+        },
+        meta2: {},
+        twoColumnSheets: {
+        },
+        expression: {
+        }
+    };
+};
+
+module.exports = function (app) {
+    if (app) {
+
+        // Load Custom Workbook
+        app.get("/upload-custom-workbook", function (req, res) {
+            let workbook = createCustomWorkbook(req.query.genes, req.query.links);
+            return processCustomWorkbook(req.query.name, res, app, workbook);
+        });
+    }
+};

package/server/controllers/demo-workbooks.js

@@ -532,7 +532,7 @@ var demoWorkbook1 = function (path, res, app) {
                 taxon_id: 559292
             }
         },
-        test: {
+        twoColumnSheets: {
             production_rates: {
                 data: {
                     ACE2: 0.2236,
@@ -2695,7 +2695,7 @@ var demoWorkbook2 = function (path, res, app) {
                 taxon_id: 559292
             }
         },
-        test: {
+        twoColumnSheets: {
             production_rates: {
                 data: {
                     ACE2: 0.2236,
@@ -4622,7 +4622,7 @@ var demoWorkbook3 = function (path, res, app) {
                 taxon_id: 559292
             }
         },
-        test: {},
+        twoColumnSheets: {},
         expression: {
             wt_log2_expression: {
                 errors: [],
@@ -5683,7 +5683,7 @@ var demoWorkbook4 = function (path, res, app) {
                 taxon_id: 559292
             }
         },
-        test: {},
+        twoColumnSheets: {},
         expression: {
             wt_log2_expression: {
                 errors: [],

package/server/controllers/exporters/xlsx.js

@@ -87,94 +87,71 @@ const isExpressionSheet = (sheetName) => {
 const buildExpressionSheets = function (expressions) {
     const builtExpressionSheets = [];
     Object.keys(expressions).forEach((expression) => {
-        let expressionName = expression;
-        if (!isExpressionSheet(expression)) {
-            expressionName = expression + "_expression";
+        if (expressions[expression] !== null) {
+            let expressionName = expression;
+            if (!isExpressionSheet(expression)) {
+                expressionName = expression + "_expression";
+            }
+            const builtSheet = { name: expressionName, data: [] };
+            Object.keys(expressions[expression]["data"]).forEach((key) => {
+                const expressionData = expressions[expression]["data"][key];
+                builtSheet["data"].push([key, ...expressionData]);
+            });
+            builtExpressionSheets.push(builtSheet);
         }
-        const builtSheet = { name: expressionName, data: [] };
-        Object.keys(expressions[expression]["data"]).forEach((key) => {
-            const expressionData = expressions[expression]["data"][key];
-            builtSheet["data"].push([key, ...expressionData]);
-        });
-        builtExpressionSheets.push(builtSheet);
     });
     return builtExpressionSheets;
 };
 
 const buildXlsxSheet = function (workbook) {
     const resultSheet = [];
-    const exportNetworkType = workbook.exportNetworkType;
 
-    Object.keys(workbook).forEach((key) => {
-        switch (key) {
-        case "network":
-            if (Object.keys(workbook.network).length > 0) {
-                resultSheet.push(
-                    {
-                        "name": "network",
-                        "data": buildNetworkSheet(workbook.network.genes, workbook.network.links)
-                    }
-                );
-            }
-            break;
-        case "networkOptimizedWeights":
-            if (exportNetworkType === "weighted") {
-                if (Object.keys(workbook.networkOptimizedWeights).length > 0) {
+    Object.keys(workbook.exportSheets).forEach((type) => {
+        switch (type) {
+        case "networks":
+            for (let network in workbook.exportSheets.networks) {
+                if (Object.keys(workbook.exportSheets.networks[network]).length > 0) {
                     resultSheet.push(
                         {
-                            "name": "network_optimized_weights",
-                            "data": buildNetworkSheet(workbook.networkOptimizedWeights.genes,
-                                workbook.networkOptimizedWeights.links)
+                            "name": network,
+                            "data": buildNetworkSheet(workbook.exportSheets.networks[network].genes,
+                                workbook.exportSheets.networks[network].links)
                         }
                     );
                 }
             }
             break;
-        case "networkWeights":
-            if (Object.keys(workbook.networkWeights).length > 0) {
+        case "optimization_parameters":
+            if (workbook.exportSheets[type] !== null && Object.keys(workbook.exportSheets[type]).length > 0) {
                 resultSheet.push(
                     {
-                        "name": "network_weights",
-                        "data": buildNetworkSheet(workbook.networkWeights.genes, workbook.networkWeights.links)
+                        "name": type,
+                        "data": buildMetaSheet(workbook.exportSheets[type])
                     }
                 );
             }
             break;
-        case "meta":
-            if (Object.keys(workbook.meta).length > 0) {
-                resultSheet.push(
-                    {
-                        "name": "optimization_parameters",
-                        "data": buildMetaSheet(workbook.meta)
-                    }
-                );
-            }
-            break;
-        case "meta2":
+        case "optimization_diagnostics":
             // Optimization Diagnostics sheet not properly  implemented yet.
-            if (Object.keys(workbook.meta2).length > 0) {
+            if (Object.keys(workbook.exportSheets[type]).length > 0) {
                 resultSheet.push(
                     {
                         "name": "optimization_diagnostics",
-                        "data": buildMeta2Sheet(workbook.meta2)
+                        "data": buildMeta2Sheet(workbook.exportSheets[type])
                     }
                 );
             }
             break;
-        case "test":
-            resultSheet.push(...buildTestSheets(workbook[key]));
+        case "two_column_sheets":
+            resultSheet.push(...buildTestSheets(workbook.exportSheets[type]));
             break;
         case "expression":
-            // resultSheet.push(...buildExpressionSheets(workbook[key]));
-            break;
-        case "exportExpression":
-            resultSheet.push(...buildExpressionSheets(workbook[key]));
+            resultSheet.push(...buildExpressionSheets(workbook.exportSheets.expression));
             break;
         default:
             break;
         }
     });
-
     return resultSheet;
 };
 

package/server/controllers/expression-database-controller.js

@@ -0,0 +1,19 @@
+
+var expressionDal = require(__dirname + "/../dals/expression-dal");
+
+
+module.exports = function (app) {
+
+    app.get("/expressiondb", function (req, res) {
+        try {
+            return expressionDal.queryExpressionDatabase(req, res);
+        } catch (e) {
+            res.json({error: e.stack});
+            res.json({error: e.name});
+            res.json({error: e.message});
+
+        }
+    });
+
+
+};

package/server/controllers/network-database-controller.js

@@ -0,0 +1,18 @@
+
+var expressionDal = require(__dirname + "/../dals/network-dal");
+
+
+module.exports = function (app) {
+
+    app.get("/networkdb", function (req, res) {
+        try {
+            return expressionDal.queryNetworkDatabase(req, res);
+        } catch (e) {
+            res.json({error: e.stack});
+            res.json({error: e.name});
+            res.json({error: e.message});
+
+        }
+    });
+
+};

package/server/controllers/spreadsheet-controller.js

@@ -158,19 +158,19 @@ var crossSheetInteractions = function (workbookFile) {
         }
     }
 
-    if (additionalData && additionalData.test) {
-        // Add errors and warnings from test sheets
-        for (let sheet in additionalData.test) {
-            additionalData.test[sheet].errors.forEach(data => workbook.errors.push(data));
+    if (additionalData && additionalData.twoColumnSheets) {
+        // Add errors and warnings from two column sheets
+        for (let sheet in additionalData.twoColumnSheets) {
+            additionalData.twoColumnSheets[sheet].errors.forEach(data => workbook.errors.push(data));
         }
 
-        for (let sheet in additionalData.test) {
-            additionalData.test[sheet].warnings.forEach(data => workbook.warnings.push(data));
+        for (let sheet in additionalData.twoColumnSheets) {
+            additionalData.twoColumnSheets[sheet].warnings.forEach(data => workbook.warnings.push(data));
         }
     }
 
     if (additionalData && additionalData.meta2) {
-        // Add errors and warnings from test sheets
+        // Add errors and warnings from two column sheets
         if (additionalData.meta2.errors !== undefined) {
             additionalData.meta2.errors.forEach(data => workbook.errors.push(data));
         }
@@ -266,7 +266,7 @@ var crossSheetInteractions = function (workbookFile) {
     workbook.networkOptimizedWeights = networks.networkOptimizedWeights;
     workbook.networkWeights = networks.networkWeights;
     workbook.meta = additionalData.meta;
-    workbook.test = additionalData.test;
+    workbook.twoColumnSheets = additionalData.twoColumnSheets;
     workbook.meta2 = additionalData.meta2;
     workbook.expression = expressionData.expression;
     return workbook;

package/server/dals/expression-dal.js

@@ -0,0 +1,131 @@
+const Sequelize = require("sequelize");
+require("dotenv").config();
+var env = process.env.NODE_ENV || "development";
+var config = require("../config/config")[env];
+var sequelize = new Sequelize(
+    config.databaseName,
+    process.env.EXPRESSION_DB_USERNAME,
+    process.env.EXPRESSION_DB_PASSWORD,
+    {
+        host: config.databaseHost,
+        dialect: config.databaseDialect,
+        pool: {
+            max: 5,
+            min: 0,
+            idle: 10000
+        }
+    }
+);
+
+const buildExpressionGenesQuery = function (geneString) {
+    let genes = "";
+    let geneList = geneString.split(",");
+    geneList.forEach(x => genes += ( `(fall2021.gene.display_gene_id =\'${x}\') OR `));
+    return genes.substring(0, genes.length - 4);
+};
+
+const buildExpressionProductionDegradationRatesQuery = function (rateType, genes) {
+    return `
+    SELECT gene.display_gene_id, ${rateType}  FROM fall2021.${rateType}, fall2021.gene WHERE
+    ((${buildExpressionGenesQuery(genes)}) 
+    AND fall2021.gene.gene_id = fall2021.${rateType}.gene_id) ORDER BY display_gene_id;`;
+};
+
+const buildExpressionTimepointsFromDatasetQuery = function (dataset) {
+    return `
+    SELECT DISTINCT time_point, sample_id FROM fall2021.expression WHERE
+    dataset = '${dataset}' ORDER BY time_point ASC;`;
+};
+
+const buildExpressionDataQuery = function (dataset, genes) {
+    return `SELECT * FROM fall2021.expression, fall2021.gene WHERE fall2021.expression.dataset='${dataset}'
+    AND ((${buildExpressionGenesQuery(genes)}) 
+    AND fall2021.gene.gene_id = fall2021.expression.gene_id) ORDER BY sort_index;`;
+};
+
+
+
+const buildExpressionQuery = function (query) {
+    const expressionQueries = {
+        "DegradationRates": () => buildExpressionProductionDegradationRatesQuery("degradation_rate", query.genes),
+        "ProductionRates" : () => buildExpressionProductionDegradationRatesQuery("production_rate", query.genes),
+        "ExpressionData" : () => buildExpressionDataQuery(query.dataset, query.genes),
+        "ExpressionDatasets" : () => "SELECT DISTINCT dataset FROM fall2021.expression ORDER BY dataset ASC;",
+        "ExpressionTimePoints": () => buildExpressionTimepointsFromDatasetQuery(query.dataset)
+    };
+    if (Object.keys(expressionQueries).includes(query.type)) {
+        return expressionQueries[query.type]();
+    }
+};
+
+const listExpressionGeneData = function (gene, totalOutput) {
+    let listOfData = [];
+    totalOutput.forEach(function (x) {
+        if (x.display_gene_id === gene) {
+            listOfData.push(Number(x.expression));
+        }
+    });
+    return listOfData;
+};
+
+const convertExpressionToJSON = function (totalOutput, dataset, timePoints, allGenes) {
+    let JSONOutput = {
+        timePoints,
+        data: {
+            id: timePoints
+        }
+    };
+    allGenes.forEach(x => JSONOutput.data[x.toString()] = listExpressionGeneData(x, totalOutput));
+    return JSONOutput;
+};
+
+const ProductionDegradationRateToJSON = (totalOutput, rateType) => {
+    const JSONOutput = {
+    };
+    for (let gene of totalOutput) {
+        JSONOutput[gene.display_gene_id] = gene[rateType];
+    }
+    return JSONOutput;
+};
+
+const DatasetToJSON = (totalOutput) => {
+    const JSONOutput = {
+        expressionDatasets : []
+    };
+    for (let dataset of totalOutput) {
+        JSONOutput.expressionDatasets.push(dataset.dataset);
+    }
+    return JSONOutput;
+};
+
+const TimePointsToJSON = (totalOutput, dataset) => {
+    const JSONOutput = {};
+    JSONOutput[dataset] = [];
+    for (let timePoint of totalOutput) {
+        JSONOutput[dataset].push(timePoint.time_point);
+    }
+    return JSONOutput;
+};
+
+module.exports = {
+    queryExpressionDatabase: function (req, res) {
+        return sequelize.query(buildExpressionQuery(req.query),
+                { type: sequelize.QueryTypes.SELECT })
+                    .then(function (stdname) {
+                        const convertToJSON = {
+                            "DegradationRates" : () => ProductionDegradationRateToJSON(stdname, "degradation_rate"),
+                            "ProductionRates" : () => ProductionDegradationRateToJSON(stdname, "production_rate"),
+                            "ExpressionData" : () => convertExpressionToJSON(
+                                        stdname,
+                                        req.query.dataset,
+                                        req.query.timepoints.split(",").map(x => Number(x)),
+                                        req.query.genes.split(",")),
+                            "ExpressionDatasets": () => DatasetToJSON(stdname),
+                            "ExpressionTimePoints": () => TimePointsToJSON(stdname, req.query.dataset)
+                        };
+                        const type = req.query.type;
+                        return (Object.keys(convertToJSON).includes(type)) ? res.send(convertToJSON[type]()) :
+                            res.send(500, { errors: "Something went wrong."});
+                    });
+    }
+};

package/server/dals/network-dal.js

@@ -0,0 +1,101 @@
+/* eslint-disable max-len */
+const Sequelize = require("sequelize");
+require("dotenv").config();
+var env = process.env.NODE_ENV || "development";
+var config = require("../config/config")[env];
+var sequelize = new Sequelize(
+    config.databaseName,
+    process.env.EXPRESSION_DB_USERNAME,
+    process.env.EXPRESSION_DB_PASSWORD,
+    {
+        host: config.databaseHost,
+        dialect: config.databaseDialect,
+        pool: {
+            max: 5,
+            min: 0,
+            idle: 10000
+        }
+    }
+);
+
+const buildNetworkSourceQuery = function () {
+    return "SELECT * FROM spring2022_network.source ORDER BY time_stamp;";
+};
+
+const buildNetworkGeneFromSourceQuery = function (gene, source, timestamp) {
+    return `SELECT DISTINCT gene_id, display_gene_id FROM spring2022_network.network, spring2022_network.gene WHERE
+ network.time_stamp='${timestamp}' AND network.source='${source}' AND
+ (gene.gene_id ='${gene}' OR gene.display_gene_id ='${gene}') AND
+ (gene.gene_id = network.regulator_gene_id OR gene.gene_id = network.target_gene_id);`;
+};
+
+const buildNetworkGenesQuery = function (geneString) {
+    let genes = "(";
+    let geneList = geneString.split(",");
+    geneList.forEach(x => genes += ( `(network.regulator_gene_id =\'${x}\') OR `));
+    genes = `${genes.substring(0, genes.length - 4)}) AND (`;
+    geneList.forEach(x => genes += ( `(network.target_gene_id =\'${x}\') OR `));
+    return `${genes.substring(0, genes.length - 4)})`;
+
+};
+
+const buildGenerateNetworkQuery = function (genes, source, timestamp) {
+    return `SELECT DISTINCT regulator_gene_id, target_gene_id FROM
+ spring2022_network.network WHERE
+ time_stamp='${timestamp}' AND source='${source}' AND
+ ${buildNetworkGenesQuery(genes)} ORDER BY regulator_gene_id DESC;`;
+};
+
+const buildQueryByType = function (queryType, query) {
+    const networkQueries = {
+        "NetworkSource": () => buildNetworkSourceQuery(),
+        "NetworkGeneFromSource": () => buildNetworkGeneFromSourceQuery(query.gene, query.source, query.timestamp),
+        "GenerateNetwork": () => buildGenerateNetworkQuery(query.genes, query.source, query.timestamp)
+    };
+    if (Object.keys(networkQueries).includes(query.type)) {
+        return networkQueries[query.type]();
+    }
+};
+
+const convertResponseToJSON = function (queryType, query, totalOutput) {
+    let JSONOutput = {};
+    switch (queryType) {
+    case "NetworkSource":
+        JSONOutput.sources = {};
+        totalOutput.forEach(function (x) {
+            const timestamp = x.time_stamp;
+            const source = x.source;
+            const displayName = x.display_name;
+            JSONOutput.sources[`${displayName} : ${timestamp.toISOString().split("T")[0]}`] = {timestamp, source};
+        });
+        return JSONOutput;
+    case "NetworkGeneFromSource":
+        JSONOutput.displayGeneId = totalOutput.length > 0 ? totalOutput[0].display_gene_id : null;
+        JSONOutput.geneId = totalOutput.length > 0 ? totalOutput[0].gene_id : null;
+        return JSONOutput;
+    case "GenerateNetwork":
+        JSONOutput.links = {};
+        for (let connection of totalOutput) {
+            if (JSONOutput.links[connection.regulator_gene_id] === undefined) {
+                JSONOutput.links[connection.regulator_gene_id] = [connection.target_gene_id];
+            } else {
+                JSONOutput.links[connection.regulator_gene_id].push(connection.target_gene_id);
+            }
+        }
+        return JSONOutput;
+    default:
+        return JSONOutput;
+    }
+
+};
+
+module.exports = {
+    buildNetworkSourceQuery: buildNetworkSourceQuery,
+    queryNetworkDatabase: function (req, res) {
+        sequelize.query(buildQueryByType(req.query.type, req.query), { type: sequelize.QueryTypes.SELECT })
+            .then(function (stdname) {
+                const response = convertResponseToJSON(req.query.type, req.query, stdname);
+                return res.send(response);
+            });
+    }
+};

package/test/additional-sheet-parser-tests.js

@@ -103,7 +103,7 @@ describe("additional-sheet-parser", function () {
             var workbook = xlsx.parse(__dirname + "/../test-files/spreadsheet-controller-test-files/" +
             "2_column_data_format_test.xlsx");
             var data = parseAdditionalSheets(workbook);
-            assert(data["test"]["degradation_rates"], {
+            assert(data.twoColumnSheets["degradation_rates"], {
                 "ACE2": 0.1118,
                 "ASH1": 0.2166,
                 "CIN5": 0.1005,

package/test/api-tests.js

@@ -1,6 +1,6 @@
 const jsdom = require("jsdom");
 
-// Our fake document needs a #service-root element so that a fake "host" can be found by the code.
+// Our fake document needs a .service-root element so that a fake "host" can be found by the code.
 const { document } = (new jsdom.JSDOM("<input type='hidden' id='service-root' value='http://test'>")).window;
 global.document = document;
 global.window = document;
@@ -101,9 +101,6 @@ describe("The Gene Page", () => {
         </eSearchResult>`;
 
 
-        let NCBIDoc = document.implementation.createDocument("", "", null);
-        let sequenceText = NCBIDoc.createTextNode(testString);
-        NCBIDoc.appendChild(sequenceText);
         server.respondWith([
             200,
         {"Content-Type": "text/plain" }, testString]);
@@ -242,4 +239,4 @@ describe("The Gene Page", () => {
 
     });
 
-});
+});