imerss-bioinfo 1.0.0

package/src/client/json/phyloMap.json

@@ -0,0 +1,58 @@
+{
+    "kingdom:Animalia": {
+        "pic": "%resourceBase/img/Animalia_2.svg",
+        "colour": "#f26522"
+    },
+    "kingdom:Chromista": {
+        "pic": "%resourceBase/img/Chromista.svg",
+        "colour": "#009fd3"
+    },
+    "kingdom:Plantae": {
+        "pic": "%resourceBase/img/Plantae_2.svg",
+        "colour": "#8cc63f"
+    },
+    "kingdom:Protozoa": {
+        "pic": "%resourceBase/img/Protista.svg",
+        "colour": "#1eb28f"
+    },
+    "phylum:Annelida": {
+        "pic": "%resourceBase/img/Animalia_Annelida.svg",
+        "colour": "#f26522"
+    },
+    "phylum:Chaetognatha": {
+        "pic": "%resourceBase/img/Animalia_Chaetognatha.svg",
+        "colour": "#f26522"
+    },
+    "phylum:Cnidaria": {
+        "pic": "%resourceBase/img/Animalia_Cnidaria.svg",
+        "colour": "#f26522"
+    },
+    "phylum:Ctenophora": {
+        "pic": "%resourceBase/img/Animalia_Ctenophora.svg",
+        "colour": "#f26522"
+    },
+    "phylum:Nemertea": {
+        "pic": "%resourceBase/img/Animalia_Nemertea.svg",
+        "colour": "#f26522"
+    },
+    "phylum:Phoronida": {
+        "pic": "%resourceBase/img/Animalia_Phoronida.svg",
+        "colour": "#f26522"
+    },
+    "phylum:Porifera": {
+        "pic": "%resourceBase/img/Animalia_Porifera.svg",
+        "colour": "#f26522"
+    },
+    "phylum:Platyhelminthes": {
+        "pic": "%resourceBase/img/Animalia_Platyhelminthes.svg",
+        "colour": "#f26522"
+    },
+    "phylum:Sipuncula": {
+        "pic": "%resourceBase/img/Animalia_Sipuncula.svg",
+        "colour": "#f26522"
+    },
+    "life:Life": {
+        "taxonPic": "%resourceBase/img/Plankton.jpg",
+        "taxonPicDescription": "<em>\"The Importance of Plankton\"</em> by Susan Taylor"
+    }
+}

package/src/compare.js

@@ -0,0 +1,97 @@
+/* eslint-env node */
+
+"use strict";
+
+var fluid = require("infusion");
+var minimist = require("minimist");
+fluid.require("%imerss-bioinfo");
+
+require("./dataProcessing/readJSON.js");
+require("./dataProcessing/readCSV.js");
+require("./dataProcessing/readCSVwithMap.js");
+require("./dataProcessing/writeCSV.js");
+
+var hortis = fluid.registerNamespace("hortis");
+
+hortis.ranks = fluid.freezeRecursive(require("../data/ranks.json"));
+// Map of taxon name to list of higher taxa
+hortis.higherTaxa = fluid.transform(fluid.arrayToHash(hortis.ranks), function (troo, rank) {
+    return hortis.ranks.slice(0, hortis.ranks.indexOf(rank));
+});
+
+fluid.setLogging(true);
+
+var parsedArgs = minimist(process.argv.slice(2));
+
+var file1 = parsedArgs._[0];
+var file2 = parsedArgs._[1];
+
+var mapFile = parsedArgs.map || "%imerss-bioinfo/data/coreOutMap.json";
+var map = hortis.readJSONSync(fluid.module.resolvePath(mapFile), "reading map file");
+
+var files = [file1, file2];
+
+var readers = files.map(function (oneFile) {
+    return hortis.csvReaderWithMap({
+        inputFile: oneFile,
+        mapColumns: map.columns
+    });
+});
+
+var promises = fluid.getMembers(readers, "completionPromise");
+
+hortis.findExcess = function (hash1, hash2) {
+    var togo = [];
+    fluid.each(hash1, function (value, name) {
+        if (!hash2[name]) {
+            togo.push(name);
+        }
+    });
+    return togo;
+};
+
+hortis.reportExcess = function (names, hash, mapColumns, filename) {
+    var excessRowHash = fluid.filterKeys(hash, names);
+    var excessRows = Object.values(excessRowHash);
+    hortis.writeCSV(filename, map.columns, excessRows, fluid.promise());
+};
+
+hortis.castOutOneHigherTaxa = function (hash, row, rank) {
+    if (row[rank]) {
+        hortis.higherTaxa[rank].forEach(function (higherRank) {
+            delete hash[row[higherRank]];
+        });
+    }
+};
+
+hortis.castOutHigherTaxa = function (hash) {
+    console.log("Original size: " + Object.keys(hash).length);
+    fluid.each(hash, function (row) {
+        hortis.castOutOneHigherTaxa(hash, row, "species");
+        hortis.castOutOneHigherTaxa(hash, row, "genus");
+    });
+    console.log("After casting out higher taxa: " + Object.keys(hash).length);
+};
+
+fluid.promise.sequence(promises).then(function () {
+    console.log("Loaded " + readers.length + " CSV files");
+    var rows = fluid.getMembers(readers, "rows");
+    var names = rows.map(function (oneRows) {
+        return fluid.getMembers(oneRows, "iNaturalistTaxonName");
+    });
+    var hashes = names.map(function (oneNames, index) {
+        var togo = {};
+        oneNames.forEach(function (name, rowIndex) {
+            togo[name] = rows[index][rowIndex];
+        });
+        return togo;
+    });
+    hashes.forEach(hortis.castOutHigherTaxa);
+    var onlyOne = hortis.findExcess(hashes[0], hashes[1]);
+    hortis.reportExcess(onlyOne, hashes[0], map.columns, "excess1.csv");
+    var onlyTwo = hortis.findExcess(hashes[1], hashes[0]);
+    hortis.reportExcess(onlyTwo, hashes[1], map.columns, "excess2.csv");
+
+}, function (err) {
+    console.log("Error ", err);
+});

package/src/compilecsv.js

@@ -0,0 +1,47 @@
+/* eslint-env node */
+
+"use strict";
+
+var fluid = require("infusion");
+var minimist = require("minimist");
+var glob = require("glob");
+fluid.require("%imerss-bioinfo");
+
+require("./dataProcessing/readJSON.js");
+require("./dataProcessing/readCSV.js");
+require("./dataProcessing/readCSVwithMap.js");
+require("./dataProcessing/writeCSV.js");
+
+var hortis = fluid.registerNamespace("hortis");
+
+fluid.setLogging(true);
+
+var parsedArgs = minimist(process.argv.slice(2));
+
+var outputFile = parsedArgs.o || "%imerss-bioinfo/data/dataPaper-I-in/Animalia.csv";
+var inputFile = parsedArgs._[0] || "%imerss-bioinfo/data/dataPaper-I-in/Animalia/*.csv";
+var mapFile = parsedArgs.map || "%imerss-bioinfo/data/dataPaper-I-in/dataPaper-map.json";
+
+var map = hortis.readJSONSync(fluid.module.resolvePath(mapFile), "reading map file");
+
+var files = glob.sync(fluid.module.resolvePath(inputFile));
+
+var readers = files.map(function (oneFile) {
+    // TODO: Worry if the schemas of the CSV files diverge - we stopped using the map file since we don't want
+    // its conversion, but we do want to verify the headers
+    // We probably want to grab the "onHeaders.validateHeaders" listener from csvReaderWithMap
+    return hortis.csvReaderWithoutMap({
+        inputFile: oneFile
+    });
+});
+
+var promises = fluid.getMembers(readers, "completionPromise");
+
+fluid.promise.sequence(promises).then(function () {
+    console.log("Loaded " + readers.length + " CSV files");
+    var allRows = fluid.flatten(fluid.getMembers(readers, "rows"));
+    var togo = fluid.promise();
+    hortis.writeCSV(fluid.module.resolvePath(outputFile), Object.keys(allRows[0]), allRows, togo);
+}, function (err) {
+    console.log("Error ", err);
+});

package/src/dataProcessing/LZ4.js

@@ -0,0 +1,27 @@
+/* eslint-env node */
+
+"use strict";
+
+const fluid = require("infusion");
+const lz4 = require("lz4");
+const fs = require("fs");
+const stream = require("stream");
+
+const hortis = fluid.registerNamespace("hortis");
+
+// TODO: move to pako https://github.com/nodeca/pako
+
+hortis.writeLZ4File = function (text, filename) {
+    const input = new stream.Readable();
+    input.push(text);
+    input.push(null);
+
+    const output = fs.createWriteStream(filename);
+
+    const encoder = lz4.createEncoderStream();
+    input.pipe(encoder).pipe(output);
+    output.on("finish", function () {
+        const stats = fs.statSync(filename);
+        console.log("Written " + stats.size + " bytes to " + filename);
+    });
+};

package/src/dataProcessing/coordinatePatch.js

@@ -0,0 +1,151 @@
+/* eslint-env node */
+
+"use strict";
+
+var fluid = require("infusion");
+
+var hortis = fluid.registerNamespace("hortis");
+
+// TODO: grab this from the fusion's input map
+hortis.stableDatasetIds = {
+    iNat: true,
+    CMN: true,
+    RBCM: true
+};
+
+// Also in leafletMap.js
+hortis.datasetIdFromObs = function (obsId) {
+    var colpos = obsId.indexOf(":");
+    return obsId.substring(0, colpos);
+};
+
+hortis.equalCoordinate = function (a, b) {
+    // 6 decimals is a common standard - about 10cm accuracy, e.g. RBCM truncated to this
+    return Math.abs(a - b) < 1e-6;
+};
+
+hortis.equalCoordinates = function (lat1, long1, lat2, long2) {
+    return hortis.equalCoordinate(lat1, lat2) && hortis.equalCoordinate(long1, long2);
+};
+
+hortis.matchOnePatchRow = function (patchRow, obsRow) {
+    var score = 0;
+    if (patchRow.id === obsRow.observationId) {
+        var dataset = hortis.datasetIdFromObs(patchRow.id);
+        score += hortis.stableDatasetIds[dataset] ? 1 : 0.5;
+    }
+    if (hortis.equalCoordinates(patchRow.latitude, patchRow.longitude, obsRow.latitude, obsRow.longitude)) {
+        score += 1;
+    }
+    if (patchRow.iNaturalistTaxonName === obsRow.iNaturalistTaxonName) {
+        score += 1;
+    }
+    return score;
+};
+
+hortis.dumpPatchRow = function (patchRow, loc) {
+    return Object.values(patchRow).join(", ") + loc;
+};
+
+hortis.dumpObsMatch = function (match) {
+    var obs = match.obsRow;
+    return "obs of " + obs.iNaturalistTaxonName + " with coordinates " + JSON.stringify(obs.latitude) + ", " + JSON.stringify(obs.longitude) +
+        " with id " + obs.observationId + " at index " + match.obsIndex;
+};
+
+/** Returns one match or null, plus some side-effects
+ * @param {Object} patchRow - One row of the patch file
+ * @param {ObsRow[]} obsRows - All observation rows
+ * @param {String} loc - A diagnostic string indicating the location of the `patchRow`
+ * @return {ObsRow|Null} The best matching observation row, or null
+ */
+hortis.matchCoordinatePatch = function (patchRow, obsRows, loc) {
+    var matches = [];
+    obsRows.forEach(function (obsRow, obsIndex) {
+        var score = hortis.matchOnePatchRow(patchRow, obsRow);
+        if (score > 0) {
+            matches.push({
+                score: score,
+                obsRow: obsRow,
+                obsIndex: obsIndex
+            });
+        }
+    });
+    var diag = "Warning: patch " + hortis.dumpPatchRow(patchRow, loc);
+    if (matches.length === 0) {
+        console.log(diag + " failed to match in any coordinate");
+        return null;
+    } else {
+        matches.sort(function (a, b) {
+            return b.score - a.score;
+        });
+        var topScore = matches[0].score;
+        var topScoring = matches.filter(function (match) {
+            return match.score === topScore;
+        });
+        if (topScoring.length > 1) {
+            console.log(diag + " matched " + topScoring.length + " rows equally well: these were: "
+                + topScoring.map(hortis.dumpObsMatch).join("\n"));
+        } else if (topScore < 2.5) {
+            console.log(diag + " matched " + hortis.dumpObsMatch(topScoring[0]) + " with reduced accuracy of " + topScore);
+            var obsRow = topScoring[0].obsRow;
+            console.log(hortis.equalCoordinates(patchRow.latitude, patchRow.longitude, obsRow.latitude, obsRow.longitude));
+        }
+        return topScoring[0].obsRow;
+    }
+};
+
+hortis.coordinatesOutMap = {
+    columns: {
+        coordinatesCorrected: "Coordinates Corrected",
+        coordinatesCorrectedNote: "Coordinates Corrected Note"
+    }
+};
+
+hortis.parseToNumber = function (row, field) {
+    var parsed = hortis.parseFloat(row[field]);
+    row[field] = parsed;
+};
+
+hortis.patchNumberFields = ["latitude", "longitude", "latitudeOut", "longitudeOut"];
+
+/**
+ * Resolve a coordinate patch file against the observation list
+ * @param {Object} resolved - Resolved observations containing members obsOutMap and obsRows which will be modified
+ * @param {PatchFile} patch - The loaded patch structure
+ * @param {String} key - Key of the patch
+ */
+hortis.processCoordinatePatch = function (resolved, patch, key) {
+
+    patch.patchData.rows.forEach(function (patchRow, index) {
+        var loc = " at row " + index + " of patch " + key;
+        hortis.patchNumberFields.forEach(function (field) {
+            hortis.parseToNumber(patchRow, field);
+        });
+        var match = hortis.matchCoordinatePatch(patchRow, resolved.obsRows, loc);
+
+        var expectCoordinates = function (testEqual) {
+            var equal = hortis.equalCoordinates(match.latitude, match.longitude, patchRow.latitudeOut, patchRow.longitudeOut);
+            if (equal !== testEqual) {
+                console.log("Error: Coordinate corrected value for patch " + hortis.dumpPatchRow(patchRow, loc) + " does not conform with original coordinates "
+                    + match.latitude + ", " + match.longitude + " since value of \"Corrected\" is " + patchRow.corrected);
+            }
+        };
+
+        if (match) {
+            if (patchRow.corrected === "yes") {
+                expectCoordinates(false);
+                match.coordinatesCorrected = "yes";
+            } else if (patchRow.corrected === "no") {
+                expectCoordinates(true);
+                match.coordinatesCorrected = "no";
+            } else {
+                console.log("Unexpected \"Corrected\" value " + patchRow.corrected + " for patch " + hortis.dumpPatchRow(patchRow, loc));
+            }
+            match.latitude = patchRow.latitudeOut;
+            match.longitude = patchRow.longitudeOut;
+            match.coordinatesCorrectedNote = patchRow.correction;
+        }
+    });
+    resolved.combinedObsOutMap = hortis.combineMaps([resolved.combinedObsOutMap, hortis.coordinatesOutMap]);
+};

package/src/dataProcessing/readCSV.js

@@ -0,0 +1,107 @@
+/* eslint-env node */
+
+"use strict";
+
+const fluid = require("infusion");
+const csvModule = require("csv-parser");
+const fs = require("fs");
+const removeBOM = require("remove-bom-stream");
+
+const hortis = fluid.registerNamespace("hortis");
+
+fluid.defaults("hortis.csvReader", {
+    gradeNames: "fluid.component",
+    members: {
+        line: 1,
+        rowStream: null,
+        headers: {},
+        completionPromise: "@expand:fluid.promise()"
+    },
+    csvOptions: {
+    },
+    events: {
+        onHeaders: null,
+        onRow: null,
+        onError: null,
+        onCompletion: null
+    },
+    listeners: {
+        "onRow.lineCounter": {
+            funcName: "hortis.csvReader.incLine",
+            args: "{that}",
+            priority: "first"
+        },
+        "onHeaders.storeHeaders": "hortis.csvReader.storeHeaders({that}, {arguments}.0)",
+        "onError.rejector": "hortis.csvReader.rejector({that}, {arguments}.0)",
+        "onCreate.openStream": "fluid.notImplemented",
+        "onCreate.bindStream": {
+            priority: "after:openStream",
+            funcName: "hortis.csvReader.bindStream"
+        }
+    }
+});
+
+hortis.csvReader.incLine = function (that) {
+    ++that.line;
+};
+
+hortis.csvReader.storeHeaders = function (that, headers) {
+    that.headers = headers;
+};
+
+hortis.csvReader.rejector = function (that, error) {
+    that.completionPromise.reject({
+        message: "Error at line " + that.line + " reading file " + that.options.inputFile + ": " + error
+    });
+};
+
+hortis.csvReader.bindStream = function (that) {
+    const now = Date.now();
+    that.rowStream.on("data", function (data) {
+        that.events.onRow.fire(data);
+    });
+    that.rowStream.on("error", function (error) {
+        that.events.onError.fire(error);
+    });
+    that.rowStream.on("headers", function (headers) {
+        that.events.onHeaders.fire(headers);
+    });
+    that.rowStream.on("end", function () {
+        console.log("Read " + that.line + " lines in " + (Date.now() - now) + " ms");
+        if (!that.completionPromise.disposition) {
+            that.events.onCompletion.fire();
+        }
+    });
+};
+
+
+
+fluid.defaults("hortis.csvFileReader", {
+    gradeNames: "hortis.csvReader",
+    inputFile: null,
+    listeners: {
+        "onCreate.openStream": "hortis.csvFileReader.openFileStream"
+    }
+});
+
+hortis.csvFileReader.openFileStream = function (that) {
+    that.rowStream = fs.createReadStream(that.options.inputFile)
+        .pipe(removeBOM("utf-8"))
+        .pipe(csvModule(that.options.csvOptions)
+        );
+};
+
+fluid.defaults("hortis.storingCSVReader", {
+    members: {
+        rows: []
+    },
+    listeners: {
+        "onCompletion.resolve": {
+            func: "{that}.completionPromise.resolve",
+            args: {
+                rows: "{that}.rows",
+                headers: "{that}.headers"
+            }
+        }
+    }
+});

package/src/dataProcessing/readCSVwithMap.js

@@ -0,0 +1,90 @@
+/* eslint-env node */
+
+"use strict";
+
+const fluid = require("infusion");
+const moment = require("moment"); // Horrid - required to apply template parsing to Date fields
+
+const hortis = fluid.registerNamespace("hortis");
+
+fluid.defaults("hortis.csvReaderWithMap", {
+    gradeNames: ["hortis.csvFileReader", "hortis.storingCSVReader"],
+    // "columns" member of the map structure
+    mapColumns: null,
+    // true if the map file defines template interpolation via % rather than simply mapping column names e.g. as an "outMap"
+    templateMap: false,
+    listeners: {
+        "onHeaders.validateHeaders": {
+            funcName: "hortis.validateHeaders",
+            args: ["{that}.options.mapColumns", "{that}.options.templateMap", "{that}.headers", "{that}.events.onError"]
+        },
+        "onRow.storeRow": "hortis.storeRow({that}, {arguments}.0)"
+    }
+});
+
+hortis.templateMapField = function (that, data, template) {
+    if (template.startsWith("!Date")) {
+        const match = template.match(/!Date\((.*),(.*)\)/);
+        const format = match[1], smallTemplate = match[2];
+        const value = fluid.stringTemplate(smallTemplate, data);
+        return moment.utc(value, format).toISOString();
+    } else if (template.startsWith("@")) {
+        const rec = fluid.compactStringToRec(template.substring(1), "expander");
+        const func = fluid.getGlobalValue(rec.funcName);
+        if (!func) {
+            fluid.fail("Unable to look up " + rec.funcName + " as a global function");
+        }
+        const args = fluid.transform(rec.args, function (arg) {
+            return fluid.stringTemplate(arg, data);
+        });
+        return func.apply(null, args);
+    } else {
+        const warnFunc = function (key) {
+            console.log("Warning: template key %" + key + " did not match a column in the data for file " + that.options.inputFile);
+        };
+        const togo = fluid.stringTemplateWarn(template, data, warnFunc);
+        return togo;
+    }
+};
+
+// Speed on taxa - 500ms on IoC dispatch, 1300ms on mapping
+hortis.mapRow = function (that, data, mapColumns, templateMap, index) {
+    const togo = {};
+    fluid.each(mapColumns, function (label, key) {
+        togo[key] = templateMap ? hortis.templateMapField(that, data, label) : data[label];
+    });
+    togo.index = index;
+    return togo;
+};
+
+hortis.storeRow = function (that, row) {
+    const mappedRow = hortis.mapRow(that, row, that.options.mapColumns, that.options.templateMap, that.rows.length + 1);
+    that.rows.push(mappedRow);
+};
+
+hortis.validateHeaders = function (mapColumns, templateMap, headers, onError) {
+    if (!mapColumns || mapColumns.length === 0) {
+        fluid.fail("Error in csvReaderWithMap - no mapColumns were supplied");
+    }
+    const headerHash = fluid.arrayToHash(headers);
+    fluid.each(mapColumns, function (label) {
+        if (templateMap) {
+            const interp = fluid.stringTemplate(label, headerHash);
+            if (interp.indexOf("%") !== -1) {
+                onError.fire("Error in headers - template " + label + " did not match headers");
+            }
+        } else {
+            if (headers.indexOf(label) === -1) {
+                onError.fire("Error in headers - field " + label + " required in map file was not found");
+            }
+        }
+    });
+};
+
+// Used in writeCSV
+hortis.mapToArray = function (row, map) {
+    const keys = Object.keys(map);
+    return keys.map(function (header) {
+        return row[header];
+    });
+};

package/src/dataProcessing/readCSVwithoutMap.js

@@ -0,0 +1,25 @@
+/* eslint-env node */
+
+"use strict";
+
+var fluid = require("infusion");
+
+var hortis = fluid.registerNamespace("hortis");
+
+fluid.defaults("hortis.csvReaderWithoutMap", {
+    gradeNames: ["hortis.csvFileReader", "hortis.storingCSVReader"],
+    listeners: {
+        "onRow.storeRow": "hortis.storeRowWithoutMap({that}, {arguments}.0)"
+    }
+});
+
+hortis.storeRowWithoutMap = function (that, row) {
+    that.rows.push(row);
+};
+
+// Used in writeCSV
+hortis.rowToArray = function (row, headers) {
+    return headers.map(function (header) {
+        return row[header];
+    });
+};

package/src/dataProcessing/readJSON.js

@@ -0,0 +1,23 @@
+/* eslint-env node */
+
+"use strict";
+
+const fluid = require("infusion");
+const kettle = require("kettle");
+
+const hortis = fluid.registerNamespace("hortis");
+
+hortis.readJSONSync = function (fileName, message) {
+    const promise = kettle.JSON.readFileSync(fileName, message + " " + fileName);
+    let togo;
+    promise.then(function (parsed) {
+        togo = parsed;
+    }, function (err) {
+        throw err;
+    });
+    return togo;
+};
+
+hortis.readModuleJSONSync = function (fileName, message) {
+    return hortis.readJSONSync(fluid.module.resolvePath(fileName), message);
+};