imerss-bioinfo 1.0.0

package/src/utils/dataSource.js

@@ -0,0 +1,173 @@
+/* eslint-env node */
+
+"use strict";
+
+const fluid = require("infusion");
+
+fluid.defaults("fluid.dataSource.noencoding", {
+    components: {
+        encoding: {
+            type: "fluid.dataSource.encoding.none"
+        }
+    }
+});
+
+// Patch this from Kettle DataSource to interpret iNat errors which have
+fluid.extractHtmlError = function (received) {
+    const preMatches = /<pre>(.*)<\/pre>/gm.exec(received);
+    if (preMatches) {
+        return preMatches[1];
+    }
+    const titleMatches = /<title>(.*)<\/title>/gm.exec(received);
+    return titleMatches ? titleMatches[1] : received;
+};
+
+fluid.defaults("fluid.dataSource.retrying", {
+    gradeNames: "fluid.dataSource",
+    retryInterval: 1000,
+    retries: 3,
+    invokers: {
+        get: {
+            funcName: "fluid.dataSource.retrying.get",
+            args: ["{that}", "{arguments}.0", "{arguments}.1"] // directModel, directOptions
+        }
+    }
+});
+
+fluid.promise.delay = function (delay) {
+    const togo = fluid.promise();
+    setTimeout(togo.resolve, delay);
+    return togo;
+};
+
+fluid.dataSource.retrying.get = async function (that, directModel, directOptions) {
+    const underlyingGet = function () {
+        return fluid.dataSource.get(that, directModel, directOptions);
+    };
+    let err = null;
+    for (let i = 0; i < that.options.retries; ++i) {
+        try {
+            const payload = await underlyingGet();
+            return payload;
+        } catch (e) {
+            fluid.log("Got exception ", e.message , " from dataSource, retry " + (i + 1) + "/" + that.options.retries);
+            err = e;
+            await fluid.promise.delay(that.options.retryInterval);
+        }
+    }
+    throw err;
+};
+
+fluid.setLogging(true);
+
+// A rate limited dataSource that will guarantee not to make more than 1 call every <rateLimit> ms to the underlying
+// dataSource
+
+// Neither throttle nor debounce seem appropriate for this - https://underscorejs.org/#throttle - since they do
+// not guarantee one for one invocation of target source
+
+fluid.defaults("fluid.dataSource.rateLimiter", {
+    gradeNames: "fluid.component",
+    // minimum time in ms between calls
+    rateLimit: 1000,
+    members: {
+        pending: null,
+        lastFired: 0,
+        queue: []
+    },
+    invokers: {
+        requestStart: "fluid.dataSource.rateLimiter.requestStart({that}, {arguments}.0)",
+        requestEnd: "fluid.dataSource.rateLimiter.requestEnd({that}, {arguments}.0)",
+        next: "fluid.dataSource.rateLimiter.next({that})"
+    }
+});
+
+fluid.dataSource.rateLimiter.next = function (that) {
+    const now = Date.now();
+    const resolve = function (toResolve) {
+        that.lastFired = now;
+        toResolve.resolve(toResolve.payload);
+    };
+
+    const uncess = that.lastFired + that.options.rateLimit - now;
+    // console.log("rateLimit: " + that.options.rateLimit + " uncess " + uncess);
+    if (!that.pending && that.queue.length > 0) {
+        if (uncess > 0) {
+            that.pending = setTimeout(function () {
+                that.pending = null;
+                that.next();
+            }, uncess);
+        } else {
+            const top = that.queue.shift();
+            resolve(top);
+        }
+    }
+};
+
+fluid.dataSource.rateLimiter.requestStart = function (that, payload) {
+    const togo = fluid.promise();
+    togo.payload = payload;
+    that.queue.push(togo);
+    that.next();
+    return togo;
+};
+
+fluid.dataSource.rateLimiter.requestEnd = function (that, payload) {
+//    fluid.log("requestEnd");
+    that.next();
+    return payload;
+};
+
+fluid.defaults("fluid.dataSource.withRateLimiter", {
+    gradeNames: "fluid.dataSource.rateLimiter",
+    listeners: {
+        "onRead.rateLimitBefore": {
+            priority: "before:impl",
+            func: "{that}.requestStart"
+        },
+        "onRead.rateLimitAfter": {
+            priority: "after:impl",
+            func: "{that}.requestEnd"
+        }
+    }
+});
+
+fluid.defaults("fluid.inMemoryCachedSource", {
+    gradeNames: ["fluid.dataSource", "fluid.dataSource.noencoding", "fluid.dataSource.writable"],
+    members: {
+        cache: {}
+    },
+    listeners: {
+        "onRead.impl": {
+            func: "fluid.inMemoryCachedSource.read",
+            args: ["{that}", "{arguments}.0", "{arguments}.1"]
+        },
+        "onWrite.impl": {
+            func: "fluid.inMemoryCachedSource.write",
+            args: ["{that}", "{arguments}.0", "{arguments}.1"]
+        }
+    }
+});
+
+fluid.inMemoryCachedSource.toCacheKey = function (directModel) {
+    if (fluid.isPrimitive(directModel)) {
+        return directModel.toString();
+    } else {
+        let togo = "";
+        for (const key in directModel) {
+            togo += key + "|" + directModel[key] + "|";
+        }
+        return togo;
+    }
+};
+
+fluid.inMemoryCachedSource.read = function (that, payload, options) {
+    const key = fluid.inMemoryCachedSource.toCacheKey(options.directModel);
+    return that.cache[key];
+};
+
+fluid.inMemoryCachedSource.write = function (that, payload, options) {
+    const key = fluid.inMemoryCachedSource.toCacheKey(options.directModel);
+    that.cache[key] = payload;
+    return payload;
+};

package/src/utils/settleStructure.js

@@ -0,0 +1,62 @@
+/* eslint-env node */
+
+"use strict";
+
+var fluid = require("infusion");
+var hortis = fluid.registerNamespace("hortis");
+
+hortis.settleStructure = function (structure) {
+    if (fluid.isPromise(structure)) {
+        return structure;
+    }
+    var settleRec = {
+        // This flag is an awkward consequence of our choice to allow synchronous promise resolution
+        inSync: true, // Are we still in synchronous scanning - necessary to avoid double resolution on hitting unresolved === 0 on initial scan
+        unresolved: 0,
+        depth: 0,
+        promise: fluid.promise(),
+        structure: structure,
+        resolve: function () {
+            settleRec.promise.resolve(structure);
+        }
+    };
+    hortis.settleStructureRecurse(structure, settleRec);
+    settleRec.inSync = false;
+    if (settleRec.unresolved === 0) { // Case of 0 asynchronous promises found
+        settleRec.resolve();
+    }
+    return settleRec.promise;
+};
+
+hortis.settleStructureRecurse = function (structure, settleRec) {
+    ++settleRec.depth;
+    if (settleRec.depth > fluid.strategyRecursionBailout) {
+        fluid.fail("Recursion exceeded for value " + JSON.stringify(structure) + " overall structure " + JSON.stringify(settleRec.structure, null, 2));
+    }
+    if (fluid.isPlainObject(structure)) {
+        fluid.each(structure, function (value, key) {
+            if (fluid.isPromise(value)) {
+                hortis.settleStructurePush(settleRec, structure, value, key);
+            } else {
+                hortis.settleStructureRecurse(value, settleRec);
+            }
+        });
+    }
+    --settleRec.depth;
+};
+
+hortis.settleStructurePush = function (settleRec, holder, promise, key) {
+    ++settleRec.unresolved;
+    promise.then(function (value) {
+        holder[key] = value;
+        --settleRec.unresolved;
+        if (settleRec.unresolved === 0 && !settleRec.inSync) {
+            settleRec.resolve();
+        }
+    }, function (err) {
+        if (!settleRec.promise.disposition) {
+            settleRec.promise.reject(hortis.upgradeError(err, " while resolving promise with key " + key));
+        }
+    });
+};
+

package/src/utils/utils.js

@@ -0,0 +1,92 @@
+/* eslint-env node */
+
+"use strict";
+
+const fluid = require("infusion");
+
+const hortis = fluid.registerNamespace("hortis");
+
+hortis.capitalize = function (string) {
+    return string.charAt(0).toUpperCase() + string.slice(1);
+};
+
+hortis.pluralise = function (number, noun) {
+    return number + " " + noun + (number === 1 ? "" : "s");
+};
+
+// Helpful answer: https://stackoverflow.com/a/35759874
+hortis.parseFloat = function (str) {
+    return isNaN(str) ? NaN : parseFloat(str);
+};
+
+// https://stackoverflow.com/a/24457420
+hortis.isInteger = function isNumeric(value) {
+    return /^-?\d+$/.test(value);
+};
+
+/** Upgrades a promise rejection payload (or Error) by suffixing an additional "while" reason into its "message" field
+ * @param {Object|Error} originError - A rejection payload. This should (at least) have the member `isError: true` set, as well as a String `message` holding a rejection reason.
+ * @param {String} whileMsg - A message describing the activity which led to this error
+ * @return {Object} The rejected payload formed by shallow cloning the supplied argument (if it is not an `Error`) and suffixing its `message` member
+ */
+hortis.upgradeError = function (originError, whileMsg) {
+    const error = originError instanceof Error ? originError : fluid.extend({}, originError);
+    error.message = originError.message + whileMsg;
+    return error;
+};
+
+hortis.roundDecimals = function (text, places) {
+    const parsed = hortis.parseFloat(text);
+    if (!isNaN(parsed)) {
+        return +parsed.toFixed(places);
+    } else {
+        return text;
+    }
+};
+
+hortis.stringTemplateRegex = /\${([^\}]*)}/g;
+
+// Template values with a ${key} syntax using a straightforward regex strategy and optional warning for missing values
+hortis.stringTemplate = function (template, vars, warnFunc) {
+    const replacer = function (all, match) {
+        const segs = match.split(".");
+        const fetched = fluid.getImmediate(vars, segs);
+        if (warnFunc && fetched === undefined) {
+            warnFunc(match);
+        }
+        return fetched || "";
+    };
+    return template.replace(hortis.stringTemplateRegex, replacer);
+};
+
+// Template values with a %key syntax and optional warning for missing values
+fluid.stringTemplateWarn = function (template, values, warnFunc) {
+    let keys = Object.keys(values);
+    keys = keys.sort(fluid.compareStringLength());
+    for (let i = 0; i < keys.length; ++i) {
+        const key = keys[i];
+        const templatePlaceholder = "%" + key;
+        const replacementValue = values[key];
+        if (warnFunc && replacementValue === undefined) {
+            warnFunc(key);
+        }
+        template = template.replaceAll(templatePlaceholder, replacementValue);
+    }
+    return template;
+};
+
+hortis.findDuplicates = function (array) {
+    return array.filter(function (item, index) {
+        return array.indexOf(item) !== index;
+    });
+};
+
+// Taken from https://stackoverflow.com/a/64909887
+hortis.asyncMap = async function (arr, fn) {
+    const result = [];
+    for (let idx = 0; idx < arr.length; ++idx) {
+        const cur = arr[idx];
+        result.push(await fn(cur, idx, arr));
+    }
+    return result;
+};

package/src/viz/plethify.js

@@ -0,0 +1,39 @@
+"use strict";
+// This gets injected into the Leaflet app autogenerated by QGIS2web
+
+var hortis = fluid ? fluid.registerNamespace("hortis") : {};
+
+hortis.plethLevels = {
+    0: "#f7fcfd",
+    5: "#e5f5f9",
+    10: "#ccece6",
+    20: "#99d8c9",
+    50: "#66c2a4",
+    100: "#41ae76",
+    200: "#238b45",
+    500: "#005824"
+};
+
+hortis.revKeys = Object.keys(hortis.plethLevels).reverse().map(function (stringKey) {
+    return Number(stringKey);
+});
+
+hortis.getColor = function (count) {
+    for (var i = 0; i < hortis.revKeys.length; ++i) {
+        var key = hortis.revKeys[i];
+        if (count > key) {
+            return hortis.plethLevels[key];
+        }
+    }
+};
+
+hortis.plethStyle = function (feature) {
+    return {
+        fillColor: hortis.getColor(feature.properties.speciesCount),
+        weight: 1,
+        opacity: 1,
+        color: "black",
+        //dashArray: '3',
+        fillOpacity: 0.7
+    };
+};

package/src/wormify.js

@@ -0,0 +1,154 @@
+/* eslint-env node */
+
+"use strict";
+
+var fluid = require("infusion");
+var fs = require("fs");
+var kettle = require("kettle");
+var minimist = require("minimist");
+
+require("./utils/utils.js");
+require("./utils/settleStructure.js");
+require("./dataProcessing/readJSON.js");
+require("./dataProcessing/readCSV.js");
+require("./dataProcessing/readCSVwithMap.js");
+require("./dataProcessing/writeCSV.js");
+require("./dataProcessing/taxonScrape.js");
+
+require("./WoRMS/taxonAPI.js");
+
+fluid.setLogging(true);
+
+var hortis = fluid.registerNamespace("hortis");
+var taxonAPIFileBase = "data/WoRMS/taxonAPI";
+
+var source = kettle.dataSource.URL({
+    url: "https://www.marinespecies.org/rest/AphiaRecordsByName/%name?like=false&marine_only=false&offset=1",
+    termMap: {
+        name: "%name"
+    }
+});
+
+var parsedArgs = minimist(process.argv.slice(2));
+
+var mapFile = parsedArgs.map || "data/dataPaper-I-in/dataPaper-out-map.json";
+var map = hortis.readJSONSync(mapFile);
+
+var file = parsedArgs._[0] || "data/dataPaper-I-in/reintegrated.csv";
+var outFile = "reintegrated-WoRMS.csv";
+
+var resultsPromise = hortis.csvReaderWithMap({
+    inputFile: file,
+    mapColumns: map.columns
+}).completionPromise;
+
+var outMapColumns = fluid.copy(map.columns);
+
+delete outMapColumns.authority;
+outMapColumns.authority = "Authority";
+outMapColumns.WoRMSAuthority = "WoRMS Authority";
+outMapColumns.mismatch = "Authority mismatch";
+outMapColumns.accepted = "WoRMS Accepted";
+
+resultsPromise.then(function (results) {
+    var rows = results.rows;
+    console.log("Got " + rows.length + " rows");
+
+    var queue = fluid.transform(rows, function (row) {
+        return {name: row.iNaturalistTaxonName};
+    });
+    var cachePromise = hortis.queueFetchWork(queue);
+    cachePromise.then(function (cache) {
+        var newRows = rows.map(function (oneRow) {
+            var newRow = fluid.copy(oneRow);
+            var entry = cache[oneRow.iNaturalistTaxonName];
+            newRow.WoRMSAuthority = entry ? (entry.authority ? entry.authority : entry.message) : "Internal error";
+            var authority = newRow.subtaxonAuthority || newRow.authority;
+            newRow.mismatch = newRow.WoRMSAuthority === authority ? "" : "x";
+            newRow.accepted = entry && entry.status;
+            return newRow;
+        });
+        hortis.writeCSV(outFile, outMapColumns, newRows, fluid.promise());
+    });
+});
+
+// Values of "status" field in decreasing order of desirability
+hortis.WoRMSstatus = ["accepted", "unaccepted", "deleted"];
+
+hortis.WoRMSstatusIndex = function (status) {
+    var index = hortis.WoRMSstatus.indexOf(status);
+    return index === -1 ? hortis.WoRMSstatus.length : index;
+};
+
+hortis.findBestWoRMSDoc = function (docs) {
+    var byIndex = [];
+    fluid.each(docs, function (doc) {
+        var index = hortis.WoRMSstatusIndex(doc.status);
+        fluid.pushArray(byIndex, index, doc);
+    });
+    return byIndex.find(function (slot) {
+        return slot && slot.length > 0;
+    });
+};
+
+hortis.queueFetchWork = function (queue) {
+    var that = {
+        queue: queue,
+        cache: {},
+        skipCount: 0,
+        fetched: [],
+        completion: fluid.promise()
+    };
+    var oneWork = function () {
+        if (that.queue.length) {
+            var head = that.queue.shift();
+            var name = head.name;
+            var filename = hortis.WoRMSTaxa.filenameFromTaxonName(taxonAPIFileBase, name);
+            var doc;
+            if (that.cache[name] || fs.existsSync(filename)) {
+                // hortis.noteSkip(that, "File " + filename + " exists, skipping");
+                doc = that.cache[filename];
+                if (!doc) {
+                    doc = hortis.readJSONSync(filename);
+                    that.cache[name] = doc;
+                }
+                // hortis.enqueueAncestry(doc, that.queue);
+                fluid.invokeLater(oneWork);
+            } else {
+                doc = source.get(head).then(function (docs) {
+                    that.fetched.push(head);
+                    var sortedDocs = hortis.findBestWoRMSDoc(docs);
+                    if (!sortedDocs || sortedDocs.length === 0) {
+                        console.log("Received no results for name " + head.name);
+                        that.cache[name] = {
+                            isError: true,
+                            message: "Document not found"
+                        };
+                    } else {
+                        if (sortedDocs.length > 1) {
+                            console.log("Warning: received " + sortedDocs.length + " equally good results for name " + head.name);
+                        }
+                        // hortis.enqueueAncestry(doc, that.queue);
+                        hortis.writeTaxonDoc(filename, sortedDocs[0]);
+                        that.cache[name] = sortedDocs[0];
+                    }
+                    // that.completion.resolve(that.cache);
+                    nextWork();
+                }, function (err) {
+                    console.log("Received ERROR for name " + head.name, err);
+                    that.cache[name] = {
+                        isError: true,
+                        message: err
+                    };
+                });
+            }
+        } else {
+            that.completion.resolve(that.cache);
+        }
+    };
+    var nextWork = function () {
+        setTimeout(oneWork, 1000);
+    };
+    nextWork();
+    return that.completion;
+};

package/src/xetthecum/xetthecumFilters.js

@@ -0,0 +1,126 @@
+/* eslint-env node */
+
+"use strict";
+
+var fluid = require("infusion");
+var hortis = fluid.registerNamespace("hortis");
+
+require("../dataProcessing/writeCSV.js");
+
+fluid.registerNamespace("hortis.xetthecum");
+
+hortis.xetthecum.names = [{
+    field: "hulqThom",
+    auth: "(Thom, 2011)"
+}, {
+    field: "hulqTurner1",
+    auth: "(Turner, 2014)"
+},  {
+    field: "hulqTurner2",
+    auth: "(Turner, 2014)"
+}];
+
+hortis.xetthecum.valueOutMap = {
+    columns: {
+        "foodValue": "Food Value",
+        "medicinalValue": "Medicinal Value",
+        "spiritualValue": "Spiritual Value",
+        "materialValue": "Material Value",
+        "tradeValue": "Trade Value",
+        "indicatorValue": "Indicator Value"
+    }
+};
+
+hortis.xetthecum.extraOutMap = {
+    columns: {
+        "hulqName": "Hulquminum Name",
+        "hulqAuth": "Hulquminum Authority"
+    }
+};
+
+hortis.xetthecum.mediaOutMap = {
+    columns: {
+        "audioLink": "Audio Link"
+    }
+};
+
+hortis.xetthecum.chooseName = function (target, hulqRow) {
+    var bestRec = hortis.xetthecum.names.find(function (rec) {
+        return !!hulqRow[rec.field];
+    });
+    if (bestRec) {
+        target.hulqName = hulqRow[bestRec.field];
+        target.hulqAuth = bestRec.auth;
+        return true;
+    } else {
+        return false;
+    }
+};
+
+hortis.xetthecum.summariseUnseen = function (byTaxon, usedTaxa, message) {
+    var unseen = [];
+    fluid.each(byTaxon, function (value, key) {
+        if (!usedTaxa[key]) {
+            unseen.push(key);
+        }
+    });
+    console.log("The following ", unseen.length, " taxa with " + message + " were not observed:\n" + unseen.join("\n"));
+};
+
+hortis.xetthecum.assignHulqNames = function (resolved, patch) {
+    var byTaxon = {};
+    var usedTaxa = {};
+    var withoutHulq = [];
+    var hits = 0;
+    var valueFields = Object.keys(hortis.xetthecum.valueOutMap.columns);
+    patch.patchData.rows.forEach(function (row) {
+        byTaxon[row.scientificName] = row;
+    });
+    resolved.summarisedRows.forEach(function (row) {
+        var taxon = row.iNaturalistTaxonName;
+        var hulqRow = byTaxon[taxon];
+        if (!hulqRow) {
+            withoutHulq.push({
+                taxon: taxon,
+                found: "no record"
+            });
+        } else {
+            usedTaxa[taxon] = true;
+            var transfer = fluid.filterKeys(hulqRow, valueFields);
+            var foundName = hortis.xetthecum.chooseName(transfer, hulqRow);
+            if (foundName) {
+                ++hits;
+            } else {
+                withoutHulq.push({
+                    taxon: taxon,
+                    found: "no name"
+                });
+            }
+            fluid.extend(row, transfer);
+        }
+    });
+    console.log("Matched " + hits + " taxa with Hul'qumi'num names");
+
+    hortis.xetthecum.summariseUnseen(byTaxon, usedTaxa, "Hul'qumi'num name records");
+    hortis.writeCSV("hulq-mismatches.csv", ["taxon", "found"], withoutHulq, fluid.promise());
+    resolved.combinedOutMap = hortis.combineMaps([resolved.combinedOutMap, hortis.xetthecum.valueOutMap, hortis.xetthecum.extraOutMap]);
+};
+
+hortis.xetthecum.assignHulqMedia = function (resolved, patch) {
+    var byTaxon = {};
+    var usedTaxa = {};
+    patch.patchData.rows.forEach(function (row) {
+        var sanitized = hortis.sanitizeSpeciesName(row.scientificName);
+        byTaxon[sanitized] = row;
+    });
+    resolved.summarisedRows.forEach(function (row) {
+        var taxon = row.iNaturalistTaxonName;
+        var hulqRow = byTaxon[taxon];
+        if (hulqRow) {
+            usedTaxa[taxon] = true;
+            row.audioLink = hulqRow.audioLink;
+        }
+    });
+    resolved.combinedOutMap = hortis.combineMaps([resolved.combinedOutMap, hortis.xetthecum.mediaOutMap]);
+    hortis.xetthecum.summariseUnseen(byTaxon, usedTaxa, "Hul'qumi'num media data");
+};