imerss-bioinfo 1.0.0

package/src/inatObs.js

@@ -0,0 +1,120 @@
+/* eslint-env node */
+
+"use strict";
+
+const fluid = require("infusion");
+const minimist = require("minimist");
+const fs = require("fs");
+
+require("./dataProcessing/readJSON.js");
+require("./dataProcessing/writeJSON.js");
+require("./dataProcessing/writeCSV.js");
+
+require("./iNaturalist/obsAPI.js");
+
+const hortis = fluid.registerNamespace("hortis");
+
+hortis.iNatProjects = {
+    GalianoData: {
+        paramMap: {
+            project_id: 5799,
+            taxon_id: 1
+        },
+        outputFile: "data/iNaturalist/Galiano_Catalogue_Animalia_%Date.csv"
+    },
+    Valdes: {
+        paramMap: {
+            project_id: 43522,
+            taxon_id: 48460 // Life
+        },
+        outputFile: "data/Valdes/iNaturalist_Catalogue_%Date.csv"
+    },
+    Xetthecum: {
+        paramMap: {
+            project_id: "xetthecum",
+            taxon_id: 48460 // Life
+        },
+        outputFile: "data/Xetthecum/iNaturalist_Catalogue_%Date.csv"
+    },
+    Pepiowelh: {
+        paramMap: {
+            project_id: "pe-pi-ow-elh",
+            taxon_id: 48460 // Life
+        },
+        outputFile: "data/Pepiowelh/iNaturalist_Catalogue_%Date.csv"
+    }
+};
+
+const parsedArgs = minimist(process.argv.slice(2));
+
+console.log("parsedArgs", parsedArgs);
+
+const projectArgs = hortis.iNatProjects[parsedArgs._[0] || "GalianoData"];
+
+const jwt = hortis.readJSONSync("jwt.json", "reading JWT token file");
+
+const source = hortis.iNat.obsSource({
+    headers: {
+        Authorization: "Bearer " + jwt.api_token
+    },
+    paramMap: projectArgs.paramMap
+});
+
+const fileVars = {
+    Date: new Date().toISOString().substring(0, 10).replaceAll("-", "_")
+};
+
+fluid.setLogging(true);
+
+const rows = [];
+
+const directModel = {
+    per_page: 200
+};
+
+hortis.logObsResponse = function (data) {
+    const tolog = Object.assign({}, data);
+    tolog.results = "[ " + data.results.length + " ]";
+    fluid.log("Got response " + JSON.stringify(tolog, null, 4));
+};
+
+hortis.writeObs = function (filename, rows) {
+    const togo = fluid.promise();
+    const headers = Object.keys(rows[0]);
+    hortis.writeCSV(filename, headers, rows, togo);
+    return togo;
+};
+
+hortis.applyResponse = function (data) {
+    hortis.logObsResponse(data);
+    hortis.writeJSONSync("obsoutput.json", data);
+    if (data.results.length > 0) {
+        hortis.pushResultRows(rows, data);
+        const lastId = fluid.peek(rows).id;
+        console.log("got last id " + lastId);
+        directModel.id_above = lastId;
+        setTimeout(function () {
+            hortis.makeObsRequest(directModel);
+        }, 1000);
+    } else {
+        hortis.writeObs("obsoutput.csv", rows).then(function () {
+            const fileTarget = fluid.stringTemplate(projectArgs.outputFile, fileVars);
+            fs.copyFileSync("obsoutput.csv", fileTarget);
+            const stats = fs.statSync(fileTarget);
+            console.log("Written " + stats.size + " bytes to " + fileTarget);
+        });
+    }
+};
+
+hortis.makeObsRequest = function (directModel) {
+    console.log("Making request ", directModel);
+    const promise = source.get(directModel);
+
+    promise.then(function (data) {
+        hortis.applyResponse(data);
+    }, function (error) {
+        console.log("Got ERROUR: ", error);
+    });
+};
+
+hortis.makeObsRequest(directModel);

package/src/inattify.js

@@ -0,0 +1,123 @@
+/* eslint-env node */
+
+"use strict";
+
+// This script is now obsoleted by the new-style "taxonomise.js" using taxonAPI.js to cache results within the SQLite database
+// %imerss-bioinfo/data/iNaturalist/taxa.db/taxa.sqlite3 - will be kept for a while for reference purposes
+
+const fluid = require("infusion");
+const fs = require("fs");
+const kettle = require("kettle");
+const minimist = require("minimist");
+
+require("./utils/utils.js");
+require("./utils/settleStructure.js");
+require("./dataProcessing/readJSON.js");
+require("./dataProcessing/readCSV.js");
+require("./dataProcessing/readCSVwithMap.js");
+require("./dataProcessing/taxonScrape.js");
+
+require("./iNaturalist/taxonAPI.js");
+
+const hortis = fluid.registerNamespace("hortis");
+const taxonAPIFileBase = "data/iNaturalist/taxonAPI";
+
+const source = kettle.dataSource.URL({
+    url: "https://www.inaturalist.org/taxa/%id.json",
+    termMap: {
+        id: "%id"
+    }
+});
+
+const parsedArgs = minimist(process.argv.slice(2));
+
+const mapFile = parsedArgs.map || "data/Galiano WoL/Galiano-data-out-map.json";
+const map = hortis.readJSONSync(mapFile);
+
+const files = parsedArgs._;
+
+const readFiles = files.map(function (file) {
+    return hortis.csvReaderWithMap({
+        inputFile: file,
+        mapColumns: map.columns
+    }).completionPromise;
+});
+
+const resultsPromise = hortis.settleStructure(readFiles);
+
+resultsPromise.then(function (results) {
+    const allRows = fluid.flatten(fluid.transform(results, function (oneResult) {
+        return oneResult.rows;
+    }));
+
+    console.log("Got " + allRows.length + " rows");
+
+    console.log(allRows[0]);
+    const queue = fluid.transform(allRows, function (row) {
+        return {id: row.iNaturalistTaxonId};
+    });
+    hortis.queueFetchWork(queue);
+});
+
+// fluid.setLogging(true);
+
+hortis.enqueueAncestry = function (doc, queue) {
+    const ancestours = hortis.iNat.parentTaxaIds(doc);
+    const elements = ancestours.map(function (oneAnc) {
+        return {id: oneAnc};
+    });
+    queue.unshift.apply(queue, elements);
+};
+
+hortis.logWork = function (that) {
+    console.log("Remaining " + that.queue.length + "/" + (that.queue.length + that.fetched.length));
+};
+
+hortis.noteSkip = function (that, message) {
+    ++that.skipCount;
+    if (that.skipCount % 50000 === 0) {
+        console.log(message);
+        hortis.logWork(that);
+    }
+};
+
+hortis.queueFetchWork = function (queue) {
+    const nextWork = function () {
+        hortis.logWork(that);
+        setTimeout(oneWork, 1000);
+    };
+    const that = {
+        queue: queue,
+        cache: {},
+        skipCount: 0,
+        fetched: []
+    };
+    const oneWork = function () {
+        if (that.queue.length) {
+            const head = that.queue.shift();
+            const filename = hortis.iNat.filenameFromTaxonId(taxonAPIFileBase, head.id);
+            let doc;
+            if (that.cache[filename] || fs.existsSync(filename)) {
+                hortis.noteSkip(that, "File " + filename + " exists, skipping");
+                doc = that.cache[filename];
+                if (!doc) {
+                    doc = hortis.readJSONSync(filename);
+                    that.cache[filename] = doc;
+                }
+                hortis.enqueueAncestry(doc, that.queue);
+                fluid.invokeLater(oneWork);
+            } else {
+                doc = source.get(head).then(function (doc) {
+                    that.fetched.push(head);
+                    hortis.enqueueAncestry(doc, that.queue);
+                    hortis.writeTaxonDoc(filename, doc);
+                    that.cache[filename] = doc;
+                    nextWork();
+                }, function (err) {
+                    console.log("Received ERROR for id " + head.id, err);
+                });
+            }
+        }
+    };
+    nextWork();
+};

package/src/intersect.js

@@ -0,0 +1,80 @@
+/* eslint-env node */
+
+"use strict";
+
+var fluid = require("infusion");
+var fs = require("fs");
+
+require("./dataProcessing/readCSV.js");
+require("./dataProcessing/readCSVwithMap.js");
+require("./lib/point-in-polygon.js");
+require("./geom/geoJSON.js");
+
+var hortis = fluid.registerNamespace("hortis");
+fluid.registerNamespace("hortis.intersect");
+
+var protected_areas = require("../data/Galiano/Protected_Areas_2013_2.json");
+
+var map = require("../data/iNaturalist-obs-map.json");
+
+var completionPromise = hortis.imerss.csvReader({
+    inputFile: "data/Galiano/observations-iNat.csv",
+    mapColumns: map.columns
+}).completionPromise;
+
+hortis.countUniqueInFeature = function (counts, feature, mappedRows) {
+    mappedRows.forEach(function (row) {
+        if (hortis.intersectsFeature(feature, row)) {
+            if (counts[row.id]) {
+                counts[row.id]++;
+            } else {
+                counts[row.id] = 1;
+            }
+        }
+    });
+};
+
+hortis.intersect.mapOneiNaturalistRow = function (row) {
+    return {
+        id: row.scientificName,
+        point: [
+            row.longitude,
+            row.latitude
+        ]
+    };
+};
+
+var fileHeader = "var json_Protected_Areas_2013_2 = {\n\"type\": \"FeatureCollection\",\n\"name\": \"Protected_Areas_2013_2\",\n\"crs\": { \"type\": \"name\""
+    + ", \"properties\": { \"name\": \"urn:ogc:def:crs:OGC:1.3:CRS84\" } },\n\"features\": [\n";
+var fileFooter = "\n]\n}\n";
+
+hortis.intersect.writeUpdatedFeatures = function (featuresWithCounts) {
+    var outFile = fs.openSync("Protected_Areas_2013_2_counts.js", "w");
+    fs.writeSync(outFile, fileHeader);
+    var body = featuresWithCounts.map(function (feature) {
+        return JSON.stringify(feature);
+    });
+    fs.writeSync(outFile, body.join(",\n"));
+    fs.writeSync(outFile, fileFooter);
+    fs.closeSync(outFile);
+};
+
+var start = Date.now();
+
+completionPromise.then(function (result) {
+    var read = Date.now();
+    console.log("Read " + result.rows.length + " in " + (read - start) + "ms");
+    console.log(JSON.stringify(result.rows[0], null, 2));
+    var mappedRows = result.rows.map(hortis.intersect.mapOneiNaturalistRow);
+    var featureCounts = protected_areas.features.map(function (feature) {
+        console.log("Mapping feature " + feature.properties.NAME + " id " + feature.properties.ID);
+        var counts = hortis.countUniqueInFeature(counts, feature, mappedRows);
+        return Object.keys(counts).length;
+    });
+    console.log("Mapped " + result.rows.length + " against " + protected_areas.features.length + " features in " + (Date.now() - read) + "ms");
+    console.log("Got featureCounts ", featureCounts);
+    var featuresWithCounts = protected_areas.features.map(function (feature, index) {
+        return fluid.extend(true, {}, feature, {properties: {speciesCount: featureCounts[index]}});
+    });
+    hortis.intersect.writeUpdatedFeatures(featuresWithCounts);
+});

package/src/intersectFilter.js

@@ -0,0 +1,48 @@
+/* eslint-env node */
+
+"use strict";
+
+var fluid = require("infusion");
+
+require("./dataProcessing/readCSV.js");
+require("./dataProcessing/writeCSV.js");
+require("./dataProcessing/readCSVwithoutMap.js");
+require("./lib/point-in-polygon.js");
+require("./geom/geoJSON.js");
+
+var hortis = fluid.registerNamespace("hortis");
+fluid.registerNamespace("hortis.intersect");
+
+var polygon = require("../data/Galiano/Galiano-boundary.json");
+
+var completionPromise = hortis.imerss.csvReaderWithoutMap({
+    inputFile: process.argv[2],
+    csvOptions: {
+        separator: "\t",
+        quote: "\0",
+        escape: "\0"
+    }
+}).completionPromise;
+
+hortis.intersect.mapGBIFRow = function (row) {
+    return {
+        point: [
+            Number.parseFloat(row.decimalLongitude),
+            Number.parseFloat(row.decimalLatitude)
+        ]
+    };
+};
+
+completionPromise.then(function (result) {
+    console.log(JSON.stringify(result.rows[0], null, 2));
+    var feature = polygon.features[0];
+    var outs = [];
+    result.rows.forEach(function (oneRow) {
+        var mappedRow = hortis.intersect.mapGBIFRow(oneRow);
+        if (hortis.intersectsFeature(feature, mappedRow)) {
+            outs.push(oneRow);
+        }
+    });
+    console.log("Found " + outs.length + " intersecting rows");
+    hortis.imerss.writeCSV("intersected.csv", result.headers, outs, fluid.promise());
+});

package/src/landlocked.js

@@ -0,0 +1,137 @@
+/* eslint-env node */
+
+"use strict";
+
+var fluid = require("infusion");
+var minimist = require("minimist");
+fluid.require("%imerss-bioinfo");
+
+require("./utils/utils.js");
+require("./dataProcessing/readJSON.js");
+require("./dataProcessing/readCSV.js");
+require("./dataProcessing/readCSVwithMap.js");
+require("./dataProcessing/readCSVwithoutMap.js");
+require("./dataProcessing/writeCSV.js");
+require("./lib/point-in-polygon.js");
+require("./geom/geoJSON.js");
+
+var hortis = fluid.registerNamespace("hortis");
+
+fluid.setLogging(true);
+
+fluid.loadInContext(fluid.module.resolvePath("%imerss-bioinfo/data/Galiano/Galiano_map_0.js"), true);  // produces json_Galiano_map_0
+fluid.loadInContext(fluid.module.resolvePath("%imerss-bioinfo/data/Xetthecum/Site_class_4.js"), true); // produces json_Site_Class_4
+fluid.loadInContext(fluid.module.resolvePath("%imerss-bioinfo/data/Xetthecum/Trincomali_2.js"), true); // produces json_Trincomali_2
+// var xetthecum_boundary = hortis.readJSONSync(fluid.module.resolvePath("%imerss-bioinfo/data/Xetthecum/Xetthecum_Project_Boundary_WGS_84.geojson"), "Xetthecum boundary");
+
+var parsedArgs = minimist(process.argv.slice(2));
+
+var outputFile = parsedArgs.o || "landlocked.csv";
+var inputFile = parsedArgs._[0] || "%imerss-bioinfo/data/dataPaper/reintegrated-obs.csv";
+var mapFile = parsedArgs.map || "%imerss-bioinfo/data/dataPaper/combinedOutMap-obs.json";
+
+var map = hortis.readJSONSync(fluid.module.resolvePath(mapFile), "reading map file");
+
+var reader = hortis.csvReaderWithMap({
+    inputFile: fluid.module.resolvePath(inputFile),
+    mapColumns: map.columns
+});
+
+var Galiano_island = fluid.global.json_Galiano_map_0.features;
+var Xetthecum_site_class = fluid.global.json_Site_Class_4.features;
+var Trincomali = fluid.global.json_Trincomali_2.features;
+
+var Xetthecum_extract_props = ["CLASS", "SUB_CLASS"];
+var Xetthecum_id_prop = "fid";
+
+hortis.findFeatureProperties = function (allFeatures, row, properties, idprop) {
+    var features = allFeatures.filter(function (feature) {
+        return hortis.intersectsFeature(feature, row);
+    });
+    // features array of each feature which contains the row
+    var propValues = properties.map(function (propKey) {
+        return fluid.getMembers(features, ["properties", propKey]);
+    });
+    // propValues has one element for each property, element for each feature containing property value
+    var propVals = propValues.map(function (propList, index) {
+        var hash = fluid.arrayToHash(propList);
+        var keys = Object.keys(hash);
+        if (keys.length > 1) {
+            console.log("Warning: row ", row, " matched multiple features with ids ", fluid.getMembers(features, idprop),
+                " with mismatching values for property " + properties[index] + ": values were ", keys);
+        }
+        return keys[0];
+    });
+    return propVals;
+};
+
+var TrincomaliFeature = fluid.extend(true, Trincomali[0], {
+    properties: {
+        fid: 99,
+        CLASS: "Trincomali",
+        SUB_CLASS: ""
+    }
+});
+
+var allXetthecumFeatures = Xetthecum_site_class.concat(TrincomaliFeature);
+
+hortis.getXetthecumProps = function (obsRows) {
+    var now = Date.now();
+    var extractedProps = obsRows.map(function (row) {
+        row.point = [hortis.parseFloat(row.longitude), hortis.parseFloat(row.latitude)];
+        return hortis.findFeatureProperties(allXetthecumFeatures, row, Xetthecum_extract_props, Xetthecum_id_prop);
+    });
+    console.log("Evaluated " + hortis.intersections + " intersections in " + (Date.now() - now) + " ms");
+    var filtered = [];
+    obsRows.forEach(function (row, index) {
+        if (extractedProps[index][0]) {
+            var asProps = {};
+            Xetthecum_extract_props.forEach(function (prop, propIndex) {
+                asProps[prop] = extractedProps[index][propIndex];
+            });
+            filtered.push(fluid.extend(true, {}, row, asProps));
+        }
+    });
+    return filtered;
+};
+
+// NB: These are not (yet) the same as "filters" in "taxonomise"
+fluid.registerNamespace("hortis.filters");
+
+hortis.filters.Xetthecum = {
+    filter: function (rows) {
+        return hortis.getXetthecumProps(rows);
+    },
+    extraProps: Xetthecum_extract_props
+};
+
+hortis.filters.Galiano = {
+    filter: hortis.makeFeatureRowFilter(Galiano_island),
+    extraProps: []
+};
+
+var activeFilter =
+    // hortis.filters.Xetthecum;
+    hortis.filters.Galiano;
+
+reader.completionPromise.then(function () {
+    var obsRows = reader.rows;
+    console.log("Matching against " + obsRows.length + " observations");
+
+    var filtered = activeFilter.filter(obsRows);
+
+    var mapped = filtered.map(function (row) {
+        var outRow = {
+            observationId: row.observationId,
+            latitude: row.point[1],
+            longitude: row.point[0],
+            iNaturalistTaxonName: row.iNaturalistTaxonName
+        };
+        activeFilter.extraProps.forEach(function (prop) {
+            outRow[prop] = row[prop];
+        });
+        return outRow;
+    });
+    var completion = fluid.promise();
+    hortis.writeCSV(fluid.module.resolvePath(outputFile), ["observationId", "latitude", "longitude", "iNaturalistTaxonName"].concat(activeFilter.extraProps), mapped, completion);
+});

package/src/lib/jquery.binarytransport.js

@@ -0,0 +1,51 @@
+/*
+ * jquery.binarytransport.js
+ *
+ * @description. jQuery ajax transport for making binary data type requests.
+ * @version 1.0 
+ * @author Henry Algus <henryalgus@gmail.com>
+ *
+ */
+ 
+// use this transport for "binary" data type
+$.ajaxTransport("+binary", function(options, originalOptions, jqXHR){
+    // check for conditions and support for blob / arraybuffer response type
+    if (window.FormData && ((options.dataType && (options.dataType == 'binary')) || (options.data && ((window.ArrayBuffer && options.data instanceof ArrayBuffer) || (window.Blob && options.data instanceof Blob)))))
+    {
+        return {
+            // create new XMLHttpRequest
+            send: function(headers, callback){
+    // setup all variables
+                var xhr = new XMLHttpRequest(),
+    url = options.url,
+    type = options.type,
+    async = options.async || true,
+    // blob or arraybuffer. Default is blob
+    dataType = options.responseType || "blob",
+    data = options.data || null,
+    username = options.username || null,
+    password = options.password || null;
+          
+                xhr.addEventListener('load', function(){
+      var data = {};
+      data[options.dataType] = xhr.response;
+      // make callback and send data
+      callback(xhr.status, xhr.statusText, data, xhr.getAllResponseHeaders());
+                });
+ 
+                xhr.open(type, url, async, username, password);
+        
+    // setup custom headers
+    for (var i in headers ) {
+      xhr.setRequestHeader(i, headers[i] );
+    }
+        
+                xhr.responseType = dataType;
+                xhr.send(data);
+            },
+            abort: function(){
+                jqXHR.abort();
+            }
+        };
+    }
+});

package/src/lib/lz4-commentary.txt

@@ -0,0 +1,9 @@
+This implementation was grabbed from the inside of the bower package for the 0.6.0 release of 
+https://github.com/pierrec/node-lz4 since
+ - the build/lz4.js file was found to be mysteriously irrelevant, despite being updated recently
+ - the actual build shell script failed even when tested in a Linux VM on "not finding ./bindings", perhaps a node mismatch issue
+ 
+There is a sign we could revisit this at some point and produce a smaller and more reproducible decoder by mashing
+together the 3 files in "Benzinga's" https://github.com/Benzinga/lz4js
+
+Better notion: Move to standard zlib operated by pako https://github.com/nodeca/pako