imerss-bioinfo 1.0.0

package/src/filterFirst.js

@@ -0,0 +1,51 @@
+/* eslint-env node */
+
+"use strict";
+
+var fluid = require("infusion");
+var minimist = require("minimist");
+
+require("./dataProcessing/readCSV.js");
+require("./dataProcessing/summarise.js");
+
+var hortis = fluid.registerNamespace("hortis");
+
+fluid.defaults("hortis.summarise.csvReader", {
+    gradeNames: "hortis.summarise",
+    outputFile: null,
+    listeners: {
+        "onHeaders.validateHeaders": {
+            funcName: "hortis.summarise.validateHeaders",
+            args: ["{that}.headers", "{that}.options.dateField", "{that}.options.uniqueField", "{that}.events.onError"]
+        },
+        "onRow.storeRow": "{that}.storeRow",
+        "onCompletion.writeCSV": {
+            funcName: "hortis.imerss.writeCSV",
+            args: ["{that}.options.outputFile", "{that}.headers", "{that}.uniqueRows", "{that}.completionPromise"]
+        }
+    }
+});
+
+hortis.summarise.validateHeaders = function (headers, dateField, uniqueField, onError) {
+    if (!headers.includes(dateField)) {
+        onError.fire("Date field " + dateField + " not found in headers ", headers);
+    }
+    if (!headers.includes(uniqueField)) {
+        onError.fire("Unique filter field " + uniqueField + " not found in headers ", headers);
+    }
+};
+
+var parsedArgs = minimist(process.argv.slice(2));
+
+var outputFile = parsedArgs.o || "filtered.csv";
+
+var that = hortis.summarise.csvReader({
+    uniqueField: "Taxon name",
+    dateField: "Date observed",
+    inputFile: "data/Galiano/Galiano_marine_species_catalogue.csv",
+    outputFile: outputFile
+});
+
+that.completionPromise.then(null, function (error) {
+    console.error(error);
+});

package/src/geom/geoJSON.js

@@ -0,0 +1,121 @@
+/* eslint-env node */
+
+"use strict";
+
+const fluid = require("infusion");
+
+require("../lib/point-in-polygon.js");
+
+const hortis = fluid.registerNamespace("hortis");
+
+hortis.countTrue = function (array) {
+    return array.reduce(function (a, c) {return a + c;}, 0);
+};
+
+hortis.intersections = 0;
+
+hortis.intersectsFeature = function (feature, mappedRow) {
+    if (!hortis.isPoint(mappedRow.point)) {
+        return false;
+    } else {
+        var reject = function (message) {
+            fluid.fail(message + " for feature ", feature);
+        };
+        var intersectPolygon = function (polyPolygon) {
+            var intersects = polyPolygon.map(function (polygon) {
+                ++hortis.intersections;
+                return hortis.pointInPolygon(mappedRow.point, polygon);
+            });
+            var count = hortis.countTrue(intersects);
+            return count % 2;
+        };
+        var geometry = feature.geometry;
+        // Stupid export of Galiano polygon has the latter type
+        if (geometry.type !== "MultiPolygon" && geometry.type !== "MultiLineString") {
+            reject("Cannot handle feature type " + geometry.type);
+        } else {
+            var coords = geometry.coordinates;
+            var intersects = false;
+            if (geometry.type === "MultiLineString") {
+                intersects = intersectPolygon(coords);
+            } else {
+                intersects = coords.some(function (polyPolygon) {
+                    return intersectPolygon(polyPolygon);
+                });
+            }
+            return intersects;
+        }
+    }
+};
+
+hortis.intersectsAnyFeature = function (features, row) {
+    const intersects = features.map(function (feature) {
+        return hortis.intersectsFeature(feature, row);
+    });
+    const intersectCount = hortis.countTrue(intersects);
+    return intersectCount > 0;
+};
+
+hortis.rowToLoggable = function (row) {
+    return fluid.filterKeys(row, ["iNaturalistTaxonName", "observationId", "placeName", "latitude", "longitude"]);
+};
+
+hortis.makeFeatureRowFilter = function (feature, options) {
+    options = options || {};
+    return function (rows) {
+        let rejections = 0;
+        const togo = rows.filter(function (row) {
+            row.point = [hortis.parseFloat(row.longitude), hortis.parseFloat(row.latitude)];
+            const accept = hortis.intersectsAnyFeature(feature, row);
+            if (!accept) {
+                ++rejections;
+                if (options.logRejection) {
+                    const tolog = hortis.rowToLoggable(row);
+                    console.log("Rejecting observation ", tolog, " as lying outside polygon " + options.featureName);
+                }
+            }
+            return accept;
+        });
+        console.log("Rejecting " + rejections + " observations as lying outside polygon " + options.featureName);
+        return togo;
+    };
+};
+
+hortis.processRegionFilter = function (resolved, patch, key) {
+    const filter = hortis.makeFeatureRowFilter(patch.patchData.features, {
+        logRejection: patch.logRejection,
+        featureName: key
+    });
+    // TODO: MATT-based immutable filtering!
+    resolved.obsRows = filter(resolved.obsRows);
+};
+
+hortis.processAssignFeature = function (resolved, patch) {
+    const extraCols = {};
+    const features = patch.patchData.features;
+    const last = Date.now();
+    resolved.obsRows.forEach(function (row) {
+        row.point = [hortis.parseFloat(row.longitude), hortis.parseFloat(row.latitude)];
+        const intersects = features.filter(function (feature) {
+            return hortis.intersectsFeature(feature, row);
+        });
+        if (intersects.length === 0) {
+            console.log("Warning: row ", hortis.rowToLoggable(row), " did not intersect any feature");
+        } else {
+            if (intersects.length > 1) {
+                const allProps = fluid.getMembers(intersects, "properties");
+                console.log("Warning: row ", hortis.rowToLoggable(row), " intersected multiple features: ", allProps);
+            }
+            const props = intersects[0].properties;
+            fluid.extend(row, props);
+            fluid.extend(extraCols, props);
+        }
+    });
+    console.log("Intersected " + resolved.obsRows.length + " observations in " + (Date.now() - last) + " ms");
+    const extraOutMap = {
+        columns: fluid.transform(extraCols, function (junk, key) {
+            return key;
+        })
+    };
+    resolved.combinedObsOutMap = hortis.combineMaps([resolved.combinedObsOutMap, extraOutMap], true);
+};

package/src/iNaturalist/examples/api.inaturalist.org-v1-taxa!q=Acmispon americanus americanus.json

@@ -0,0 +1,200 @@
+{
+    "total_results": 4,
+    "page": 1,
+    "per_page": 30,
+    "results": [
+        {
+            "observations_count": 3409,
+            "taxon_schemes_count": 2,
+            "ancestry": "48460/47126/211194/47125/47124/47123/47122/507502/533443/64336",
+            "is_active": true,
+            "flag_counts": {
+                "unresolved": 0,
+                "resolved": 0
+            },
+            "wikipedia_url": "http://en.wikipedia.org/wiki/Acmispon_americanus",
+            "current_synonymous_taxon_ids": null,
+            "iconic_taxon_id": 47126,
+            "rank_level": 10,
+            "taxon_changes_count": 3,
+            "atlas_id": null,
+            "complete_species_count": null,
+            "parent_id": 64336,
+            "name": "Acmispon americanus",
+            "rank": "species",
+            "extinct": false,
+            "id": 57049,
+            "default_photo": {
+                "square_url": "https://inaturalist-open-data.s3.amazonaws.com/photos/78383082/square.jpeg",
+                "attribution": "(c) Jim Frisinger, some rights reserved (CC BY-NC), uploaded by Jim Frisinger",
+                "flags": [],
+                "medium_url": "https://inaturalist-open-data.s3.amazonaws.com/photos/78383082/medium.jpeg",
+                "id": 78383082,
+                "license_code": "cc-by-nc",
+                "original_dimensions": {
+                    "width": 987,
+                    "height": 1318
+                },
+                "url": "https://inaturalist-open-data.s3.amazonaws.com/photos/78383082/square.jpeg"
+            },
+            "ancestor_ids": [
+                48460,
+                47126,
+                211194,
+                47125,
+                47124,
+                47123,
+                47122,
+                507502,
+                533443,
+                64336,
+                57049
+            ],
+            "matched_term": "Acmispon americanus",
+            "iconic_taxon_name": "Plantae",
+            "preferred_common_name": "Spanish clover"
+        },
+        {
+            "observations_count": 236,
+            "taxon_schemes_count": 3,
+            "ancestry": "48460/47126/211194/47125/47124/47123/47122/507502/533443/64336/57049",
+            "is_active": true,
+            "flag_counts": {
+                "unresolved": 0,
+                "resolved": 0
+            },
+            "wikipedia_url": "http://en.wikipedia.org/wiki/Acmispon_americanus",
+            "current_synonymous_taxon_ids": null,
+            "iconic_taxon_id": 47126,
+            "rank_level": 5,
+            "taxon_changes_count": 2,
+            "atlas_id": null,
+            "complete_species_count": null,
+            "parent_id": 57049,
+            "name": "Acmispon americanus americanus",
+            "rank": "variety",
+            "extinct": false,
+            "id": 57053,
+            "default_photo": {
+                "square_url": "https://inaturalist-open-data.s3.amazonaws.com/photos/997642/square.jpg",
+                "attribution": "(c) Todd Plummer, some rights reserved (CC BY-NC-SA)",
+                "flags": [],
+                "medium_url": "https://inaturalist-open-data.s3.amazonaws.com/photos/997642/medium.jpg",
+                "id": 997642,
+                "license_code": "cc-by-nc-sa",
+                "original_dimensions": {
+                    "width": 2048,
+                    "height": 1568
+                },
+                "url": "https://inaturalist-open-data.s3.amazonaws.com/photos/997642/square.jpg"
+            },
+            "ancestor_ids": [
+                48460,
+                47126,
+                211194,
+                47125,
+                47124,
+                47123,
+                47122,
+                507502,
+                533443,
+                64336,
+                57049,
+                57053
+            ],
+            "matched_term": "Acmispon americanus americanus",
+            "iconic_taxon_name": "Plantae",
+            "preferred_common_name": "Spanish clover"
+        },
+        {
+            "observations_count": 9,
+            "taxon_schemes_count": 1,
+            "ancestry": "48460/47126/211194/47125/47124/47123/47122/507502/533443/64336",
+            "is_active": true,
+            "flag_counts": {
+                "unresolved": 0,
+                "resolved": 0
+            },
+            "wikipedia_url": null,
+            "current_synonymous_taxon_ids": null,
+            "iconic_taxon_id": 47126,
+            "rank_level": 10,
+            "taxon_changes_count": 1,
+            "atlas_id": null,
+            "complete_species_count": null,
+            "parent_id": 64336,
+            "name": "Acmispon helleri",
+            "rank": "species",
+            "extinct": false,
+            "id": 311602,
+            "default_photo": {
+                "square_url": "https://static.inaturalist.org/photos/51103129/square.jpg",
+                "attribution": "(c) Lenny Lampel, all rights reserved, uploaded by Lenny Lampel",
+                "flags": [],
+                "medium_url": "https://static.inaturalist.org/photos/51103129/medium.jpg",
+                "id": 51103129,
+                "license_code": null,
+                "original_dimensions": {
+                    "width": 2048,
+                    "height": 1535
+                },
+                "url": "https://static.inaturalist.org/photos/51103129/square.jpg"
+            },
+            "ancestor_ids": [
+                48460,
+                47126,
+                211194,
+                47125,
+                47124,
+                47123,
+                47122,
+                507502,
+                533443,
+                64336,
+                311602
+            ],
+            "matched_term": "Acmispon americanus helleri",
+            "iconic_taxon_name": "Plantae",
+            "preferred_common_name": "Carolina prairie trefoil"
+        },
+        {
+            "observations_count": 1,
+            "taxon_schemes_count": 1,
+            "ancestry": "48460/47126/211194/47125/47124/47123/47122/507502/533443/64336/57049",
+            "is_active": true,
+            "flag_counts": {
+                "unresolved": 0,
+                "resolved": 0
+            },
+            "wikipedia_url": "http://en.wikipedia.org/wiki/Acmispon_americanus",
+            "current_synonymous_taxon_ids": null,
+            "iconic_taxon_id": 47126,
+            "rank_level": 5,
+            "taxon_changes_count": 2,
+            "atlas_id": null,
+            "complete_species_count": null,
+            "parent_id": 57049,
+            "name": "Acmispon americanus helleri",
+            "rank": "variety",
+            "extinct": false,
+            "id": 57052,
+            "default_photo": null,
+            "ancestor_ids": [
+                48460,
+                47126,
+                211194,
+                47125,
+                47124,
+                47123,
+                47122,
+                507502,
+                533443,
+                64336,
+                57049,
+                57052
+            ],
+            "matched_term": "Acmispon americanus helleri",
+            "iconic_taxon_name": "Plantae"
+        }
+    ]
+}

package/src/iNaturalist/examples/api.inaturalist.org-v1-taxa!q=Flabellina japonica.json

@@ -0,0 +1,61 @@
+{
+    "total_results":1,
+    "page":1,
+    "per_page":30,
+    "results":[
+        {
+            "observations_count":11,
+            "taxon_schemes_count":1,
+            "ancestry":"48460/1/47115/47114/551391/775797/775799/775801/47113/801475/500752/603234/603236",
+            "is_active":true,
+            "flag_counts":{
+                "unresolved":0,
+                "resolved":0
+            },
+            "wikipedia_url":"https://en.wikipedia.org/wiki/Ziminella_japonica",
+            "current_synonymous_taxon_ids":null,
+            "iconic_taxon_id":47115,
+            "rank_level":10,
+            "taxon_changes_count":1,
+            "atlas_id":null,
+            "complete_species_count":null,
+            "parent_id":603236,
+            "name":"Ziminella japonica",
+            "rank":"species",
+            "extinct":false,
+            "id":877359,
+            "default_photo":{
+                "square_url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/square.jpeg",
+                "attribution":"(c) Sara Thiebaud, some rights reserved (CC BY-NC)",
+                "flags":[
+                ],
+                "medium_url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/medium.jpeg",
+                "id":88409951,
+                "license_code":"cc-by-nc",
+                "original_dimensions":{
+                    "width":1886,
+                    "height":1084
+                },
+                "url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/square.jpeg"
+            },
+            "ancestor_ids":[
+                48460,
+                1,
+                47115,
+                47114,
+                551391,
+                775797,
+                775799,
+                775801,
+                47113,
+                801475,
+                500752,
+                603234,
+                603236,
+                877359
+            ],
+            "matched_term":"Flabellina japonica",
+            "iconic_taxon_name":"Mollusca"
+        }
+    ]
+}

package/src/iNaturalist/examples/www.inaturalist.org-taxa-877359.json

@@ -0,0 +1,132 @@
+{
+    "id":877359,
+    "name":"Ziminella japonica",
+    "rank":"species",
+    "source_id":15907,
+    "created_at":"2019-03-28T02:07:43.786Z",
+    "updated_at":"2021-10-29T21:05:17.230Z",
+    "iconic_taxon_id":47115,
+    "is_iconic":false,
+    "name_provider":null,
+    "observations_count":11,
+    "listed_taxa_count":32,
+    "rank_level":10.0,
+    "wikipedia_summary":"\u003ci\u003e\u003cb\u003eZiminella japonica\u003c/b\u003e\u003c/i\u003e is a species of sea slug, an aeolid nudibranch, a marine heterobranch mollusc in the family Paracoryphellidae.",
+    "wikipedia_title":"",
+    "ancestry":"48460/1/47115/47114/551391/775797/775799/775801/47113/801475/500752/603234/603236",
+    "is_active":true,
+    "taxon_framework_relationship_id":551743,
+    "uuid":"6552ecc9-0a88-4c15-8804-d0703da9b2c8",
+    "photos_locked":false,
+    "default_name":{
+        "id":1537885,
+        "name":"Ziminella japonica",
+        "is_valid":true,
+        "lexicon":"Scientific Names"
+    },
+    "photo_url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/square.jpeg",
+    "iconic_taxon_name":"Mollusca",
+    "conservation_status_name":null,
+    "common_name":null,
+    "image_url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/square.jpeg",
+    "taxon_range_kml_url":null,
+    "html":null,
+    "default_photo":{
+        "id":88409951,
+        "user_id":3418777,
+        "native_photo_id":null,
+        "created_at":"2020-08-05T18:40:31.068Z",
+        "updated_at":"2020-08-05T18:40:31.068Z",
+        "native_page_url":null,
+        "native_username":null,
+        "native_realname":null,
+        "license":2,
+        "subtype":null,
+        "native_original_image_url":null,
+        "uuid":"75d5407f-aa9e-4c0d-9dcd-5028a9a0e09f",
+        "file_extension_id":2,
+        "file_prefix_id":1,
+        "width":1886,
+        "height":1084,
+        "license_code":"CC-BY-NC",
+        "license_name":"Creative Commons Attribution-NonCommercial License",
+        "license_url":"http://creativecommons.org/licenses/by-nc/4.0/",
+        "attribution":"(c) Sara Thiebaud, some rights reserved (CC BY-NC), uploaded by Sara Thiebaud",
+        "type":"LocalPhoto",
+        "square_url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/square.jpeg",
+        "thumb_url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/thumb.jpeg",
+        "small_url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/small.jpeg",
+        "medium_url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/medium.jpeg",
+        "large_url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/large.jpeg"
+    },
+    "created_at_utc":"2019-03-28T02:07:43.786Z",
+    "updated_at_utc":"2021-10-29T21:05:17.230Z",
+    "taxon_photos":[
+        {
+            "id":944035,
+            "taxon_id":877359,
+            "photo_id":88409951,
+            "position":0,
+            "created_at":"2020-08-05T18:54:35.242Z",
+            "updated_at":"2020-08-05T18:54:35.242Z",
+            "photo":{
+                "id":88409951,
+                "user_id":3418777,
+                "native_photo_id":null,
+                "created_at":"2020-08-05T18:40:31.068Z",
+                "updated_at":"2020-08-05T18:40:31.068Z",
+                "native_page_url":null,
+                "native_username":null,
+                "native_realname":null,
+                "license":2,
+                "subtype":null,
+                "native_original_image_url":null,
+                "uuid":"75d5407f-aa9e-4c0d-9dcd-5028a9a0e09f",
+                "file_extension_id":2,
+                "file_prefix_id":1,
+                "width":1886,
+                "height":1084,
+                "license_code":"CC-BY-NC",
+                "attribution":"(c) Sara Thiebaud, some rights reserved (CC BY-NC), uploaded by Sara Thiebaud",
+                "square_url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/square.jpeg",
+                "thumb_url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/thumb.jpeg",
+                "small_url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/small.jpeg",
+                "medium_url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/medium.jpeg",
+                "large_url":"https://inaturalist-open-data.s3.amazonaws.com/photos/88409951/large.jpeg",
+                "license_name":"Creative Commons Attribution-NonCommercial License",
+                "license_url":"http://creativecommons.org/licenses/by-nc/4.0/",
+                "type":"LocalPhoto"
+            }
+        }
+    ],
+    "taxon_names":[
+        {
+            "id":1537885,
+            "name":"Ziminella japonica",
+            "is_valid":true,
+            "lexicon":"Scientific Names",
+            "taxon_id":877359,
+            "created_at":"2019-03-28T02:07:43.786Z",
+            "updated_at":"2019-03-28T02:07:43.786Z",
+            "creator_id":26610,
+            "position":0,
+            "parameterized_lexicon":"scientific-names"
+        },
+        {
+            "id":1537887,
+            "name":"Flabellina japonica",
+            "is_valid":false,
+            "lexicon":"Scientific Names",
+            "taxon_id":877359,
+            "created_at":"2019-03-28T02:13:42.092Z",
+            "updated_at":"2019-03-28T02:13:42.092Z",
+            "creator_id":1,
+            "position":1,
+            "parameterized_lexicon":"scientific-names"
+        }
+    ],
+    "iconic_taxon":{
+        "id":47115,
+        "name":"Mollusca"
+    }
+}

package/src/iNaturalist/iNatUrls.js

@@ -0,0 +1,17 @@
+/* eslint-env node */
+
+"use strict";
+
+const fluid = require("infusion");
+
+const hortis = fluid.registerNamespace("hortis");
+
+hortis.iNaturalistObsFromUrl = function (url) {
+    const lastSlashPos = url.lastIndexOf("/");
+    return url.substring(lastSlashPos + 1);
+};
+
+// Currently disused
+hortis.iNaturalistIdToLink = function (id) {
+    return "https://www.inaturalist.org/taxa/" + id;
+};