imerss-bioinfo 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +156 -0
- package/build.js +196 -0
- package/docs/css/imerss-viz-all.css +1227 -0
- package/docs/css/imerss-viz-wp-overrides.css +70 -0
- package/docs/css/pepiowelh.css +76 -0
- package/docs/css/xetthecum-external.css +47 -0
- package/docs/css/xetthecum-shared.css +80 -0
- package/docs/css/xetthecum.css +526 -0
- package/docs/js/imerss-viz-all.js +58 -0
- package/docs/js/imerss-viz-all.js.map +1 -0
- package/docs/js/jquery.js +10881 -0
- package/index.html +81 -0
- package/index.js +7 -0
- package/indexWithJustMap.html +92 -0
- package/indexWithMap.html +79 -0
- package/indexWithMapHoweSound.html +91 -0
- package/indexWithMapPepiowelh.html +119 -0
- package/indexWithMapValdes.html +90 -0
- package/indexWithMapXetthecum.html +90 -0
- package/package.json +50 -0
- package/src/WoRMS/taxonAPI.js +13 -0
- package/src/arphify.js +662 -0
- package/src/assignNames.js +103 -0
- package/src/auxBuild/restoreJQuery.js +1 -0
- package/src/buildSource/Comprehensive Lists.html +42 -0
- package/src/buildSource/Data Paper Part I Visualisation.html +50 -0
- package/src/buildSource/Galiano Life List.html +43 -0
- package/src/buildSource/Metchosin/index.html +22 -0
- package/src/buildSource/Pe'pi'ow'elh.html +72 -0
- package/src/buildSource/Squamish Life List.html +48 -0
- package/src/buildSource/Valdes Island Biodiversity.html +64 -0
- package/src/buildSource/Xetthecum.html +40 -0
- package/src/buildSource/dataPaperSunburstAndMap.html +30 -0
- package/src/buildSource/index.html +224 -0
- package/src/client/css/imerss-viz-wp-overrides.css +70 -0
- package/src/client/css/imerss-viz.css +531 -0
- package/src/client/css/pepiowelh.css +76 -0
- package/src/client/css/xetthecum-external.css +47 -0
- package/src/client/css/xetthecum-shared.css +80 -0
- package/src/client/css/xetthecum.css +526 -0
- package/src/client/html/imerss-viz-map-only.html +75 -0
- package/src/client/html/imerss-viz-map-pepi.html +77 -0
- package/src/client/html/imerss-viz-map-xetthecum.html +90 -0
- package/src/client/html/imerss-viz-map.html +75 -0
- package/src/client/html/imerss-viz.html +58 -0
- package/src/client/html/importing Xetthecum patterns.txt +10 -0
- package/src/client/html/xetthecum-patterns.html +27 -0
- package/src/client/img/Animalia_2.svg +1 -0
- package/src/client/img/Animalia_Annelida.svg +655 -0
- package/src/client/img/Animalia_Chaetognatha.svg +72 -0
- package/src/client/img/Animalia_Cnidaria.svg +173 -0
- package/src/client/img/Animalia_Ctenophora.svg +298 -0
- package/src/client/img/Animalia_Nemertea.svg +54 -0
- package/src/client/img/Animalia_Phoronida.svg +271 -0
- package/src/client/img/Animalia_Platyhelminthes.svg +58 -0
- package/src/client/img/Animalia_Porifera.svg +220 -0
- package/src/client/img/Animalia_Sipuncula.svg +155 -0
- package/src/client/img/Arachnoidiscus_schmidtii_web_1-2-scaled.webp +0 -0
- package/src/client/img/Arrow-down.svg +11 -0
- package/src/client/img/Arrow-up.svg +6 -0
- package/src/client/img/Chromista.svg +1 -0
- package/src/client/img/Pepiowelh/BlackNoWe2-wb-bb.png +0 -0
- package/src/client/img/Pepiowelh/BlackNoWe2-wb.png +0 -0
- package/src/client/img/Pepiowelh/BlackNoWe2.png +0 -0
- package/src/client/img/Pepiowelh/Ecocultural Mapping Banner.svg +1 -0
- package/src/client/img/Plankton.jpg +0 -0
- package/src/client/img/Plantae.svg +1 -0
- package/src/client/img/Plantae_2.svg +1 -0
- package/src/client/img/Protista.svg +1 -0
- package/src/client/img/Valdes/Valdes Island Conservancy.jpg +0 -0
- package/src/client/img/Xetthecum/Agricultural.jpg +0 -0
- package/src/client/img/Xetthecum/Anthropogenic_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Broad_Leaf.jpg +0 -0
- package/src/client/img/Xetthecum/Caves.jpg +0 -0
- package/src/client/img/Xetthecum/Coastal_Bluffs.jpg +0 -0
- package/src/client/img/Xetthecum/Conifer_Pole_Sapling.jpg +0 -0
- package/src/client/img/Xetthecum/Developed.jpg +0 -0
- package/src/client/img/Xetthecum/Dock.jpg +0 -0
- package/src/client/img/Xetthecum/Eelgrass.jpg +0 -0
- package/src/client/img/Xetthecum/Forests_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Freshwater_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Greig_Creek.jpg +0 -0
- package/src/client/img/Xetthecum/Intertidal.jpg +0 -0
- package/src/client/img/Xetthecum/Laughlin_Lake.jpg +0 -0
- package/src/client/img/Xetthecum/Marine_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Mature_Conifer_Forest.jpg +0 -0
- package/src/client/img/Xetthecum/Porlier_Pass_Road.jpg +0 -0
- package/src/client/img/Xetthecum/Riparian.jpg +0 -0
- package/src/client/img/Xetthecum/Rural.jpg +0 -0
- package/src/client/img/Xetthecum/Subtidal.jpg +0 -0
- package/src/client/img/Xetthecum/Wetland.jpg +0 -0
- package/src/client/img/Xetthecum/Woodland.jpg +0 -0
- package/src/client/img/Xetthecum/Woodland_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Xetthecum.jpg +0 -0
- package/src/client/img/Xetthecum/Young_Conifer_Forest.jpg +0 -0
- package/src/client/img/Xetthecum/Young_Mixed_Forest.jpg +0 -0
- package/src/client/img/Xetthecum/food.svg +38 -0
- package/src/client/img/Xetthecum/indicator.svg +52 -0
- package/src/client/img/Xetthecum/material.svg +186 -0
- package/src/client/img/Xetthecum/medicinal.svg +62 -0
- package/src/client/img/Xetthecum/missing-value.svg +3 -0
- package/src/client/img/Xetthecum/spiritual.svg +13 -0
- package/src/client/img/Xetthecum/trade.svg +74 -0
- package/src/client/img/audio.svg +5 -0
- package/src/client/img/external-link.svg +3 -0
- package/src/client/img/selected-eye.svg +4 -0
- package/src/client/img/selected-tick.svg +3 -0
- package/src/client/img/thumbs/BCLeps.jpg +0 -0
- package/src/client/img/thumbs/Checklist.jpg +0 -0
- package/src/client/img/thumbs/Data-Paper-Figma.jpg +0 -0
- package/src/client/img/thumbs/Galiano-Data-Paper.jpg +0 -0
- package/src/client/img/thumbs/Galiano.jpg +0 -0
- package/src/client/img/thumbs/Maxwell.jpg +0 -0
- package/src/client/img/thumbs/Squamish.jpg +0 -0
- package/src/client/img/thumbs/Valdes.jpg +0 -0
- package/src/client/img/thumbs/Xetthecum.jpg +0 -0
- package/src/client/img/up-arrow.svg +4 -0
- package/src/client/js/autocomplete.js +53 -0
- package/src/client/js/checklist.js +179 -0
- package/src/client/js/colour.js +139 -0
- package/src/client/js/datasetControls.js +212 -0
- package/src/client/js/imerss-viz.js +1397 -0
- package/src/client/js/leafletMap.js +357 -0
- package/src/client/js/leafletMapWithGrid.js +337 -0
- package/src/client/js/leafletMapWithRegions.js +381 -0
- package/src/client/js/renderSVG.js +108 -0
- package/src/client/js/tabs.js +53 -0
- package/src/client/js/xetthecum.js +106 -0
- package/src/client/json/emptyPhyloMap.json +1 -0
- package/src/client/json/kingdomPhyloMap.json +17 -0
- package/src/client/json/phyloMap.json +58 -0
- package/src/compare.js +97 -0
- package/src/compilecsv.js +47 -0
- package/src/dataProcessing/LZ4.js +27 -0
- package/src/dataProcessing/coordinatePatch.js +151 -0
- package/src/dataProcessing/readCSV.js +107 -0
- package/src/dataProcessing/readCSVwithMap.js +90 -0
- package/src/dataProcessing/readCSVwithoutMap.js +25 -0
- package/src/dataProcessing/readJSON.js +23 -0
- package/src/dataProcessing/sqlite.js +174 -0
- package/src/dataProcessing/summarise.js +122 -0
- package/src/dataProcessing/taxonScrape.js +14 -0
- package/src/dataProcessing/worker.js +31 -0
- package/src/dataProcessing/writeCSV.js +34 -0
- package/src/dataProcessing/writeJSON.js +15 -0
- package/src/dbtaxa.js +96 -0
- package/src/deAuthorise.js +48 -0
- package/src/deqgis.js +148 -0
- package/src/expressions/expr.js +65 -0
- package/src/expressions/jsep.js +925 -0
- package/src/filterFirst.js +51 -0
- package/src/geom/geoJSON.js +121 -0
- package/src/iNaturalist/examples/api.inaturalist.org-v1-taxa!q=Acmispon americanus americanus.json +200 -0
- package/src/iNaturalist/examples/api.inaturalist.org-v1-taxa!q=Flabellina japonica.json +61 -0
- package/src/iNaturalist/examples/www.inaturalist.org-taxa-877359.json +132 -0
- package/src/iNaturalist/iNatUrls.js +17 -0
- package/src/iNaturalist/obsAPI.js +150 -0
- package/src/iNaturalist/taxonAPI.js +460 -0
- package/src/inatObs.js +120 -0
- package/src/inattify.js +123 -0
- package/src/intersect.js +80 -0
- package/src/intersectFilter.js +48 -0
- package/src/landlocked.js +137 -0
- package/src/lib/jquery.binarytransport.js +51 -0
- package/src/lib/lz4-commentary.txt +9 -0
- package/src/lib/lz4.js +73 -0
- package/src/lib/point-in-polygon.js +36 -0
- package/src/lz4.js +40 -0
- package/src/marmalise.js +333 -0
- package/src/materialise.js +68 -0
- package/src/reintegrate.js +71 -0
- package/src/taxonomise.js +615 -0
- package/src/unscrewGBIF.js +44 -0
- package/src/utils/dataSource.js +173 -0
- package/src/utils/settleStructure.js +62 -0
- package/src/utils/utils.js +92 -0
- package/src/viz/plethify.js +39 -0
- package/src/wormify.js +154 -0
- package/src/xetthecum/xetthecumFilters.js +126 -0
- package/test/testDataSource.js +182 -0
- package/test/testiNatObsSource.js +38 -0
- package/test/testiNatSource.js +50 -0
package/src/marmalise.js
ADDED
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
/* eslint-env node */
|
|
2
|
+
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
const fluid = require("infusion");
|
|
6
|
+
const minimist = require("minimist");
|
|
7
|
+
const fs = require("fs");
|
|
8
|
+
|
|
9
|
+
require("./utils/utils.js");
|
|
10
|
+
require("./dataProcessing/readJSON.js");
|
|
11
|
+
require("./dataProcessing/readCSV.js");
|
|
12
|
+
require("./dataProcessing/readCSVwithMap.js");
|
|
13
|
+
require("./dataProcessing/LZ4.js");
|
|
14
|
+
require("./iNaturalist/taxonAPI.js");
|
|
15
|
+
|
|
16
|
+
const hortis = fluid.registerNamespace("hortis");
|
|
17
|
+
|
|
18
|
+
hortis.ranks = fluid.freezeRecursive(require("../data/ranks.json"));
|
|
19
|
+
|
|
20
|
+
hortis.newTaxon = function (iNaturalistTaxonName, rank, depth, counts) {
|
|
21
|
+
const togo = {
|
|
22
|
+
iNaturalistTaxonName: iNaturalistTaxonName,
|
|
23
|
+
id: fluid.allocateGuid(),
|
|
24
|
+
rank: rank,
|
|
25
|
+
depth: depth,
|
|
26
|
+
childCount: 0,
|
|
27
|
+
children: {}
|
|
28
|
+
};
|
|
29
|
+
fluid.each(counts, function (oneCount, key) {
|
|
30
|
+
togo[key] = 0;
|
|
31
|
+
});
|
|
32
|
+
return togo;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
hortis.addCounts = function (row, counts) {
|
|
36
|
+
fluid.each(counts, function (countDef, key) {
|
|
37
|
+
const countField = row[countDef.column];
|
|
38
|
+
row[key] = countDef.free ? Number(countField) : (countField === countDef.equals ? 1 : 0);
|
|
39
|
+
});
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
hortis.cleanRow = function (row) {
|
|
43
|
+
fluid.each(row, function (value, key) {
|
|
44
|
+
if (typeof(value) === "string") {
|
|
45
|
+
const trimmed = value.trim();
|
|
46
|
+
if (trimmed === "" || trimmed === "—") { // Note funny AS hyphen in here
|
|
47
|
+
delete row[key];
|
|
48
|
+
} else {
|
|
49
|
+
row[key] = trimmed;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
hortis.rowToTaxon = function (row, depth, counts) {
|
|
56
|
+
row.depth = depth;
|
|
57
|
+
row.id = fluid.allocateGuid();
|
|
58
|
+
row.childCount = 1;
|
|
59
|
+
row.children = {};
|
|
60
|
+
hortis.addCounts(row, counts);
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
hortis.fullRecordTransform = {
|
|
64
|
+
"wikipediaSummary": "wikipedia_summary",
|
|
65
|
+
"commonName": "common_name.name",
|
|
66
|
+
"iNaturalistTaxonId": "id",
|
|
67
|
+
"iNaturalistTaxonImage": "default_photo.medium_url"
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
hortis.addTaxonInfo = function (row, fullRecord) {
|
|
71
|
+
fluid.each(hortis.fullRecordTransform, function (path, target) {
|
|
72
|
+
const source = fluid.get(fullRecord, path);
|
|
73
|
+
if (row[target] === undefined) {
|
|
74
|
+
row[target] = source;
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
hortis.reverseMerge = function (target, source) {
|
|
80
|
+
fluid.each(source, function (value, key) {
|
|
81
|
+
if (!target[key]) {
|
|
82
|
+
target[key] = value;
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
hortis.storeAtPath = function (treeBuilder, path, row) {
|
|
88
|
+
const counts = treeBuilder.map.counts;
|
|
89
|
+
hortis.rowToTaxon(row, path.length, counts);
|
|
90
|
+
let node = treeBuilder.tree;
|
|
91
|
+
path.forEach(function (doc, index) {
|
|
92
|
+
const last = index === path.length - 1;
|
|
93
|
+
const name = doc.name;
|
|
94
|
+
let child = node.children[name];
|
|
95
|
+
if (!child) {
|
|
96
|
+
child = node.children[name] = (last ? row : hortis.newTaxon(name, doc.rank, index + 1, counts));
|
|
97
|
+
try {
|
|
98
|
+
hortis.addTaxonInfo(child, doc);
|
|
99
|
+
} catch (e) {
|
|
100
|
+
console.log("While storing row ", row);
|
|
101
|
+
throw e;
|
|
102
|
+
}
|
|
103
|
+
} else if (last) { // Store any surplus fields, expecially include "coords" here otherwise they will be lost
|
|
104
|
+
hortis.reverseMerge(child, row);
|
|
105
|
+
// It may already have had an id allocated as a result of being observed as a lower taxon - make sure
|
|
106
|
+
// row records this since this id may be used when assigning obs to regions
|
|
107
|
+
row.id = child.id;
|
|
108
|
+
}
|
|
109
|
+
++node.childCount;
|
|
110
|
+
fluid.each(counts, function (countDef, key) {
|
|
111
|
+
node[key] += row[key];
|
|
112
|
+
});
|
|
113
|
+
node = child;
|
|
114
|
+
});
|
|
115
|
+
};
|
|
116
|
+
/*
|
|
117
|
+
hortis.loadCachedTaxonDoc = function (treeBuilder, id) {
|
|
118
|
+
let existing = treeBuilder.taxonCache[id];
|
|
119
|
+
if (!existing) {
|
|
120
|
+
existing = treeBuilder.taxonCache[id] = hortis.iNat.loadTaxonDoc(treeBuilder.options.taxonAPIFileBase, id);
|
|
121
|
+
}
|
|
122
|
+
// This otherwise causes rank to be "undefined" in the interface - AS report 18/4/22
|
|
123
|
+
// Not sure what to do about this long-term
|
|
124
|
+
if (existing.rank === "complex") {
|
|
125
|
+
existing.rank = "species";
|
|
126
|
+
}
|
|
127
|
+
return existing;
|
|
128
|
+
};
|
|
129
|
+
*/
|
|
130
|
+
|
|
131
|
+
/** Produce a "path" holding each cached taxon doc from the supplied row from the root
|
|
132
|
+
* @param {TreeBuilder} treeBuilder - The treeBuilder instance
|
|
133
|
+
* @param {Row} row - An observation or summary row
|
|
134
|
+
* @return {Promise<TaxonDoc[]>} Array of taxon docs starting at the root and ending at the row's doc
|
|
135
|
+
*/
|
|
136
|
+
hortis.taxaToPathiNat = async function (treeBuilder, row) {
|
|
137
|
+
const baseDoc = (await treeBuilder.taxonSource.get({id: row.iNaturalistTaxonId})).doc;
|
|
138
|
+
const parentTaxaIds = hortis.iNat.parentTaxaIds(baseDoc).reverse();
|
|
139
|
+
const ancestourDocs = await Promise.all(parentTaxaIds.map(async function (id) {
|
|
140
|
+
return (await treeBuilder.taxonSource.get({id: id})).doc;
|
|
141
|
+
}));
|
|
142
|
+
const rankDocs = ancestourDocs.filter(function (oneDoc) {
|
|
143
|
+
return treeBuilder.options.vizRanks.includes(oneDoc.rank);
|
|
144
|
+
});
|
|
145
|
+
return rankDocs.concat([baseDoc]);
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
hortis.applyRowsToTree = async function (treeBuilder, rows) {
|
|
150
|
+
for (let i = 0; i < rows.length; ++i) {
|
|
151
|
+
const row = rows[i];
|
|
152
|
+
if (i % 100 === 0) {
|
|
153
|
+
process.stdout.write(i + " ... ");
|
|
154
|
+
}
|
|
155
|
+
hortis.cleanRow(row);
|
|
156
|
+
const path = await hortis.taxaToPathiNat(treeBuilder, row);
|
|
157
|
+
treeBuilder.storeAtPath(path, row);
|
|
158
|
+
}
|
|
159
|
+
console.log("");
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
hortis.flattenChildren = function (root) {
|
|
163
|
+
const children = [];
|
|
164
|
+
fluid.each(root.children, function (value) {
|
|
165
|
+
children.push(hortis.flattenChildren(value));
|
|
166
|
+
});
|
|
167
|
+
root.children = children;
|
|
168
|
+
return root;
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
hortis.parseCounts = function (counts) {
|
|
172
|
+
fluid.each(counts, function (oneCount) {
|
|
173
|
+
if (oneCount.equals === "") {
|
|
174
|
+
oneCount.equals = undefined;
|
|
175
|
+
}
|
|
176
|
+
});
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
hortis.loadMapWithCounts = function (mapFile) {
|
|
180
|
+
const map = hortis.readJSONSync(fluid.module.resolvePath(mapFile));
|
|
181
|
+
hortis.parseCounts(map.counts);
|
|
182
|
+
return map;
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
hortis.rootNode = function (map) {
|
|
186
|
+
return hortis.newTaxon("Life", "life", 0, map.counts);
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
hortis.filterVizRanks = function (skipRanks) {
|
|
190
|
+
return hortis.ranks.filter(function (rank) {
|
|
191
|
+
return !skipRanks.includes(rank);
|
|
192
|
+
});
|
|
193
|
+
};
|
|
194
|
+
|
|
195
|
+
fluid.defaults("hortis.treeBuilder", {
|
|
196
|
+
gradeNames: "fluid.component",
|
|
197
|
+
taxonAPIFileBase: "data/iNaturalist/taxonAPI",
|
|
198
|
+
mapFile: "data/Galiano/Galiano-map.json",
|
|
199
|
+
outputFile: "Life.json.lz4",
|
|
200
|
+
featureFile: "",
|
|
201
|
+
regionField: null, obsIdField: null,
|
|
202
|
+
inputFiles: [],
|
|
203
|
+
skipRanks: [],
|
|
204
|
+
vizRanks: "@expand:hortis.filterVizRanks({that}.options.skipRanks)",
|
|
205
|
+
members: {
|
|
206
|
+
// Mutable tree to which each file contributes its taxa
|
|
207
|
+
tree: "@expand:hortis.rootNode({that}.map)",
|
|
208
|
+
map: "@expand:hortis.loadMapWithCounts({that}.options.mapFile)",
|
|
209
|
+
features: null, // "flatFeatures" file output, e.g. by deqgis, if configured
|
|
210
|
+
obs: null, // observations file which will be applied to features - expected to contain column named "regionColumn" supplied in options
|
|
211
|
+
// map of taxonId to taxon file
|
|
212
|
+
taxonCache: {}
|
|
213
|
+
},
|
|
214
|
+
invokers: {
|
|
215
|
+
applyRowsToTree: "hortis.applyRowsToTree({that}, {arguments}.1)", // rows
|
|
216
|
+
storeAtPath: "hortis.storeAtPath({that}, {arguments}.0, {arguments}.1)" // path, row
|
|
217
|
+
},
|
|
218
|
+
listeners: {
|
|
219
|
+
"onCreate.marmalise": "hortis.marmalise({that})"
|
|
220
|
+
},
|
|
221
|
+
components: {
|
|
222
|
+
taxonSource: {
|
|
223
|
+
type: "hortis.iNatTaxonSource"
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
hortis.filterDataset = function (dataset) {
|
|
229
|
+
return fluid.filterKeys(dataset, ["name", "colour"]);
|
|
230
|
+
};
|
|
231
|
+
|
|
232
|
+
hortis.indexRegions = function (treeBuilder) {
|
|
233
|
+
const features = treeBuilder.features;
|
|
234
|
+
const summaryById = {};
|
|
235
|
+
treeBuilder.summaryRows.forEach(function (row) {
|
|
236
|
+
summaryById[row.iNaturalistTaxonId] = row;
|
|
237
|
+
});
|
|
238
|
+
const communities = fluid.transform(features.communities, function (community) {
|
|
239
|
+
community.count = 0;
|
|
240
|
+
community.byTaxonId = {};
|
|
241
|
+
return community;
|
|
242
|
+
});
|
|
243
|
+
const classes = {};
|
|
244
|
+
fluid.each(features.classes, function (clazz, classKey) {
|
|
245
|
+
classes[classKey] = {
|
|
246
|
+
count: 0,
|
|
247
|
+
byTaxonId: {}
|
|
248
|
+
};
|
|
249
|
+
});
|
|
250
|
+
const applyObs = function (container, key, taxonId, obsId) {
|
|
251
|
+
if (key) {
|
|
252
|
+
const region = container[key];
|
|
253
|
+
if (!region) {
|
|
254
|
+
fluid.fail("Unknown region key ", key, " in container with keys ", Object.keys(container).join(", "));
|
|
255
|
+
}
|
|
256
|
+
let bucketTaxa = region.byTaxonId[taxonId]; // TODO: use fluid.pushArray?
|
|
257
|
+
if (!bucketTaxa) {
|
|
258
|
+
bucketTaxa = region.byTaxonId[taxonId] = [];
|
|
259
|
+
}
|
|
260
|
+
bucketTaxa.push(obsId);
|
|
261
|
+
++region.count;
|
|
262
|
+
}
|
|
263
|
+
};
|
|
264
|
+
const options = treeBuilder.options;
|
|
265
|
+
treeBuilder.obs.rows.forEach(function (row) {
|
|
266
|
+
const obsId = row[options.obsIdField];
|
|
267
|
+
const summaryRow = summaryById[row.iNaturalistTaxonId];
|
|
268
|
+
if (summaryRow) {
|
|
269
|
+
const taxonId = summaryRow.id;
|
|
270
|
+
applyObs(communities, row.COMMUNITY, taxonId, obsId);
|
|
271
|
+
applyObs(classes, row.clazz, taxonId, obsId);
|
|
272
|
+
|
|
273
|
+
} else {
|
|
274
|
+
console.log("Warning: row with iNaturalistTaxonId " + row.iNaturalistTaxonId + " did not correspond to row in summary: ", row);
|
|
275
|
+
}
|
|
276
|
+
});
|
|
277
|
+
return {
|
|
278
|
+
communities: communities,
|
|
279
|
+
classes: fluid.extend(true, {}, features.classes, classes),
|
|
280
|
+
features: features.features
|
|
281
|
+
};
|
|
282
|
+
};
|
|
283
|
+
|
|
284
|
+
hortis.marmalise = async function (treeBuilder) {
|
|
285
|
+
const options = treeBuilder.options;
|
|
286
|
+
await hortis.asyncMap(options.inputFiles, async function (fileName) {
|
|
287
|
+
const result = await hortis.csvReaderWithMap({
|
|
288
|
+
inputFile: fluid.module.resolvePath(fileName),
|
|
289
|
+
mapColumns: treeBuilder.map.columns
|
|
290
|
+
}).completionPromise;
|
|
291
|
+
// console.log(JSON.stringify(result.rows[0], null, 2));
|
|
292
|
+
console.log("Applying " + result.rows.length + " rows from " + fileName);
|
|
293
|
+
treeBuilder.summaryRows = result.rows;
|
|
294
|
+
await treeBuilder.applyRowsToTree(fileName, result.rows);
|
|
295
|
+
});
|
|
296
|
+
if (options.featuresFile) {
|
|
297
|
+
fluid.expect("viz config file", options, ["obsFile", "obsMapFile", "regionField", "obsIdField"]);
|
|
298
|
+
treeBuilder.features = hortis.readModuleJSONSync(options.featuresFile);
|
|
299
|
+
const obsMap = hortis.readModuleJSONSync(options.obsMapFile);
|
|
300
|
+
const obs = await hortis.csvReaderWithMap({
|
|
301
|
+
inputFile: fluid.module.resolvePath(options.obsFile),
|
|
302
|
+
mapColumns: obsMap.columns
|
|
303
|
+
}).completionPromise;
|
|
304
|
+
console.log("Read " + obs.rows.length + " observation rows from " + options.obsFile);
|
|
305
|
+
treeBuilder.obs = obs;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
hortis.flattenChildren(treeBuilder.tree);
|
|
309
|
+
const output = {
|
|
310
|
+
datasets: fluid.transform(treeBuilder.map.datasets, hortis.filterDataset),
|
|
311
|
+
tree: treeBuilder.tree
|
|
312
|
+
};
|
|
313
|
+
const extraOutput = options.featuresFile ? hortis.indexRegions(treeBuilder) : {};
|
|
314
|
+
const fullOutput = fluid.extend(output, extraOutput);
|
|
315
|
+
fs.writeFileSync("marmalised.json", JSON.stringify(fullOutput, null, 4) + "\n");
|
|
316
|
+
const text = JSON.stringify(fullOutput);
|
|
317
|
+
hortis.writeLZ4File(text, fluid.module.resolvePath(treeBuilder.options.outputFile));
|
|
318
|
+
};
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
const parsedArgs = minimist(process.argv.slice(2));
|
|
322
|
+
|
|
323
|
+
const options = parsedArgs.config ? hortis.readJSONSync(parsedArgs.config) : {
|
|
324
|
+
mapFile: parsedArgs.map,
|
|
325
|
+
obsFile: parsedArgs.obsFile,
|
|
326
|
+
skipRanks: [],
|
|
327
|
+
obsMapFile: parsedArgs.obsMapFile,
|
|
328
|
+
featuresFile: parsedArgs.featuresFile,
|
|
329
|
+
outputFile: parsedArgs.o || parsedArgs.output,
|
|
330
|
+
inputFiles: parsedArgs._
|
|
331
|
+
};
|
|
332
|
+
|
|
333
|
+
hortis.treeBuilder(options);
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/* eslint-env node */
|
|
2
|
+
/* eslint dot-notation: "off"*/
|
|
3
|
+
|
|
4
|
+
"use strict";
|
|
5
|
+
|
|
6
|
+
var fluid = require("infusion");
|
|
7
|
+
var glob = require("glob");
|
|
8
|
+
var kettle = require("kettle");
|
|
9
|
+
var fs = require("fs");
|
|
10
|
+
fluid.require("%imerss-bioinfo");
|
|
11
|
+
|
|
12
|
+
var baseDir = fluid.module.resolvePath("%imerss-bioinfo/data/dataPaper-I-in/Animalia/");
|
|
13
|
+
|
|
14
|
+
fs.mkdirSync(baseDir, {
|
|
15
|
+
recursive: true
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
// Used during preparation of data paper to materialise an entire folder full of Google Sheets as CSV.
|
|
19
|
+
// Will need to updated to use Google Sheets API directrly and not rely on a Google Drive mapping
|
|
20
|
+
|
|
21
|
+
var dir = process.argv[2] || "e:/data/Google Drive/Galiano Data Paper 2021/Marine Life/Animalia";
|
|
22
|
+
var source = kettle.dataSource.URL({
|
|
23
|
+
url: "https://docs.google.com/spreadsheets/d/%id/export?format=csv&id=%id&gid=0",
|
|
24
|
+
port: 443,
|
|
25
|
+
termMap: {
|
|
26
|
+
id: "%id"
|
|
27
|
+
},
|
|
28
|
+
requestOptions: {
|
|
29
|
+
followAllRedirects: true
|
|
30
|
+
},
|
|
31
|
+
components: {
|
|
32
|
+
encoding: {
|
|
33
|
+
type: "fluid.dataSource.encoding.none"
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
var getSegment = function (path) {
|
|
39
|
+
var lastSlash = path.lastIndexOf("/");
|
|
40
|
+
var lastDot = path.lastIndexOf(".");
|
|
41
|
+
return path.substring(lastSlash + 1, lastDot);
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
fluid.setLogging(true);
|
|
45
|
+
|
|
46
|
+
glob(dir + "/*.gsheet", function (er, files) {
|
|
47
|
+
console.log("Got files ", files);
|
|
48
|
+
var records = files.map(function (oneFile) {
|
|
49
|
+
var promise = kettle.JSON.readFileSync(oneFile);
|
|
50
|
+
var parsed;
|
|
51
|
+
promise.then(function (result) {
|
|
52
|
+
parsed = result;
|
|
53
|
+
});
|
|
54
|
+
return {
|
|
55
|
+
id: parsed["doc_id"],
|
|
56
|
+
segment: getSegment(oneFile)
|
|
57
|
+
};
|
|
58
|
+
});
|
|
59
|
+
console.log("Got records ", records);
|
|
60
|
+
records.forEach(function (oneRecord) {
|
|
61
|
+
source.get({id: oneRecord.id}).then(function (doc) {
|
|
62
|
+
console.log("Got doc ", doc);
|
|
63
|
+
fs.writeFileSync(baseDir + oneRecord.segment + ".csv", doc);
|
|
64
|
+
}, function (err) {
|
|
65
|
+
console.log("Got err ", err);
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
});
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/* eslint-env node */
|
|
2
|
+
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
var fluid = require("infusion");
|
|
6
|
+
var fs = require("fs");
|
|
7
|
+
var JSON5 = require("json5");
|
|
8
|
+
|
|
9
|
+
require("./dataProcessing/readCSV.js");
|
|
10
|
+
require("./dataProcessing/readCSVwithMap.js");
|
|
11
|
+
require("./utils/settleStructure.js");
|
|
12
|
+
|
|
13
|
+
fluid.setLogging(true);
|
|
14
|
+
|
|
15
|
+
var hortis = fluid.registerNamespace("hortis");
|
|
16
|
+
|
|
17
|
+
var taxonResolveMap = hortis.readJSONSync("data/TaxonResolution-map.json", "reading taxon resolution map");
|
|
18
|
+
var swaps = hortis.readJSONSync("data/taxon-swaps.json5", "reading taxon swaps file");
|
|
19
|
+
|
|
20
|
+
// Long disused script to output a taxon-swaps JSON5 file given a CSV taxon resolution file
|
|
21
|
+
|
|
22
|
+
hortis.copyCSVProps = function (props, target, source) {
|
|
23
|
+
props.forEach(function (prop) {
|
|
24
|
+
if (fluid.isValue(source[prop]) && source[prop] !== "") {
|
|
25
|
+
target[prop] = source[prop];
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
hortis.alphabetise = function (hash) {
|
|
31
|
+
var togo = {};
|
|
32
|
+
var keys = Object.keys(hash).sort();
|
|
33
|
+
keys.forEach(function (key) {
|
|
34
|
+
togo[key] = hash[key];
|
|
35
|
+
});
|
|
36
|
+
return togo;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
hortis.updateSwaps = function (data) {
|
|
40
|
+
data.rows.forEach(function (row) {
|
|
41
|
+
var iNatName = row.iNaturalistTaxonName;
|
|
42
|
+
if (iNatName) {
|
|
43
|
+
var existing = swaps[iNatName];
|
|
44
|
+
if (!existing) {
|
|
45
|
+
existing = {
|
|
46
|
+
taxonNames: {}
|
|
47
|
+
};
|
|
48
|
+
swaps[iNatName] = existing;
|
|
49
|
+
}
|
|
50
|
+
existing.iNaturalistTaxonId = row.iNaturalistTaxonId;
|
|
51
|
+
var taxonNames = existing.taxonNames;
|
|
52
|
+
|
|
53
|
+
if (row.commonName) {
|
|
54
|
+
taxonNames[row.commonName] = {type: "commonName"};
|
|
55
|
+
}
|
|
56
|
+
var record = {};
|
|
57
|
+
hortis.copyCSVProps(["resolvedBy", "notes", "oldiNaturalistTaxonId"], record, row);
|
|
58
|
+
taxonNames[row.taxonName] = record;
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
var alphaSwaps = hortis.alphabetise(swaps);
|
|
62
|
+
fs.writeFileSync("updated-taxon-swaps.json5", JSON5.stringify(alphaSwaps, {
|
|
63
|
+
space: 4,
|
|
64
|
+
quote: "\""
|
|
65
|
+
}));
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
hortis.imerss.csvReader({
|
|
69
|
+
inputFile: process.argv[2] || "data/Valdes/Valdes Marine Dives Taxon Resolution.csv",
|
|
70
|
+
mapColumns: taxonResolveMap.columns
|
|
71
|
+
}).completionPromise.then(hortis.updateSwaps);
|