imerss-bioinfo 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/README.md +156 -0
  2. package/build.js +196 -0
  3. package/docs/css/imerss-viz-all.css +1227 -0
  4. package/docs/css/imerss-viz-wp-overrides.css +70 -0
  5. package/docs/css/pepiowelh.css +76 -0
  6. package/docs/css/xetthecum-external.css +47 -0
  7. package/docs/css/xetthecum-shared.css +80 -0
  8. package/docs/css/xetthecum.css +526 -0
  9. package/docs/js/imerss-viz-all.js +58 -0
  10. package/docs/js/imerss-viz-all.js.map +1 -0
  11. package/docs/js/jquery.js +10881 -0
  12. package/index.html +81 -0
  13. package/index.js +7 -0
  14. package/indexWithJustMap.html +92 -0
  15. package/indexWithMap.html +79 -0
  16. package/indexWithMapHoweSound.html +91 -0
  17. package/indexWithMapPepiowelh.html +119 -0
  18. package/indexWithMapValdes.html +90 -0
  19. package/indexWithMapXetthecum.html +90 -0
  20. package/package.json +50 -0
  21. package/src/WoRMS/taxonAPI.js +13 -0
  22. package/src/arphify.js +662 -0
  23. package/src/assignNames.js +103 -0
  24. package/src/auxBuild/restoreJQuery.js +1 -0
  25. package/src/buildSource/Comprehensive Lists.html +42 -0
  26. package/src/buildSource/Data Paper Part I Visualisation.html +50 -0
  27. package/src/buildSource/Galiano Life List.html +43 -0
  28. package/src/buildSource/Metchosin/index.html +22 -0
  29. package/src/buildSource/Pe'pi'ow'elh.html +72 -0
  30. package/src/buildSource/Squamish Life List.html +48 -0
  31. package/src/buildSource/Valdes Island Biodiversity.html +64 -0
  32. package/src/buildSource/Xetthecum.html +40 -0
  33. package/src/buildSource/dataPaperSunburstAndMap.html +30 -0
  34. package/src/buildSource/index.html +224 -0
  35. package/src/client/css/imerss-viz-wp-overrides.css +70 -0
  36. package/src/client/css/imerss-viz.css +531 -0
  37. package/src/client/css/pepiowelh.css +76 -0
  38. package/src/client/css/xetthecum-external.css +47 -0
  39. package/src/client/css/xetthecum-shared.css +80 -0
  40. package/src/client/css/xetthecum.css +526 -0
  41. package/src/client/html/imerss-viz-map-only.html +75 -0
  42. package/src/client/html/imerss-viz-map-pepi.html +77 -0
  43. package/src/client/html/imerss-viz-map-xetthecum.html +90 -0
  44. package/src/client/html/imerss-viz-map.html +75 -0
  45. package/src/client/html/imerss-viz.html +58 -0
  46. package/src/client/html/importing Xetthecum patterns.txt +10 -0
  47. package/src/client/html/xetthecum-patterns.html +27 -0
  48. package/src/client/img/Animalia_2.svg +1 -0
  49. package/src/client/img/Animalia_Annelida.svg +655 -0
  50. package/src/client/img/Animalia_Chaetognatha.svg +72 -0
  51. package/src/client/img/Animalia_Cnidaria.svg +173 -0
  52. package/src/client/img/Animalia_Ctenophora.svg +298 -0
  53. package/src/client/img/Animalia_Nemertea.svg +54 -0
  54. package/src/client/img/Animalia_Phoronida.svg +271 -0
  55. package/src/client/img/Animalia_Platyhelminthes.svg +58 -0
  56. package/src/client/img/Animalia_Porifera.svg +220 -0
  57. package/src/client/img/Animalia_Sipuncula.svg +155 -0
  58. package/src/client/img/Arachnoidiscus_schmidtii_web_1-2-scaled.webp +0 -0
  59. package/src/client/img/Arrow-down.svg +11 -0
  60. package/src/client/img/Arrow-up.svg +6 -0
  61. package/src/client/img/Chromista.svg +1 -0
  62. package/src/client/img/Pepiowelh/BlackNoWe2-wb-bb.png +0 -0
  63. package/src/client/img/Pepiowelh/BlackNoWe2-wb.png +0 -0
  64. package/src/client/img/Pepiowelh/BlackNoWe2.png +0 -0
  65. package/src/client/img/Pepiowelh/Ecocultural Mapping Banner.svg +1 -0
  66. package/src/client/img/Plankton.jpg +0 -0
  67. package/src/client/img/Plantae.svg +1 -0
  68. package/src/client/img/Plantae_2.svg +1 -0
  69. package/src/client/img/Protista.svg +1 -0
  70. package/src/client/img/Valdes/Valdes Island Conservancy.jpg +0 -0
  71. package/src/client/img/Xetthecum/Agricultural.jpg +0 -0
  72. package/src/client/img/Xetthecum/Anthropogenic_banner.jpg +0 -0
  73. package/src/client/img/Xetthecum/Broad_Leaf.jpg +0 -0
  74. package/src/client/img/Xetthecum/Caves.jpg +0 -0
  75. package/src/client/img/Xetthecum/Coastal_Bluffs.jpg +0 -0
  76. package/src/client/img/Xetthecum/Conifer_Pole_Sapling.jpg +0 -0
  77. package/src/client/img/Xetthecum/Developed.jpg +0 -0
  78. package/src/client/img/Xetthecum/Dock.jpg +0 -0
  79. package/src/client/img/Xetthecum/Eelgrass.jpg +0 -0
  80. package/src/client/img/Xetthecum/Forests_banner.jpg +0 -0
  81. package/src/client/img/Xetthecum/Freshwater_banner.jpg +0 -0
  82. package/src/client/img/Xetthecum/Greig_Creek.jpg +0 -0
  83. package/src/client/img/Xetthecum/Intertidal.jpg +0 -0
  84. package/src/client/img/Xetthecum/Laughlin_Lake.jpg +0 -0
  85. package/src/client/img/Xetthecum/Marine_banner.jpg +0 -0
  86. package/src/client/img/Xetthecum/Mature_Conifer_Forest.jpg +0 -0
  87. package/src/client/img/Xetthecum/Porlier_Pass_Road.jpg +0 -0
  88. package/src/client/img/Xetthecum/Riparian.jpg +0 -0
  89. package/src/client/img/Xetthecum/Rural.jpg +0 -0
  90. package/src/client/img/Xetthecum/Subtidal.jpg +0 -0
  91. package/src/client/img/Xetthecum/Wetland.jpg +0 -0
  92. package/src/client/img/Xetthecum/Woodland.jpg +0 -0
  93. package/src/client/img/Xetthecum/Woodland_banner.jpg +0 -0
  94. package/src/client/img/Xetthecum/Xetthecum.jpg +0 -0
  95. package/src/client/img/Xetthecum/Young_Conifer_Forest.jpg +0 -0
  96. package/src/client/img/Xetthecum/Young_Mixed_Forest.jpg +0 -0
  97. package/src/client/img/Xetthecum/food.svg +38 -0
  98. package/src/client/img/Xetthecum/indicator.svg +52 -0
  99. package/src/client/img/Xetthecum/material.svg +186 -0
  100. package/src/client/img/Xetthecum/medicinal.svg +62 -0
  101. package/src/client/img/Xetthecum/missing-value.svg +3 -0
  102. package/src/client/img/Xetthecum/spiritual.svg +13 -0
  103. package/src/client/img/Xetthecum/trade.svg +74 -0
  104. package/src/client/img/audio.svg +5 -0
  105. package/src/client/img/external-link.svg +3 -0
  106. package/src/client/img/selected-eye.svg +4 -0
  107. package/src/client/img/selected-tick.svg +3 -0
  108. package/src/client/img/thumbs/BCLeps.jpg +0 -0
  109. package/src/client/img/thumbs/Checklist.jpg +0 -0
  110. package/src/client/img/thumbs/Data-Paper-Figma.jpg +0 -0
  111. package/src/client/img/thumbs/Galiano-Data-Paper.jpg +0 -0
  112. package/src/client/img/thumbs/Galiano.jpg +0 -0
  113. package/src/client/img/thumbs/Maxwell.jpg +0 -0
  114. package/src/client/img/thumbs/Squamish.jpg +0 -0
  115. package/src/client/img/thumbs/Valdes.jpg +0 -0
  116. package/src/client/img/thumbs/Xetthecum.jpg +0 -0
  117. package/src/client/img/up-arrow.svg +4 -0
  118. package/src/client/js/autocomplete.js +53 -0
  119. package/src/client/js/checklist.js +179 -0
  120. package/src/client/js/colour.js +139 -0
  121. package/src/client/js/datasetControls.js +212 -0
  122. package/src/client/js/imerss-viz.js +1397 -0
  123. package/src/client/js/leafletMap.js +357 -0
  124. package/src/client/js/leafletMapWithGrid.js +337 -0
  125. package/src/client/js/leafletMapWithRegions.js +381 -0
  126. package/src/client/js/renderSVG.js +108 -0
  127. package/src/client/js/tabs.js +53 -0
  128. package/src/client/js/xetthecum.js +106 -0
  129. package/src/client/json/emptyPhyloMap.json +1 -0
  130. package/src/client/json/kingdomPhyloMap.json +17 -0
  131. package/src/client/json/phyloMap.json +58 -0
  132. package/src/compare.js +97 -0
  133. package/src/compilecsv.js +47 -0
  134. package/src/dataProcessing/LZ4.js +27 -0
  135. package/src/dataProcessing/coordinatePatch.js +151 -0
  136. package/src/dataProcessing/readCSV.js +107 -0
  137. package/src/dataProcessing/readCSVwithMap.js +90 -0
  138. package/src/dataProcessing/readCSVwithoutMap.js +25 -0
  139. package/src/dataProcessing/readJSON.js +23 -0
  140. package/src/dataProcessing/sqlite.js +174 -0
  141. package/src/dataProcessing/summarise.js +122 -0
  142. package/src/dataProcessing/taxonScrape.js +14 -0
  143. package/src/dataProcessing/worker.js +31 -0
  144. package/src/dataProcessing/writeCSV.js +34 -0
  145. package/src/dataProcessing/writeJSON.js +15 -0
  146. package/src/dbtaxa.js +96 -0
  147. package/src/deAuthorise.js +48 -0
  148. package/src/deqgis.js +148 -0
  149. package/src/expressions/expr.js +65 -0
  150. package/src/expressions/jsep.js +925 -0
  151. package/src/filterFirst.js +51 -0
  152. package/src/geom/geoJSON.js +121 -0
  153. package/src/iNaturalist/examples/api.inaturalist.org-v1-taxa!q=Acmispon americanus americanus.json +200 -0
  154. package/src/iNaturalist/examples/api.inaturalist.org-v1-taxa!q=Flabellina japonica.json +61 -0
  155. package/src/iNaturalist/examples/www.inaturalist.org-taxa-877359.json +132 -0
  156. package/src/iNaturalist/iNatUrls.js +17 -0
  157. package/src/iNaturalist/obsAPI.js +150 -0
  158. package/src/iNaturalist/taxonAPI.js +460 -0
  159. package/src/inatObs.js +120 -0
  160. package/src/inattify.js +123 -0
  161. package/src/intersect.js +80 -0
  162. package/src/intersectFilter.js +48 -0
  163. package/src/landlocked.js +137 -0
  164. package/src/lib/jquery.binarytransport.js +51 -0
  165. package/src/lib/lz4-commentary.txt +9 -0
  166. package/src/lib/lz4.js +73 -0
  167. package/src/lib/point-in-polygon.js +36 -0
  168. package/src/lz4.js +40 -0
  169. package/src/marmalise.js +333 -0
  170. package/src/materialise.js +68 -0
  171. package/src/reintegrate.js +71 -0
  172. package/src/taxonomise.js +615 -0
  173. package/src/unscrewGBIF.js +44 -0
  174. package/src/utils/dataSource.js +173 -0
  175. package/src/utils/settleStructure.js +62 -0
  176. package/src/utils/utils.js +92 -0
  177. package/src/viz/plethify.js +39 -0
  178. package/src/wormify.js +154 -0
  179. package/src/xetthecum/xetthecumFilters.js +126 -0
  180. package/test/testDataSource.js +182 -0
  181. package/test/testiNatObsSource.js +38 -0
  182. package/test/testiNatSource.js +50 -0
package/src/arphify.js ADDED
@@ -0,0 +1,662 @@
1
+ /* eslint-env node */
2
+
3
+ "use strict";
4
+
5
+ var fluid = require("infusion");
6
+ var minimist = require("minimist");
7
+ var moment = require("moment-timezone");
8
+ var ExcelJS = require("exceljs");
9
+ var fs = require("fs");
10
+
11
+ fluid.require("%imerss-bioinfo");
12
+
13
+ require("./dataProcessing/readJSON.js");
14
+ require("./dataProcessing/readCSV.js");
15
+ require("./dataProcessing/readCSVwithMap.js");
16
+ require("./dataProcessing/readCSVwithoutMap.js");
17
+ require("./dataProcessing/writeCSV.js");
18
+ require("./expressions/expr.js");
19
+ require("./utils/utils.js");
20
+
21
+ var hortis = fluid.registerNamespace("hortis");
22
+
23
+ hortis.ranks = Object.freeze(require("../data/ranks.json"));
24
+ hortis.reverseRanks = Object.freeze(fluid.makeArray(hortis.ranks).reverse());
25
+
26
+ fluid.setLogging(true);
27
+
28
+ var parsedArgs = minimist(process.argv.slice(2));
29
+
30
+ require("./WoRMS/taxonAPI.js");
31
+ var WoRMSTaxonAPIFileBase = "data/WoRMS/taxonAPI";
32
+
33
+ // Yes, if only it actually were a pipeline!
34
+ var pipeline = hortis.readJSONSync(parsedArgs.pipeline || "data/dataPaper-I-in/arpha-out.json5");
35
+
36
+ var summaryMap = hortis.readJSONSync(fluid.module.resolvePath(pipeline.summaryFileMap), "reading summary map file");
37
+
38
+ var summaryReader = hortis.csvReaderWithMap({
39
+ inputFile: fluid.module.resolvePath(pipeline.summaryFile),
40
+ mapColumns: summaryMap.columns
41
+ });
42
+
43
+ var obsMap = hortis.readJSONSync(fluid.module.resolvePath(pipeline.obsFileMap), "reading obs map file");
44
+
45
+ var obsReader = hortis.csvReaderWithMap({
46
+ inputFile: fluid.module.resolvePath(pipeline.obsFile),
47
+ mapColumns: obsMap.columns
48
+ });
49
+
50
+ var patchReader = pipeline.patchFile ? hortis.csvReaderWithoutMap({
51
+ inputFile: fluid.module.resolvePath(pipeline.patchFile)
52
+ }) : {
53
+ completionPromise: fluid.promise().resolve([]),
54
+ rows: []
55
+ };
56
+
57
+ var outputDir = fluid.module.resolvePath(pipeline.outputDir);
58
+ fs.mkdirSync(outputDir, { recursive: true });
59
+
60
+ hortis.genusEx = /\((.*)\)/;
61
+
62
+ hortis.normalise = function (str) {
63
+ return str.replace(/\s+/g, " ").trim();
64
+ };
65
+
66
+ // Variant for summaries
67
+ hortis.extractGenus = function (name, outRow) {
68
+ var matches = hortis.genusEx.exec(name);
69
+ var togo;
70
+ if (matches) {
71
+ outRow.Subgenus = matches[1];
72
+ togo = name.replace(hortis.genusEx, "");
73
+ } else {
74
+ togo = name;
75
+ }
76
+ return hortis.normalise(togo);
77
+ };
78
+
79
+ // Variant for obs -> Darwin Core
80
+ hortis.axeFromName = ["sp.", "ssp.", "spp.", "complex", "sp.A", "sp.B", "B"];
81
+
82
+ hortis.qualForming = ["agg", "aff.", "s.lat.", "cf", "sp.nov.", "var."];
83
+
84
+ hortis.extractSubgenus = function (seg, outRow) {
85
+ if (seg.startsWith("(") && seg.endsWith(")")) {
86
+ outRow.subgenus = seg.substring(1, seg.length - 1);
87
+ return 1;
88
+ } else {
89
+ return 0;
90
+ }
91
+ };
92
+
93
+ hortis.findRank = function (row, ranks) {
94
+ return ranks.find(function (rank) {
95
+ return !!row[rank];
96
+ });
97
+ };
98
+
99
+ /*
100
+ * AS of 19/7/21
101
+ * so yes, in the 'qualifier' field I think the only relevant values from our dataset are: 'cf', 'species complex', 'sp.A', and 'sp.B'
102
+ * in the 'identificationRemarks' field we will add the critical note
103
+ * so let's discard 'sp.' and 'spp.' from the catalogue
104
+ * we will only use it in the curated summary / checklists
105
+ */
106
+
107
+ hortis.extractQualifier = function (name, outRow) {
108
+ var words = name.split(" ");
109
+ // Unconditionally axe sp/spp/ssp from all words
110
+ var useWords = words.filter(function (word) {
111
+ return !hortis.axeFromName.includes(word);
112
+ });
113
+
114
+ // Filter out even qualifier-forming terms so that we can populate broken-out fields and scientificName
115
+ var bareWords = useWords.filter(function (word) {
116
+ return !hortis.qualForming.includes(word);
117
+ });
118
+
119
+ var sspPoint = 1 + hortis.extractSubgenus(bareWords[1] || "", outRow);
120
+
121
+ var lastBareWords = bareWords.slice(sspPoint);
122
+ var rank = hortis.findRank(outRow, hortis.reverseRanks);
123
+
124
+ if (rank === "genus" || rank === "species") {
125
+ outRow.genus = bareWords[0];
126
+ }
127
+
128
+ if (lastBareWords.length === 0) {
129
+ outRow.taxonRank = rank;
130
+ } else if (lastBareWords.length === 1) {
131
+ outRow.specificEpithet = lastBareWords[0];
132
+ outRow.taxonRank = "species";
133
+ } else if (lastBareWords.length === 2) {
134
+ outRow.specificEpithet = lastBareWords[0];
135
+ outRow.infraspecificEpithet = lastBareWords[1];
136
+ outRow.taxonRank = "subspecies";
137
+ }
138
+
139
+ var qualPoint = useWords.findIndex(function (word) {
140
+ return hortis.qualForming.includes(word);
141
+ });
142
+ if (outRow.subgenus) {
143
+ lastBareWords.unshift("(" + outRow.subgenus + ")");
144
+ }
145
+
146
+ outRow.identificationQualifier = qualPoint === -1 ? "" : useWords.slice(qualPoint).join(" ");
147
+ outRow.scientificName = [outRow.genus, ...lastBareWords].join(" ");
148
+
149
+ };
150
+
151
+ // Reviewer recommendation 7
152
+ hortis.normaliseSex = function (outRow) {
153
+ var sexes = [];
154
+ var sex = outRow.sex;
155
+ if (sex.match(/(([^f][^e]|^)male)|M/)) {
156
+ sexes.push("male");
157
+ }
158
+ if (sex.match(/female|F/)) {
159
+ sexes.push("female");
160
+ }
161
+ var newSexes = sexes.join(", ");
162
+ if (sex !== newSexes) {
163
+ outRow.occurrenceRemarks = outRow.sex;
164
+ outRow.sex = newSexes;
165
+ }
166
+ };
167
+
168
+ // Variant for summaries
169
+ hortis.extractSsp = function (name, outRow) {
170
+ var words = name.split(" ");
171
+
172
+ if (words.length === 3) {
173
+ var maybeSsp = words[2];
174
+ if (maybeSsp.startsWith("complex")
175
+ || maybeSsp.startsWith("agg")
176
+ || maybeSsp.startsWith("s.lat.")
177
+ || words[1].startsWith("cf")) {
178
+ outRow.Species = words[1] + " " + maybeSsp;
179
+ } else {
180
+ outRow.Species = words[1];
181
+ outRow.Subspecies = maybeSsp;
182
+ }
183
+ } else if (words.length === 2) {
184
+ outRow.Species = words[1];
185
+ } else {
186
+ fluid.fail("Unexpected species name " + name);
187
+ }
188
+ };
189
+
190
+ // Also in coordinatePatch.js, leafletMap.js
191
+ hortis.datasetIdFromObs = function (obsId) {
192
+ var colpos = obsId.indexOf(":");
193
+ return obsId.substring(0, colpos);
194
+ };
195
+
196
+ hortis.badDates = {};
197
+
198
+ hortis.formatDate = function (row, template) {
199
+ var togo = "";
200
+ var format = template.substring("!Date:".length);
201
+ // TODO: This should actually be applied in the data loader
202
+ var momentVal = moment.tz(row.dateObserved, "Canada/Pacific");
203
+ if (momentVal.isValid()) {
204
+ // RBCM records claim to have a time but they don't
205
+ var noTime = !row.dateObserved.includes("T") || row.dateObserved.includes("T00:00:00");
206
+ togo = noTime && format.includes("H") ? "" : momentVal.format(format);
207
+ } else {
208
+ var obsId = row.observationId;
209
+ if (!hortis.badDates[obsId]) {
210
+ console.log("WARNING: row " + row.observationId + " has invalid date " + row.dateObserved);
211
+ }
212
+ hortis.badDates[obsId] = true;
213
+ }
214
+ return togo;
215
+ };
216
+
217
+ // TODO: Of course we need to pipeline the whole of ARPHA export
218
+ hortis.quantiseDepth = function (outRow, places) {
219
+ var depth = outRow.verbatimDepth;
220
+ if (depth === "o-86") {
221
+ outRow.verbatimDepth = "0-86";
222
+ outRow.minimumDepthInMeters = "0";
223
+ outRow.maximumDepthInMeters = "86";
224
+ } else {
225
+ var togo = hortis.roundDecimals(depth, places);
226
+ if (togo && isNaN(togo)) {
227
+ console.log("WARNING: row " + outRow.occurrenceID + " has invalid depth " + depth);
228
+ }
229
+ outRow.verbatimDepth = togo;
230
+ }
231
+ };
232
+
233
+ // Table of bad "count" entries from PMLS data
234
+ hortis.badCountTable = {
235
+ "1 patc": "1 patch",
236
+ "1tiny": "1 tiny",
237
+ "2 (pair0": "2",
238
+ "2 larg": "2 large",
239
+ "?": "",
240
+ "`": "",
241
+ "i": "",
242
+ "-": "",
243
+ "snall": "",
244
+ "present": "",
245
+ "NA": "",
246
+ "white and cream": ""
247
+ };
248
+
249
+ // Technical reviewer recommendation 4
250
+ hortis.mapIndividualCount = function (outRow) {
251
+ var count = outRow.individualCount;
252
+ var lookup = hortis.badCountTable[count];
253
+ var mapped = lookup === undefined ? count : lookup;
254
+ if (mapped && !hortis.isInteger(mapped)) {
255
+ outRow.occurrenceRemarks = "Count: " + mapped;
256
+ outRow.individualCount = "";
257
+ } else {
258
+ outRow.individualCount = mapped;
259
+ }
260
+ };
261
+
262
+ hortis.countDecimals = function (value) {
263
+ var string = value.toString();
264
+ var dotpos = string.indexOf(".");
265
+ return dotpos === -1 ? dotpos : string.length - 1 - dotpos;
266
+ };
267
+
268
+
269
+ hortis.correctUncertainty = function (outRow) {
270
+ // 2nd round reviewer recommendation 9
271
+ var uncertainty = outRow.coordinateUncertaintyInMeters;
272
+ var val = hortis.parseFloat(uncertainty);
273
+ if (isNaN(val) || val < 1) {
274
+ outRow.coordinateUncertaintyInMeters = "";
275
+ }
276
+ // 3rd round reviewer recommendation 3
277
+ if (!outRow.coordinateUncertaintyInMeters) {
278
+ var decimals = Math.min(hortis.countDecimals(outRow.decimalLatitude), hortis.countDecimals(outRow.decimalLongitude));
279
+ if (decimals === -1) {
280
+ console.log("ERROR: unable to find coordinate resolution for row without coordinate uncertainty ", outRow);
281
+ } else { // 2 -> 4, 3 -> 3, 4 -> 2, 5-> 2
282
+ var scale = Math.min(Math.max(6 - decimals, 2), 4);
283
+ outRow.coordinateUncertaintyInMeters = Math.pow(10, scale);
284
+ var comment = "coordinate uncertainty inferred from coordinate resolution";
285
+ outRow.georeferenceRemarks = outRow.georeferenceRemarks ? outRow.georeferenceRemarks + "; " + comment : comment;
286
+ }
287
+ }
288
+ };
289
+
290
+ // 2nd round reviewer recommendation 10
291
+ hortis.correctLocalityTable = {
292
+ "e96b99b5-e978-49b0-a151-b5e5eb0f0981": "Trincomali Channel",
293
+ "29392e13-0c6b-4ea0-9c3d-bec39ae428a3": "Trincomali Channel",
294
+ "dfead0da-98f0-4ed1-b6c2-d270339704ee": "Trincomali Channel",
295
+ "f8ebf88e-6e0a-4a6e-a888-5f837d5326b6": "Trincomali Channel",
296
+ "0c5923fe-91e6-423e-bedf-1d46a0b89273": "Trincomali Channel"
297
+ };
298
+
299
+ hortis.correctLocality = function (outRow) {
300
+ var lookup = hortis.correctLocalityTable[outRow.occurrenceID];
301
+ if (lookup !== undefined) {
302
+ outRow.locality = lookup;
303
+ }
304
+ };
305
+
306
+ // 2nd round reviewer recommendation 11
307
+ hortis.cleanIdRemarks = function (outRow) {
308
+ outRow.identificationRemarks = outRow.identificationRemarks.replace(/\"/g, "");
309
+ };
310
+
311
+ hortis.normaliseRecorders = function (recordedBy) {
312
+ // Technical reviewer recommendation 5
313
+ var separators = recordedBy.replace("; ", " | ").trim();
314
+ // Technical reviewer recommendation 6
315
+ var togo = separators === "anonymous" ? "" : separators;
316
+ return togo;
317
+ };
318
+
319
+ // 2nd round reviewer recommendation 13
320
+ hortis.badEventRemarksTable = {
321
+ "/": "",
322
+ "?": "",
323
+ "??": "",
324
+ "#NAME?": ""
325
+ };
326
+
327
+ hortis.cleanEventRemarks = function (outRow) {
328
+ var lookup = hortis.badEventRemarksTable[outRow.eventRemarks];
329
+ if (lookup !== undefined) {
330
+ outRow.eventRemarks = lookup;
331
+ }
332
+ };
333
+
334
+ /* Accepts a row of a summary, and the column structure inside "Taxa" */
335
+
336
+ hortis.mapTaxaRows = function (rows, columns, materialsMap) {
337
+ return fluid.transform(rows, function (row) {
338
+ var summaryRow = materialsMap.summaryIndex[row.iNaturalistTaxonId];
339
+ var togo = {};
340
+ fluid.each(columns, function (template, target) {
341
+ togo[target] = hortis.stringTemplate(template, row);
342
+ });
343
+ if (row.species !== "" || row.genus !== "") {
344
+ var degenified = hortis.extractGenus(row.taxonName, togo);
345
+ hortis.extractSsp(degenified, togo);
346
+ }
347
+ if (summaryRow) {
348
+ // Account for observations which have been hacked by assigning them to proxy iNat taxa - in theory we should
349
+ // copy more fields but we have managed to assign these to, e.g. proxies in the same family
350
+ var proxySummary = fluid.copy(summaryRow);
351
+ // Replicate this piece of workflow from mapMaterialsRows
352
+ hortis.extractQualifier(summaryRow.taxonName, proxySummary);
353
+ togo.Genus = proxySummary.genus;
354
+ }
355
+ return togo;
356
+ });
357
+ };
358
+
359
+ hortis.stashMismatchedRow = function (mismatches, patchIndex, obsRow, summaryRow) {
360
+ var key = obsRow.previousIdentifications + "|" + obsRow.scientificName;
361
+ var patchRow = patchIndex[key];
362
+ if (patchRow) {
363
+ if (patchRow.Disposition === "P") {
364
+ obsRow.scientificName = obsRow.previousIdentifications;
365
+ } else if (patchRow.Disposition === "S") {
366
+ var desp = obsRow.previousIdentifications.replace(" sp.", "");
367
+ obsRow.scientificName = desp + " sp.";
368
+ } else if (patchRow.Disposition === "X") {
369
+ console.log("!!! Unexpected use of patch with key " + key);
370
+ }
371
+ } else {
372
+ var existing = mismatches[key];
373
+ if (!existing) {
374
+ mismatches[key] = fluid.extend({
375
+ previousIdentifications: obsRow.previousIdentifications
376
+ }, summaryRow);
377
+ console.log("Stashing mismatched row ", mismatches[key]);
378
+ }
379
+ }
380
+ };
381
+
382
+ hortis.mapMaterialsRows = function (rows, patchIndex, materialsMap, references, columns) {
383
+ return fluid.transform(rows, function (row) {
384
+ var togo = {};
385
+ var dataset = hortis.datasetIdFromObs(row.observationId);
386
+ var summaryRow = materialsMap.summaryIndex[row.iNaturalistTaxonId];
387
+ var termMap = fluid.extend({}, row, {
388
+ summary: summaryRow
389
+ });
390
+ // row.scientificName = summaryRow ? summaryRow.taxonName : "";
391
+ var refBlock = references[dataset];
392
+ fluid.each(columns, function (template, target) {
393
+ var outVal = "";
394
+ if (refBlock && refBlock[target]) {
395
+ outVal = refBlock[target];
396
+ }
397
+ if (template.startsWith("!references.")) {
398
+ var ref = template.substring("!references.".length);
399
+ outVal = refBlock && refBlock[ref] || "";
400
+ } else if (template.startsWith("!Date:")) {
401
+ outVal = hortis.formatDate(row, template);
402
+ } else {
403
+ outVal = hortis.stringTemplate(template, termMap) || outVal;
404
+ }
405
+ if (outVal === "Confidence: ") { // blatant special casing
406
+ outVal = "";
407
+ }
408
+ togo[target] = outVal;
409
+ });
410
+ if (!togo.occurrenceID) {
411
+ togo.occurrenceID = "imerss.org:" + row.observationId;
412
+ }
413
+
414
+ if (row.coordinatesCorrected === "yes") {
415
+ togo.georeferencedBy = "Andrew Simon";
416
+ togo.georeferenceProtocol = "interpretation of locality, and/or inference based on local knowledge and species ecology";
417
+ togo.georeferenceVerificationStatus = "corrected";
418
+ togo.georeferenceRemarks = row.coordinatesCorrectedNote;
419
+ }
420
+ // Note that previousIdentifications is taken from the row's own "taxonName" field from the original obs
421
+ if (togo.scientificName !== togo.previousIdentifications) {
422
+ hortis.stashMismatchedRow(materialsMap.mismatches, patchIndex, togo, summaryRow);
423
+ }
424
+ hortis.extractQualifier(togo.scientificName, togo);
425
+ hortis.normaliseSex(togo);
426
+ if (summaryRow && summaryRow.criticalNotes) {
427
+ togo.identificationRemarks = summaryRow.criticalNotes;
428
+ }
429
+
430
+ // var filename = hortis.WoRMSTaxa.filenameFromTaxonName(WoRMSTaxonAPIFileBase, row.iNaturalistTaxonName);
431
+ // var wormsRec = hortis.readJSONSync(filename);
432
+ // togo.taxonID = "WoRMS:" + wormsRec.AphiaID;
433
+ togo.taxonID = "iNaturalist:" + row.iNaturalistTaxonId;
434
+
435
+ togo.recordedBy = hortis.normaliseRecorders(togo.recordedBy);
436
+ togo.georeferencedBy = hortis.normaliseRecorders(togo.georeferencedBy);
437
+
438
+ hortis.quantiseDepth(togo, 2);
439
+ hortis.mapIndividualCount(togo);
440
+ hortis.correctUncertainty(togo);
441
+ hortis.correctLocality(togo);
442
+ hortis.cleanIdRemarks(togo);
443
+ hortis.cleanEventRemarks(togo);
444
+ return togo;
445
+ });
446
+ };
447
+
448
+ hortis.writeSheet = function (workbook, sheetName, rows) {
449
+ var sheet = workbook.addWorksheet(sheetName);
450
+ var keys = Object.keys(rows[0]);
451
+ var header = sheet.getRow(1);
452
+ keys.forEach(function (key, index) {
453
+ header.getCell(index + 1).value = key;
454
+ });
455
+ rows.forEach(function (row, rowIndex) {
456
+ var sheetRow = sheet.getRow(rowIndex + 2);
457
+ keys.forEach(function (key, index) {
458
+ sheetRow.getCell(index + 1).value = row[key];
459
+ });
460
+ });
461
+ };
462
+
463
+ hortis.writeExcel = function (sheets, key, outputDir) {
464
+ if (sheets.Taxa.length === 0) {
465
+ console.log("Skipping key " + key + " since no rows were selected");
466
+ return fluid.promise().resolve();
467
+ }
468
+ var workbook = new ExcelJS.Workbook();
469
+
470
+ fluid.each(sheets, function (sheet, sheetName) {
471
+ hortis.writeSheet(workbook, sheetName, sheet);
472
+ });
473
+
474
+ var filename = outputDir + "/" + key + ".xlsx";
475
+ var togo = workbook.xlsx.writeFile(filename);
476
+ togo.then(function () {
477
+ var stats = fs.statSync(filename);
478
+ console.log("Written " + stats.size + " bytes to " + filename);
479
+ });
480
+ return togo;
481
+ };
482
+
483
+ hortis.indexSummary = function (summaryRows) {
484
+ var togo = {};
485
+ summaryRows.forEach(function (row) {
486
+ togo[row.iNaturalistTaxonId] = row;
487
+ if (row.subtaxonAuthority) { // All downstream steps expect only a single authority
488
+ row.authority = row.subtaxonAuthority;
489
+ }
490
+ });
491
+ return togo;
492
+ };
493
+
494
+ hortis.indexPatchRows = function (patchRows) {
495
+ var togo = {};
496
+ patchRows.forEach(function (row) {
497
+ togo[row.previousIdentifications + "|" + row.taxonName] = row;
498
+ });
499
+ return togo;
500
+ };
501
+
502
+ // TODO: Worry if obs and summaries diverge in taxonomy
503
+ hortis.filterArphaRows = function (rows, rec, rowCount) {
504
+ return rows.filter(function (row, index) {
505
+ var parsed = hortis.expr.parse(rec.filter);
506
+ var match = hortis.expr.evaluate(parsed, row);
507
+ if (match) {
508
+ ++rowCount[index];
509
+ }
510
+ return match;
511
+ });
512
+ };
513
+
514
+ // Note - argument modified
515
+ hortis.sortRows = function (rows, sortBy) {
516
+ var comparator = function (ra, rb) {
517
+ return fluid.find(sortBy, function (column) {
518
+ return ra[column] > rb[column] ? 1 : (ra[column] < rb[column] ? -1 : undefined);
519
+ });
520
+ };
521
+ rows.sort(comparator);
522
+ };
523
+
524
+ hortis.verifyCounts = function (name, rowCount, rows) {
525
+ rowCount.forEach(function (count, index) {
526
+ if (count !== 1) {
527
+ console.log("Anomalous " + name + " count for row " + index + ": " + count);
528
+ console.log("Row contents: ", rows[index]);
529
+ }
530
+ });
531
+ };
532
+
533
+ hortis.eliminateEmptyColumns = function (rows) {
534
+ var hasValue = {};
535
+ rows.forEach(function (row) {
536
+ fluid.each(row, function (value, key) {
537
+ if (fluid.isValue(value) && value !== "") {
538
+ hasValue[key] = true;
539
+ }
540
+ });
541
+ });
542
+ var valueKeys = Object.keys(hasValue);
543
+ var togo = fluid.transform(rows, function (row) {
544
+ return fluid.filterKeys(row, valueKeys);
545
+ });
546
+ return togo;
547
+ };
548
+
549
+ hortis.checkDuplicates = function (rows, eliminate) {
550
+ var columns = ["occurrenceID"];
551
+ var byDataset = {};
552
+
553
+ columns.forEach(function (column) {
554
+ console.log("Checking near duplicates by omitting column " + column);
555
+ var buckets = {};
556
+ var emittedKeys = {};
557
+ var emitDuplicate = function (row) {
558
+ var key = row[column];
559
+ if (!emittedKeys[key]) {
560
+ fluid.pushArray(byDataset, row.institutionCode, row);
561
+ emittedKeys[key] = true;
562
+ }
563
+ };
564
+ var outRows = [];
565
+ var duplicates = 0;
566
+ rows.forEach(function (row) {
567
+ var filtered = fluid.censorKeys(row, [column]);
568
+ var omitted = row[column] || true;
569
+ var hash = Object.values(filtered).join("|");
570
+ var oldValue = buckets[hash];
571
+ if (oldValue) {
572
+ emitDuplicate(oldValue);
573
+ emitDuplicate(row);
574
+ fluid.log("ERROR: duplicate row with contents " + hash + " and key " + omitted);
575
+ ++duplicates;
576
+ } else {
577
+ buckets[hash] = row;
578
+ outRows.push(row);
579
+ }
580
+ });
581
+ if (duplicates) {
582
+ console.log("Found " + duplicates + " duplicates");
583
+ }
584
+ if (eliminate && rows.length !== outRows.length) {
585
+ console.log("After filtering duplicates, " + rows.length + " rows reduced to " + outRows.length);
586
+ rows = outRows;
587
+ }
588
+ });
589
+ fluid.each(byDataset.iNaturalist, function (row) {
590
+ var id = row.occurrenceID;
591
+ var iNatObs = id.substring(id.lastIndexOf(":") + 1);
592
+ row.observationUrl = "https://www.inaturalist.org/observations/" + iNatObs;
593
+ });
594
+ fluid.each(byDataset, function (oneDataset, key) {
595
+ var outfile = "materials-duplicates-" + key + ".csv";
596
+ hortis.writeCSV(outfile, Object.keys(oneDataset[0]), oneDataset, fluid.promise());
597
+ });
598
+ return rows;
599
+ };
600
+
601
+ var completion = fluid.promise.sequence([summaryReader.completionPromise, obsReader.completionPromise, patchReader.completionPromise]);
602
+
603
+ completion.then(function () {
604
+ var summaryRows = summaryReader.rows;
605
+ console.log("Summary Input: " + summaryRows.length + " rows");
606
+ var summaryRowCount = fluid.generate(summaryRows.length, 0);
607
+ var obsRows = obsReader.rows;
608
+ console.log("Obs Input: " + obsRows.length + " rows");
609
+ var obsRowCount = fluid.generate(obsRows.length, 0);
610
+ var summaryIndex = hortis.indexSummary(summaryRows); // also overrides authority with subtaxonAuthority
611
+ var patchRows = patchReader.rows;
612
+ console.log("Patch Input: " + patchRows.length + " rows");
613
+ var patchIndex = hortis.indexPatchRows(patchRows);
614
+ var materialsMap = {
615
+ summaryIndex: summaryIndex,
616
+ mismatches: {}
617
+ };
618
+ var now = Date.now();
619
+ var allMaterials = [];
620
+ var outs = fluid.transform(pipeline.files, function (rec, key) {
621
+ var outSummaryRows = hortis.filterArphaRows(summaryRows, rec, summaryRowCount);
622
+ console.log("Extracted " + outSummaryRows.length + " summary rows via filter " + key);
623
+
624
+ var Taxa = pipeline.sheets.Taxa;
625
+ var taxaRows = hortis.mapTaxaRows(outSummaryRows, Taxa.columns, materialsMap);
626
+ hortis.sortRows(taxaRows, Taxa.sortBy);
627
+
628
+ var outObsRows = hortis.filterArphaRows(obsRows, rec, obsRowCount);
629
+ console.log("Extracted " + outObsRows.length + " obs rows via filter " + key);
630
+
631
+ var materialsRows = hortis.mapMaterialsRows(outObsRows, patchIndex, materialsMap, pipeline.references, pipeline.sheets.Materials.columns);
632
+
633
+ // This is the export file destined for GBIF
634
+ allMaterials = allMaterials.concat(materialsRows);
635
+
636
+ return {
637
+ Taxa: taxaRows,
638
+ Materials: materialsRows,
639
+ ExternalLinks: [fluid.copy(pipeline.sheets.ExternalLinks.columns)]
640
+ };
641
+ });
642
+ console.log("Total extracted obs rows: " + fluid.flatten(fluid.getMembers(outs, "Materials")).length);
643
+ console.log("Filtered obs in " + (Date.now() - now) + " ms");
644
+ hortis.verifyCounts("summary", summaryRowCount, summaryRows);
645
+ hortis.verifyCounts("obs", obsRowCount, obsRows);
646
+ var mismatches = Object.values(materialsMap.mismatches);
647
+ if (mismatches.length > 0) {
648
+ console.log("Writing " + mismatches.length + " mismatched rows to arphaMismatches.csv");
649
+ hortis.writeCSV("arphaMismatches.csv", ["previousIdentifications", "taxonName"].concat(Object.keys(fluid.censorKeys(summaryRows[0], ["taxonName"]))), mismatches, fluid.promise());
650
+ }
651
+
652
+ hortis.sortRows(allMaterials, pipeline.sheets.Materials.sortBy);
653
+ var filteredMaterials = hortis.eliminateEmptyColumns(allMaterials);
654
+ filteredMaterials = hortis.checkDuplicates(filteredMaterials, true);
655
+ hortis.writeCSV(outputDir + "/Materials.csv", Object.keys(filteredMaterials[0]), filteredMaterials, fluid.promise());
656
+
657
+ fluid.each(outs, function (sheets, key) {
658
+ hortis.writeExcel(sheets, key, outputDir);
659
+ });
660
+ }, function (err) {
661
+ console.log("Error ", err);
662
+ });