imerss-bioinfo 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/README.md +156 -0
  2. package/build.js +196 -0
  3. package/docs/css/imerss-viz-all.css +1227 -0
  4. package/docs/css/imerss-viz-wp-overrides.css +70 -0
  5. package/docs/css/pepiowelh.css +76 -0
  6. package/docs/css/xetthecum-external.css +47 -0
  7. package/docs/css/xetthecum-shared.css +80 -0
  8. package/docs/css/xetthecum.css +526 -0
  9. package/docs/js/imerss-viz-all.js +58 -0
  10. package/docs/js/imerss-viz-all.js.map +1 -0
  11. package/docs/js/jquery.js +10881 -0
  12. package/index.html +81 -0
  13. package/index.js +7 -0
  14. package/indexWithJustMap.html +92 -0
  15. package/indexWithMap.html +79 -0
  16. package/indexWithMapHoweSound.html +91 -0
  17. package/indexWithMapPepiowelh.html +119 -0
  18. package/indexWithMapValdes.html +90 -0
  19. package/indexWithMapXetthecum.html +90 -0
  20. package/package.json +50 -0
  21. package/src/WoRMS/taxonAPI.js +13 -0
  22. package/src/arphify.js +662 -0
  23. package/src/assignNames.js +103 -0
  24. package/src/auxBuild/restoreJQuery.js +1 -0
  25. package/src/buildSource/Comprehensive Lists.html +42 -0
  26. package/src/buildSource/Data Paper Part I Visualisation.html +50 -0
  27. package/src/buildSource/Galiano Life List.html +43 -0
  28. package/src/buildSource/Metchosin/index.html +22 -0
  29. package/src/buildSource/Pe'pi'ow'elh.html +72 -0
  30. package/src/buildSource/Squamish Life List.html +48 -0
  31. package/src/buildSource/Valdes Island Biodiversity.html +64 -0
  32. package/src/buildSource/Xetthecum.html +40 -0
  33. package/src/buildSource/dataPaperSunburstAndMap.html +30 -0
  34. package/src/buildSource/index.html +224 -0
  35. package/src/client/css/imerss-viz-wp-overrides.css +70 -0
  36. package/src/client/css/imerss-viz.css +531 -0
  37. package/src/client/css/pepiowelh.css +76 -0
  38. package/src/client/css/xetthecum-external.css +47 -0
  39. package/src/client/css/xetthecum-shared.css +80 -0
  40. package/src/client/css/xetthecum.css +526 -0
  41. package/src/client/html/imerss-viz-map-only.html +75 -0
  42. package/src/client/html/imerss-viz-map-pepi.html +77 -0
  43. package/src/client/html/imerss-viz-map-xetthecum.html +90 -0
  44. package/src/client/html/imerss-viz-map.html +75 -0
  45. package/src/client/html/imerss-viz.html +58 -0
  46. package/src/client/html/importing Xetthecum patterns.txt +10 -0
  47. package/src/client/html/xetthecum-patterns.html +27 -0
  48. package/src/client/img/Animalia_2.svg +1 -0
  49. package/src/client/img/Animalia_Annelida.svg +655 -0
  50. package/src/client/img/Animalia_Chaetognatha.svg +72 -0
  51. package/src/client/img/Animalia_Cnidaria.svg +173 -0
  52. package/src/client/img/Animalia_Ctenophora.svg +298 -0
  53. package/src/client/img/Animalia_Nemertea.svg +54 -0
  54. package/src/client/img/Animalia_Phoronida.svg +271 -0
  55. package/src/client/img/Animalia_Platyhelminthes.svg +58 -0
  56. package/src/client/img/Animalia_Porifera.svg +220 -0
  57. package/src/client/img/Animalia_Sipuncula.svg +155 -0
  58. package/src/client/img/Arachnoidiscus_schmidtii_web_1-2-scaled.webp +0 -0
  59. package/src/client/img/Arrow-down.svg +11 -0
  60. package/src/client/img/Arrow-up.svg +6 -0
  61. package/src/client/img/Chromista.svg +1 -0
  62. package/src/client/img/Pepiowelh/BlackNoWe2-wb-bb.png +0 -0
  63. package/src/client/img/Pepiowelh/BlackNoWe2-wb.png +0 -0
  64. package/src/client/img/Pepiowelh/BlackNoWe2.png +0 -0
  65. package/src/client/img/Pepiowelh/Ecocultural Mapping Banner.svg +1 -0
  66. package/src/client/img/Plankton.jpg +0 -0
  67. package/src/client/img/Plantae.svg +1 -0
  68. package/src/client/img/Plantae_2.svg +1 -0
  69. package/src/client/img/Protista.svg +1 -0
  70. package/src/client/img/Valdes/Valdes Island Conservancy.jpg +0 -0
  71. package/src/client/img/Xetthecum/Agricultural.jpg +0 -0
  72. package/src/client/img/Xetthecum/Anthropogenic_banner.jpg +0 -0
  73. package/src/client/img/Xetthecum/Broad_Leaf.jpg +0 -0
  74. package/src/client/img/Xetthecum/Caves.jpg +0 -0
  75. package/src/client/img/Xetthecum/Coastal_Bluffs.jpg +0 -0
  76. package/src/client/img/Xetthecum/Conifer_Pole_Sapling.jpg +0 -0
  77. package/src/client/img/Xetthecum/Developed.jpg +0 -0
  78. package/src/client/img/Xetthecum/Dock.jpg +0 -0
  79. package/src/client/img/Xetthecum/Eelgrass.jpg +0 -0
  80. package/src/client/img/Xetthecum/Forests_banner.jpg +0 -0
  81. package/src/client/img/Xetthecum/Freshwater_banner.jpg +0 -0
  82. package/src/client/img/Xetthecum/Greig_Creek.jpg +0 -0
  83. package/src/client/img/Xetthecum/Intertidal.jpg +0 -0
  84. package/src/client/img/Xetthecum/Laughlin_Lake.jpg +0 -0
  85. package/src/client/img/Xetthecum/Marine_banner.jpg +0 -0
  86. package/src/client/img/Xetthecum/Mature_Conifer_Forest.jpg +0 -0
  87. package/src/client/img/Xetthecum/Porlier_Pass_Road.jpg +0 -0
  88. package/src/client/img/Xetthecum/Riparian.jpg +0 -0
  89. package/src/client/img/Xetthecum/Rural.jpg +0 -0
  90. package/src/client/img/Xetthecum/Subtidal.jpg +0 -0
  91. package/src/client/img/Xetthecum/Wetland.jpg +0 -0
  92. package/src/client/img/Xetthecum/Woodland.jpg +0 -0
  93. package/src/client/img/Xetthecum/Woodland_banner.jpg +0 -0
  94. package/src/client/img/Xetthecum/Xetthecum.jpg +0 -0
  95. package/src/client/img/Xetthecum/Young_Conifer_Forest.jpg +0 -0
  96. package/src/client/img/Xetthecum/Young_Mixed_Forest.jpg +0 -0
  97. package/src/client/img/Xetthecum/food.svg +38 -0
  98. package/src/client/img/Xetthecum/indicator.svg +52 -0
  99. package/src/client/img/Xetthecum/material.svg +186 -0
  100. package/src/client/img/Xetthecum/medicinal.svg +62 -0
  101. package/src/client/img/Xetthecum/missing-value.svg +3 -0
  102. package/src/client/img/Xetthecum/spiritual.svg +13 -0
  103. package/src/client/img/Xetthecum/trade.svg +74 -0
  104. package/src/client/img/audio.svg +5 -0
  105. package/src/client/img/external-link.svg +3 -0
  106. package/src/client/img/selected-eye.svg +4 -0
  107. package/src/client/img/selected-tick.svg +3 -0
  108. package/src/client/img/thumbs/BCLeps.jpg +0 -0
  109. package/src/client/img/thumbs/Checklist.jpg +0 -0
  110. package/src/client/img/thumbs/Data-Paper-Figma.jpg +0 -0
  111. package/src/client/img/thumbs/Galiano-Data-Paper.jpg +0 -0
  112. package/src/client/img/thumbs/Galiano.jpg +0 -0
  113. package/src/client/img/thumbs/Maxwell.jpg +0 -0
  114. package/src/client/img/thumbs/Squamish.jpg +0 -0
  115. package/src/client/img/thumbs/Valdes.jpg +0 -0
  116. package/src/client/img/thumbs/Xetthecum.jpg +0 -0
  117. package/src/client/img/up-arrow.svg +4 -0
  118. package/src/client/js/autocomplete.js +53 -0
  119. package/src/client/js/checklist.js +179 -0
  120. package/src/client/js/colour.js +139 -0
  121. package/src/client/js/datasetControls.js +212 -0
  122. package/src/client/js/imerss-viz.js +1397 -0
  123. package/src/client/js/leafletMap.js +357 -0
  124. package/src/client/js/leafletMapWithGrid.js +337 -0
  125. package/src/client/js/leafletMapWithRegions.js +381 -0
  126. package/src/client/js/renderSVG.js +108 -0
  127. package/src/client/js/tabs.js +53 -0
  128. package/src/client/js/xetthecum.js +106 -0
  129. package/src/client/json/emptyPhyloMap.json +1 -0
  130. package/src/client/json/kingdomPhyloMap.json +17 -0
  131. package/src/client/json/phyloMap.json +58 -0
  132. package/src/compare.js +97 -0
  133. package/src/compilecsv.js +47 -0
  134. package/src/dataProcessing/LZ4.js +27 -0
  135. package/src/dataProcessing/coordinatePatch.js +151 -0
  136. package/src/dataProcessing/readCSV.js +107 -0
  137. package/src/dataProcessing/readCSVwithMap.js +90 -0
  138. package/src/dataProcessing/readCSVwithoutMap.js +25 -0
  139. package/src/dataProcessing/readJSON.js +23 -0
  140. package/src/dataProcessing/sqlite.js +174 -0
  141. package/src/dataProcessing/summarise.js +122 -0
  142. package/src/dataProcessing/taxonScrape.js +14 -0
  143. package/src/dataProcessing/worker.js +31 -0
  144. package/src/dataProcessing/writeCSV.js +34 -0
  145. package/src/dataProcessing/writeJSON.js +15 -0
  146. package/src/dbtaxa.js +96 -0
  147. package/src/deAuthorise.js +48 -0
  148. package/src/deqgis.js +148 -0
  149. package/src/expressions/expr.js +65 -0
  150. package/src/expressions/jsep.js +925 -0
  151. package/src/filterFirst.js +51 -0
  152. package/src/geom/geoJSON.js +121 -0
  153. package/src/iNaturalist/examples/api.inaturalist.org-v1-taxa!q=Acmispon americanus americanus.json +200 -0
  154. package/src/iNaturalist/examples/api.inaturalist.org-v1-taxa!q=Flabellina japonica.json +61 -0
  155. package/src/iNaturalist/examples/www.inaturalist.org-taxa-877359.json +132 -0
  156. package/src/iNaturalist/iNatUrls.js +17 -0
  157. package/src/iNaturalist/obsAPI.js +150 -0
  158. package/src/iNaturalist/taxonAPI.js +460 -0
  159. package/src/inatObs.js +120 -0
  160. package/src/inattify.js +123 -0
  161. package/src/intersect.js +80 -0
  162. package/src/intersectFilter.js +48 -0
  163. package/src/landlocked.js +137 -0
  164. package/src/lib/jquery.binarytransport.js +51 -0
  165. package/src/lib/lz4-commentary.txt +9 -0
  166. package/src/lib/lz4.js +73 -0
  167. package/src/lib/point-in-polygon.js +36 -0
  168. package/src/lz4.js +40 -0
  169. package/src/marmalise.js +333 -0
  170. package/src/materialise.js +68 -0
  171. package/src/reintegrate.js +71 -0
  172. package/src/taxonomise.js +615 -0
  173. package/src/unscrewGBIF.js +44 -0
  174. package/src/utils/dataSource.js +173 -0
  175. package/src/utils/settleStructure.js +62 -0
  176. package/src/utils/utils.js +92 -0
  177. package/src/viz/plethify.js +39 -0
  178. package/src/wormify.js +154 -0
  179. package/src/xetthecum/xetthecumFilters.js +126 -0
  180. package/test/testDataSource.js +182 -0
  181. package/test/testiNatObsSource.js +38 -0
  182. package/test/testiNatSource.js +50 -0
@@ -0,0 +1,333 @@
1
+ /* eslint-env node */
2
+
3
+ "use strict";
4
+
5
+ const fluid = require("infusion");
6
+ const minimist = require("minimist");
7
+ const fs = require("fs");
8
+
9
+ require("./utils/utils.js");
10
+ require("./dataProcessing/readJSON.js");
11
+ require("./dataProcessing/readCSV.js");
12
+ require("./dataProcessing/readCSVwithMap.js");
13
+ require("./dataProcessing/LZ4.js");
14
+ require("./iNaturalist/taxonAPI.js");
15
+
16
+ const hortis = fluid.registerNamespace("hortis");
17
+
18
+ hortis.ranks = fluid.freezeRecursive(require("../data/ranks.json"));
19
+
20
+ hortis.newTaxon = function (iNaturalistTaxonName, rank, depth, counts) {
21
+ const togo = {
22
+ iNaturalistTaxonName: iNaturalistTaxonName,
23
+ id: fluid.allocateGuid(),
24
+ rank: rank,
25
+ depth: depth,
26
+ childCount: 0,
27
+ children: {}
28
+ };
29
+ fluid.each(counts, function (oneCount, key) {
30
+ togo[key] = 0;
31
+ });
32
+ return togo;
33
+ };
34
+
35
+ hortis.addCounts = function (row, counts) {
36
+ fluid.each(counts, function (countDef, key) {
37
+ const countField = row[countDef.column];
38
+ row[key] = countDef.free ? Number(countField) : (countField === countDef.equals ? 1 : 0);
39
+ });
40
+ };
41
+
42
+ hortis.cleanRow = function (row) {
43
+ fluid.each(row, function (value, key) {
44
+ if (typeof(value) === "string") {
45
+ const trimmed = value.trim();
46
+ if (trimmed === "" || trimmed === "—") { // Note funny AS hyphen in here
47
+ delete row[key];
48
+ } else {
49
+ row[key] = trimmed;
50
+ }
51
+ }
52
+ });
53
+ };
54
+
55
+ hortis.rowToTaxon = function (row, depth, counts) {
56
+ row.depth = depth;
57
+ row.id = fluid.allocateGuid();
58
+ row.childCount = 1;
59
+ row.children = {};
60
+ hortis.addCounts(row, counts);
61
+ };
62
+
63
+ hortis.fullRecordTransform = {
64
+ "wikipediaSummary": "wikipedia_summary",
65
+ "commonName": "common_name.name",
66
+ "iNaturalistTaxonId": "id",
67
+ "iNaturalistTaxonImage": "default_photo.medium_url"
68
+ };
69
+
70
+ hortis.addTaxonInfo = function (row, fullRecord) {
71
+ fluid.each(hortis.fullRecordTransform, function (path, target) {
72
+ const source = fluid.get(fullRecord, path);
73
+ if (row[target] === undefined) {
74
+ row[target] = source;
75
+ }
76
+ });
77
+ };
78
+
79
+ hortis.reverseMerge = function (target, source) {
80
+ fluid.each(source, function (value, key) {
81
+ if (!target[key]) {
82
+ target[key] = value;
83
+ }
84
+ });
85
+ };
86
+
87
+ hortis.storeAtPath = function (treeBuilder, path, row) {
88
+ const counts = treeBuilder.map.counts;
89
+ hortis.rowToTaxon(row, path.length, counts);
90
+ let node = treeBuilder.tree;
91
+ path.forEach(function (doc, index) {
92
+ const last = index === path.length - 1;
93
+ const name = doc.name;
94
+ let child = node.children[name];
95
+ if (!child) {
96
+ child = node.children[name] = (last ? row : hortis.newTaxon(name, doc.rank, index + 1, counts));
97
+ try {
98
+ hortis.addTaxonInfo(child, doc);
99
+ } catch (e) {
100
+ console.log("While storing row ", row);
101
+ throw e;
102
+ }
103
+ } else if (last) { // Store any surplus fields, expecially include "coords" here otherwise they will be lost
104
+ hortis.reverseMerge(child, row);
105
+ // It may already have had an id allocated as a result of being observed as a lower taxon - make sure
106
+ // row records this since this id may be used when assigning obs to regions
107
+ row.id = child.id;
108
+ }
109
+ ++node.childCount;
110
+ fluid.each(counts, function (countDef, key) {
111
+ node[key] += row[key];
112
+ });
113
+ node = child;
114
+ });
115
+ };
116
+ /*
117
+ hortis.loadCachedTaxonDoc = function (treeBuilder, id) {
118
+ let existing = treeBuilder.taxonCache[id];
119
+ if (!existing) {
120
+ existing = treeBuilder.taxonCache[id] = hortis.iNat.loadTaxonDoc(treeBuilder.options.taxonAPIFileBase, id);
121
+ }
122
+ // This otherwise causes rank to be "undefined" in the interface - AS report 18/4/22
123
+ // Not sure what to do about this long-term
124
+ if (existing.rank === "complex") {
125
+ existing.rank = "species";
126
+ }
127
+ return existing;
128
+ };
129
+ */
130
+
131
+ /** Produce a "path" holding each cached taxon doc from the supplied row from the root
132
+ * @param {TreeBuilder} treeBuilder - The treeBuilder instance
133
+ * @param {Row} row - An observation or summary row
134
+ * @return {Promise<TaxonDoc[]>} Array of taxon docs starting at the root and ending at the row's doc
135
+ */
136
+ hortis.taxaToPathiNat = async function (treeBuilder, row) {
137
+ const baseDoc = (await treeBuilder.taxonSource.get({id: row.iNaturalistTaxonId})).doc;
138
+ const parentTaxaIds = hortis.iNat.parentTaxaIds(baseDoc).reverse();
139
+ const ancestourDocs = await Promise.all(parentTaxaIds.map(async function (id) {
140
+ return (await treeBuilder.taxonSource.get({id: id})).doc;
141
+ }));
142
+ const rankDocs = ancestourDocs.filter(function (oneDoc) {
143
+ return treeBuilder.options.vizRanks.includes(oneDoc.rank);
144
+ });
145
+ return rankDocs.concat([baseDoc]);
146
+ };
147
+
148
+
149
+ hortis.applyRowsToTree = async function (treeBuilder, rows) {
150
+ for (let i = 0; i < rows.length; ++i) {
151
+ const row = rows[i];
152
+ if (i % 100 === 0) {
153
+ process.stdout.write(i + " ... ");
154
+ }
155
+ hortis.cleanRow(row);
156
+ const path = await hortis.taxaToPathiNat(treeBuilder, row);
157
+ treeBuilder.storeAtPath(path, row);
158
+ }
159
+ console.log("");
160
+ };
161
+
162
+ hortis.flattenChildren = function (root) {
163
+ const children = [];
164
+ fluid.each(root.children, function (value) {
165
+ children.push(hortis.flattenChildren(value));
166
+ });
167
+ root.children = children;
168
+ return root;
169
+ };
170
+
171
+ hortis.parseCounts = function (counts) {
172
+ fluid.each(counts, function (oneCount) {
173
+ if (oneCount.equals === "") {
174
+ oneCount.equals = undefined;
175
+ }
176
+ });
177
+ };
178
+
179
+ hortis.loadMapWithCounts = function (mapFile) {
180
+ const map = hortis.readJSONSync(fluid.module.resolvePath(mapFile));
181
+ hortis.parseCounts(map.counts);
182
+ return map;
183
+ };
184
+
185
+ hortis.rootNode = function (map) {
186
+ return hortis.newTaxon("Life", "life", 0, map.counts);
187
+ };
188
+
189
+ hortis.filterVizRanks = function (skipRanks) {
190
+ return hortis.ranks.filter(function (rank) {
191
+ return !skipRanks.includes(rank);
192
+ });
193
+ };
194
+
195
+ fluid.defaults("hortis.treeBuilder", {
196
+ gradeNames: "fluid.component",
197
+ taxonAPIFileBase: "data/iNaturalist/taxonAPI",
198
+ mapFile: "data/Galiano/Galiano-map.json",
199
+ outputFile: "Life.json.lz4",
200
+ featureFile: "",
201
+ regionField: null, obsIdField: null,
202
+ inputFiles: [],
203
+ skipRanks: [],
204
+ vizRanks: "@expand:hortis.filterVizRanks({that}.options.skipRanks)",
205
+ members: {
206
+ // Mutable tree to which each file contributes its taxa
207
+ tree: "@expand:hortis.rootNode({that}.map)",
208
+ map: "@expand:hortis.loadMapWithCounts({that}.options.mapFile)",
209
+ features: null, // "flatFeatures" file output, e.g. by deqgis, if configured
210
+ obs: null, // observations file which will be applied to features - expected to contain column named "regionColumn" supplied in options
211
+ // map of taxonId to taxon file
212
+ taxonCache: {}
213
+ },
214
+ invokers: {
215
+ applyRowsToTree: "hortis.applyRowsToTree({that}, {arguments}.1)", // rows
216
+ storeAtPath: "hortis.storeAtPath({that}, {arguments}.0, {arguments}.1)" // path, row
217
+ },
218
+ listeners: {
219
+ "onCreate.marmalise": "hortis.marmalise({that})"
220
+ },
221
+ components: {
222
+ taxonSource: {
223
+ type: "hortis.iNatTaxonSource"
224
+ }
225
+ }
226
+ });
227
+
228
+ hortis.filterDataset = function (dataset) {
229
+ return fluid.filterKeys(dataset, ["name", "colour"]);
230
+ };
231
+
232
+ hortis.indexRegions = function (treeBuilder) {
233
+ const features = treeBuilder.features;
234
+ const summaryById = {};
235
+ treeBuilder.summaryRows.forEach(function (row) {
236
+ summaryById[row.iNaturalistTaxonId] = row;
237
+ });
238
+ const communities = fluid.transform(features.communities, function (community) {
239
+ community.count = 0;
240
+ community.byTaxonId = {};
241
+ return community;
242
+ });
243
+ const classes = {};
244
+ fluid.each(features.classes, function (clazz, classKey) {
245
+ classes[classKey] = {
246
+ count: 0,
247
+ byTaxonId: {}
248
+ };
249
+ });
250
+ const applyObs = function (container, key, taxonId, obsId) {
251
+ if (key) {
252
+ const region = container[key];
253
+ if (!region) {
254
+ fluid.fail("Unknown region key ", key, " in container with keys ", Object.keys(container).join(", "));
255
+ }
256
+ let bucketTaxa = region.byTaxonId[taxonId]; // TODO: use fluid.pushArray?
257
+ if (!bucketTaxa) {
258
+ bucketTaxa = region.byTaxonId[taxonId] = [];
259
+ }
260
+ bucketTaxa.push(obsId);
261
+ ++region.count;
262
+ }
263
+ };
264
+ const options = treeBuilder.options;
265
+ treeBuilder.obs.rows.forEach(function (row) {
266
+ const obsId = row[options.obsIdField];
267
+ const summaryRow = summaryById[row.iNaturalistTaxonId];
268
+ if (summaryRow) {
269
+ const taxonId = summaryRow.id;
270
+ applyObs(communities, row.COMMUNITY, taxonId, obsId);
271
+ applyObs(classes, row.clazz, taxonId, obsId);
272
+
273
+ } else {
274
+ console.log("Warning: row with iNaturalistTaxonId " + row.iNaturalistTaxonId + " did not correspond to row in summary: ", row);
275
+ }
276
+ });
277
+ return {
278
+ communities: communities,
279
+ classes: fluid.extend(true, {}, features.classes, classes),
280
+ features: features.features
281
+ };
282
+ };
283
+
284
+ hortis.marmalise = async function (treeBuilder) {
285
+ const options = treeBuilder.options;
286
+ await hortis.asyncMap(options.inputFiles, async function (fileName) {
287
+ const result = await hortis.csvReaderWithMap({
288
+ inputFile: fluid.module.resolvePath(fileName),
289
+ mapColumns: treeBuilder.map.columns
290
+ }).completionPromise;
291
+ // console.log(JSON.stringify(result.rows[0], null, 2));
292
+ console.log("Applying " + result.rows.length + " rows from " + fileName);
293
+ treeBuilder.summaryRows = result.rows;
294
+ await treeBuilder.applyRowsToTree(fileName, result.rows);
295
+ });
296
+ if (options.featuresFile) {
297
+ fluid.expect("viz config file", options, ["obsFile", "obsMapFile", "regionField", "obsIdField"]);
298
+ treeBuilder.features = hortis.readModuleJSONSync(options.featuresFile);
299
+ const obsMap = hortis.readModuleJSONSync(options.obsMapFile);
300
+ const obs = await hortis.csvReaderWithMap({
301
+ inputFile: fluid.module.resolvePath(options.obsFile),
302
+ mapColumns: obsMap.columns
303
+ }).completionPromise;
304
+ console.log("Read " + obs.rows.length + " observation rows from " + options.obsFile);
305
+ treeBuilder.obs = obs;
306
+ }
307
+
308
+ hortis.flattenChildren(treeBuilder.tree);
309
+ const output = {
310
+ datasets: fluid.transform(treeBuilder.map.datasets, hortis.filterDataset),
311
+ tree: treeBuilder.tree
312
+ };
313
+ const extraOutput = options.featuresFile ? hortis.indexRegions(treeBuilder) : {};
314
+ const fullOutput = fluid.extend(output, extraOutput);
315
+ fs.writeFileSync("marmalised.json", JSON.stringify(fullOutput, null, 4) + "\n");
316
+ const text = JSON.stringify(fullOutput);
317
+ hortis.writeLZ4File(text, fluid.module.resolvePath(treeBuilder.options.outputFile));
318
+ };
319
+
320
+
321
+ const parsedArgs = minimist(process.argv.slice(2));
322
+
323
+ const options = parsedArgs.config ? hortis.readJSONSync(parsedArgs.config) : {
324
+ mapFile: parsedArgs.map,
325
+ obsFile: parsedArgs.obsFile,
326
+ skipRanks: [],
327
+ obsMapFile: parsedArgs.obsMapFile,
328
+ featuresFile: parsedArgs.featuresFile,
329
+ outputFile: parsedArgs.o || parsedArgs.output,
330
+ inputFiles: parsedArgs._
331
+ };
332
+
333
+ hortis.treeBuilder(options);
@@ -0,0 +1,68 @@
1
+ /* eslint-env node */
2
+ /* eslint dot-notation: "off"*/
3
+
4
+ "use strict";
5
+
6
+ var fluid = require("infusion");
7
+ var glob = require("glob");
8
+ var kettle = require("kettle");
9
+ var fs = require("fs");
10
+ fluid.require("%imerss-bioinfo");
11
+
12
+ var baseDir = fluid.module.resolvePath("%imerss-bioinfo/data/dataPaper-I-in/Animalia/");
13
+
14
+ fs.mkdirSync(baseDir, {
15
+ recursive: true
16
+ });
17
+
18
+ // Used during preparation of data paper to materialise an entire folder full of Google Sheets as CSV.
19
+ // Will need to updated to use Google Sheets API directrly and not rely on a Google Drive mapping
20
+
21
+ var dir = process.argv[2] || "e:/data/Google Drive/Galiano Data Paper 2021/Marine Life/Animalia";
22
+ var source = kettle.dataSource.URL({
23
+ url: "https://docs.google.com/spreadsheets/d/%id/export?format=csv&id=%id&gid=0",
24
+ port: 443,
25
+ termMap: {
26
+ id: "%id"
27
+ },
28
+ requestOptions: {
29
+ followAllRedirects: true
30
+ },
31
+ components: {
32
+ encoding: {
33
+ type: "fluid.dataSource.encoding.none"
34
+ }
35
+ }
36
+ });
37
+
38
+ var getSegment = function (path) {
39
+ var lastSlash = path.lastIndexOf("/");
40
+ var lastDot = path.lastIndexOf(".");
41
+ return path.substring(lastSlash + 1, lastDot);
42
+ };
43
+
44
+ fluid.setLogging(true);
45
+
46
+ glob(dir + "/*.gsheet", function (er, files) {
47
+ console.log("Got files ", files);
48
+ var records = files.map(function (oneFile) {
49
+ var promise = kettle.JSON.readFileSync(oneFile);
50
+ var parsed;
51
+ promise.then(function (result) {
52
+ parsed = result;
53
+ });
54
+ return {
55
+ id: parsed["doc_id"],
56
+ segment: getSegment(oneFile)
57
+ };
58
+ });
59
+ console.log("Got records ", records);
60
+ records.forEach(function (oneRecord) {
61
+ source.get({id: oneRecord.id}).then(function (doc) {
62
+ console.log("Got doc ", doc);
63
+ fs.writeFileSync(baseDir + oneRecord.segment + ".csv", doc);
64
+ }, function (err) {
65
+ console.log("Got err ", err);
66
+ });
67
+ });
68
+ });
@@ -0,0 +1,71 @@
1
+ /* eslint-env node */
2
+
3
+ "use strict";
4
+
5
+ var fluid = require("infusion");
6
+ var fs = require("fs");
7
+ var JSON5 = require("json5");
8
+
9
+ require("./dataProcessing/readCSV.js");
10
+ require("./dataProcessing/readCSVwithMap.js");
11
+ require("./utils/settleStructure.js");
12
+
13
+ fluid.setLogging(true);
14
+
15
+ var hortis = fluid.registerNamespace("hortis");
16
+
17
+ var taxonResolveMap = hortis.readJSONSync("data/TaxonResolution-map.json", "reading taxon resolution map");
18
+ var swaps = hortis.readJSONSync("data/taxon-swaps.json5", "reading taxon swaps file");
19
+
20
+ // Long disused script to output a taxon-swaps JSON5 file given a CSV taxon resolution file
21
+
22
+ hortis.copyCSVProps = function (props, target, source) {
23
+ props.forEach(function (prop) {
24
+ if (fluid.isValue(source[prop]) && source[prop] !== "") {
25
+ target[prop] = source[prop];
26
+ }
27
+ });
28
+ };
29
+
30
+ hortis.alphabetise = function (hash) {
31
+ var togo = {};
32
+ var keys = Object.keys(hash).sort();
33
+ keys.forEach(function (key) {
34
+ togo[key] = hash[key];
35
+ });
36
+ return togo;
37
+ };
38
+
39
+ hortis.updateSwaps = function (data) {
40
+ data.rows.forEach(function (row) {
41
+ var iNatName = row.iNaturalistTaxonName;
42
+ if (iNatName) {
43
+ var existing = swaps[iNatName];
44
+ if (!existing) {
45
+ existing = {
46
+ taxonNames: {}
47
+ };
48
+ swaps[iNatName] = existing;
49
+ }
50
+ existing.iNaturalistTaxonId = row.iNaturalistTaxonId;
51
+ var taxonNames = existing.taxonNames;
52
+
53
+ if (row.commonName) {
54
+ taxonNames[row.commonName] = {type: "commonName"};
55
+ }
56
+ var record = {};
57
+ hortis.copyCSVProps(["resolvedBy", "notes", "oldiNaturalistTaxonId"], record, row);
58
+ taxonNames[row.taxonName] = record;
59
+ }
60
+ });
61
+ var alphaSwaps = hortis.alphabetise(swaps);
62
+ fs.writeFileSync("updated-taxon-swaps.json5", JSON5.stringify(alphaSwaps, {
63
+ space: 4,
64
+ quote: "\""
65
+ }));
66
+ };
67
+
68
+ hortis.imerss.csvReader({
69
+ inputFile: process.argv[2] || "data/Valdes/Valdes Marine Dives Taxon Resolution.csv",
70
+ mapColumns: taxonResolveMap.columns
71
+ }).completionPromise.then(hortis.updateSwaps);