imerss-bioinfo 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +156 -0
- package/build.js +196 -0
- package/docs/css/imerss-viz-all.css +1227 -0
- package/docs/css/imerss-viz-wp-overrides.css +70 -0
- package/docs/css/pepiowelh.css +76 -0
- package/docs/css/xetthecum-external.css +47 -0
- package/docs/css/xetthecum-shared.css +80 -0
- package/docs/css/xetthecum.css +526 -0
- package/docs/js/imerss-viz-all.js +58 -0
- package/docs/js/imerss-viz-all.js.map +1 -0
- package/docs/js/jquery.js +10881 -0
- package/index.html +81 -0
- package/index.js +7 -0
- package/indexWithJustMap.html +92 -0
- package/indexWithMap.html +79 -0
- package/indexWithMapHoweSound.html +91 -0
- package/indexWithMapPepiowelh.html +119 -0
- package/indexWithMapValdes.html +90 -0
- package/indexWithMapXetthecum.html +90 -0
- package/package.json +50 -0
- package/src/WoRMS/taxonAPI.js +13 -0
- package/src/arphify.js +662 -0
- package/src/assignNames.js +103 -0
- package/src/auxBuild/restoreJQuery.js +1 -0
- package/src/buildSource/Comprehensive Lists.html +42 -0
- package/src/buildSource/Data Paper Part I Visualisation.html +50 -0
- package/src/buildSource/Galiano Life List.html +43 -0
- package/src/buildSource/Metchosin/index.html +22 -0
- package/src/buildSource/Pe'pi'ow'elh.html +72 -0
- package/src/buildSource/Squamish Life List.html +48 -0
- package/src/buildSource/Valdes Island Biodiversity.html +64 -0
- package/src/buildSource/Xetthecum.html +40 -0
- package/src/buildSource/dataPaperSunburstAndMap.html +30 -0
- package/src/buildSource/index.html +224 -0
- package/src/client/css/imerss-viz-wp-overrides.css +70 -0
- package/src/client/css/imerss-viz.css +531 -0
- package/src/client/css/pepiowelh.css +76 -0
- package/src/client/css/xetthecum-external.css +47 -0
- package/src/client/css/xetthecum-shared.css +80 -0
- package/src/client/css/xetthecum.css +526 -0
- package/src/client/html/imerss-viz-map-only.html +75 -0
- package/src/client/html/imerss-viz-map-pepi.html +77 -0
- package/src/client/html/imerss-viz-map-xetthecum.html +90 -0
- package/src/client/html/imerss-viz-map.html +75 -0
- package/src/client/html/imerss-viz.html +58 -0
- package/src/client/html/importing Xetthecum patterns.txt +10 -0
- package/src/client/html/xetthecum-patterns.html +27 -0
- package/src/client/img/Animalia_2.svg +1 -0
- package/src/client/img/Animalia_Annelida.svg +655 -0
- package/src/client/img/Animalia_Chaetognatha.svg +72 -0
- package/src/client/img/Animalia_Cnidaria.svg +173 -0
- package/src/client/img/Animalia_Ctenophora.svg +298 -0
- package/src/client/img/Animalia_Nemertea.svg +54 -0
- package/src/client/img/Animalia_Phoronida.svg +271 -0
- package/src/client/img/Animalia_Platyhelminthes.svg +58 -0
- package/src/client/img/Animalia_Porifera.svg +220 -0
- package/src/client/img/Animalia_Sipuncula.svg +155 -0
- package/src/client/img/Arachnoidiscus_schmidtii_web_1-2-scaled.webp +0 -0
- package/src/client/img/Arrow-down.svg +11 -0
- package/src/client/img/Arrow-up.svg +6 -0
- package/src/client/img/Chromista.svg +1 -0
- package/src/client/img/Pepiowelh/BlackNoWe2-wb-bb.png +0 -0
- package/src/client/img/Pepiowelh/BlackNoWe2-wb.png +0 -0
- package/src/client/img/Pepiowelh/BlackNoWe2.png +0 -0
- package/src/client/img/Pepiowelh/Ecocultural Mapping Banner.svg +1 -0
- package/src/client/img/Plankton.jpg +0 -0
- package/src/client/img/Plantae.svg +1 -0
- package/src/client/img/Plantae_2.svg +1 -0
- package/src/client/img/Protista.svg +1 -0
- package/src/client/img/Valdes/Valdes Island Conservancy.jpg +0 -0
- package/src/client/img/Xetthecum/Agricultural.jpg +0 -0
- package/src/client/img/Xetthecum/Anthropogenic_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Broad_Leaf.jpg +0 -0
- package/src/client/img/Xetthecum/Caves.jpg +0 -0
- package/src/client/img/Xetthecum/Coastal_Bluffs.jpg +0 -0
- package/src/client/img/Xetthecum/Conifer_Pole_Sapling.jpg +0 -0
- package/src/client/img/Xetthecum/Developed.jpg +0 -0
- package/src/client/img/Xetthecum/Dock.jpg +0 -0
- package/src/client/img/Xetthecum/Eelgrass.jpg +0 -0
- package/src/client/img/Xetthecum/Forests_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Freshwater_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Greig_Creek.jpg +0 -0
- package/src/client/img/Xetthecum/Intertidal.jpg +0 -0
- package/src/client/img/Xetthecum/Laughlin_Lake.jpg +0 -0
- package/src/client/img/Xetthecum/Marine_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Mature_Conifer_Forest.jpg +0 -0
- package/src/client/img/Xetthecum/Porlier_Pass_Road.jpg +0 -0
- package/src/client/img/Xetthecum/Riparian.jpg +0 -0
- package/src/client/img/Xetthecum/Rural.jpg +0 -0
- package/src/client/img/Xetthecum/Subtidal.jpg +0 -0
- package/src/client/img/Xetthecum/Wetland.jpg +0 -0
- package/src/client/img/Xetthecum/Woodland.jpg +0 -0
- package/src/client/img/Xetthecum/Woodland_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Xetthecum.jpg +0 -0
- package/src/client/img/Xetthecum/Young_Conifer_Forest.jpg +0 -0
- package/src/client/img/Xetthecum/Young_Mixed_Forest.jpg +0 -0
- package/src/client/img/Xetthecum/food.svg +38 -0
- package/src/client/img/Xetthecum/indicator.svg +52 -0
- package/src/client/img/Xetthecum/material.svg +186 -0
- package/src/client/img/Xetthecum/medicinal.svg +62 -0
- package/src/client/img/Xetthecum/missing-value.svg +3 -0
- package/src/client/img/Xetthecum/spiritual.svg +13 -0
- package/src/client/img/Xetthecum/trade.svg +74 -0
- package/src/client/img/audio.svg +5 -0
- package/src/client/img/external-link.svg +3 -0
- package/src/client/img/selected-eye.svg +4 -0
- package/src/client/img/selected-tick.svg +3 -0
- package/src/client/img/thumbs/BCLeps.jpg +0 -0
- package/src/client/img/thumbs/Checklist.jpg +0 -0
- package/src/client/img/thumbs/Data-Paper-Figma.jpg +0 -0
- package/src/client/img/thumbs/Galiano-Data-Paper.jpg +0 -0
- package/src/client/img/thumbs/Galiano.jpg +0 -0
- package/src/client/img/thumbs/Maxwell.jpg +0 -0
- package/src/client/img/thumbs/Squamish.jpg +0 -0
- package/src/client/img/thumbs/Valdes.jpg +0 -0
- package/src/client/img/thumbs/Xetthecum.jpg +0 -0
- package/src/client/img/up-arrow.svg +4 -0
- package/src/client/js/autocomplete.js +53 -0
- package/src/client/js/checklist.js +179 -0
- package/src/client/js/colour.js +139 -0
- package/src/client/js/datasetControls.js +212 -0
- package/src/client/js/imerss-viz.js +1397 -0
- package/src/client/js/leafletMap.js +357 -0
- package/src/client/js/leafletMapWithGrid.js +337 -0
- package/src/client/js/leafletMapWithRegions.js +381 -0
- package/src/client/js/renderSVG.js +108 -0
- package/src/client/js/tabs.js +53 -0
- package/src/client/js/xetthecum.js +106 -0
- package/src/client/json/emptyPhyloMap.json +1 -0
- package/src/client/json/kingdomPhyloMap.json +17 -0
- package/src/client/json/phyloMap.json +58 -0
- package/src/compare.js +97 -0
- package/src/compilecsv.js +47 -0
- package/src/dataProcessing/LZ4.js +27 -0
- package/src/dataProcessing/coordinatePatch.js +151 -0
- package/src/dataProcessing/readCSV.js +107 -0
- package/src/dataProcessing/readCSVwithMap.js +90 -0
- package/src/dataProcessing/readCSVwithoutMap.js +25 -0
- package/src/dataProcessing/readJSON.js +23 -0
- package/src/dataProcessing/sqlite.js +174 -0
- package/src/dataProcessing/summarise.js +122 -0
- package/src/dataProcessing/taxonScrape.js +14 -0
- package/src/dataProcessing/worker.js +31 -0
- package/src/dataProcessing/writeCSV.js +34 -0
- package/src/dataProcessing/writeJSON.js +15 -0
- package/src/dbtaxa.js +96 -0
- package/src/deAuthorise.js +48 -0
- package/src/deqgis.js +148 -0
- package/src/expressions/expr.js +65 -0
- package/src/expressions/jsep.js +925 -0
- package/src/filterFirst.js +51 -0
- package/src/geom/geoJSON.js +121 -0
- package/src/iNaturalist/examples/api.inaturalist.org-v1-taxa!q=Acmispon americanus americanus.json +200 -0
- package/src/iNaturalist/examples/api.inaturalist.org-v1-taxa!q=Flabellina japonica.json +61 -0
- package/src/iNaturalist/examples/www.inaturalist.org-taxa-877359.json +132 -0
- package/src/iNaturalist/iNatUrls.js +17 -0
- package/src/iNaturalist/obsAPI.js +150 -0
- package/src/iNaturalist/taxonAPI.js +460 -0
- package/src/inatObs.js +120 -0
- package/src/inattify.js +123 -0
- package/src/intersect.js +80 -0
- package/src/intersectFilter.js +48 -0
- package/src/landlocked.js +137 -0
- package/src/lib/jquery.binarytransport.js +51 -0
- package/src/lib/lz4-commentary.txt +9 -0
- package/src/lib/lz4.js +73 -0
- package/src/lib/point-in-polygon.js +36 -0
- package/src/lz4.js +40 -0
- package/src/marmalise.js +333 -0
- package/src/materialise.js +68 -0
- package/src/reintegrate.js +71 -0
- package/src/taxonomise.js +615 -0
- package/src/unscrewGBIF.js +44 -0
- package/src/utils/dataSource.js +173 -0
- package/src/utils/settleStructure.js +62 -0
- package/src/utils/utils.js +92 -0
- package/src/viz/plethify.js +39 -0
- package/src/wormify.js +154 -0
- package/src/xetthecum/xetthecumFilters.js +126 -0
- package/test/testDataSource.js +182 -0
- package/test/testiNatObsSource.js +38 -0
- package/test/testiNatSource.js +50 -0
package/src/arphify.js
ADDED
|
@@ -0,0 +1,662 @@
|
|
|
1
|
+
/* eslint-env node */
|
|
2
|
+
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
var fluid = require("infusion");
|
|
6
|
+
var minimist = require("minimist");
|
|
7
|
+
var moment = require("moment-timezone");
|
|
8
|
+
var ExcelJS = require("exceljs");
|
|
9
|
+
var fs = require("fs");
|
|
10
|
+
|
|
11
|
+
fluid.require("%imerss-bioinfo");
|
|
12
|
+
|
|
13
|
+
require("./dataProcessing/readJSON.js");
|
|
14
|
+
require("./dataProcessing/readCSV.js");
|
|
15
|
+
require("./dataProcessing/readCSVwithMap.js");
|
|
16
|
+
require("./dataProcessing/readCSVwithoutMap.js");
|
|
17
|
+
require("./dataProcessing/writeCSV.js");
|
|
18
|
+
require("./expressions/expr.js");
|
|
19
|
+
require("./utils/utils.js");
|
|
20
|
+
|
|
21
|
+
var hortis = fluid.registerNamespace("hortis");
|
|
22
|
+
|
|
23
|
+
hortis.ranks = Object.freeze(require("../data/ranks.json"));
|
|
24
|
+
hortis.reverseRanks = Object.freeze(fluid.makeArray(hortis.ranks).reverse());
|
|
25
|
+
|
|
26
|
+
fluid.setLogging(true);
|
|
27
|
+
|
|
28
|
+
var parsedArgs = minimist(process.argv.slice(2));
|
|
29
|
+
|
|
30
|
+
require("./WoRMS/taxonAPI.js");
|
|
31
|
+
var WoRMSTaxonAPIFileBase = "data/WoRMS/taxonAPI";
|
|
32
|
+
|
|
33
|
+
// Yes, if only it actually were a pipeline!
|
|
34
|
+
var pipeline = hortis.readJSONSync(parsedArgs.pipeline || "data/dataPaper-I-in/arpha-out.json5");
|
|
35
|
+
|
|
36
|
+
var summaryMap = hortis.readJSONSync(fluid.module.resolvePath(pipeline.summaryFileMap), "reading summary map file");
|
|
37
|
+
|
|
38
|
+
var summaryReader = hortis.csvReaderWithMap({
|
|
39
|
+
inputFile: fluid.module.resolvePath(pipeline.summaryFile),
|
|
40
|
+
mapColumns: summaryMap.columns
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
var obsMap = hortis.readJSONSync(fluid.module.resolvePath(pipeline.obsFileMap), "reading obs map file");
|
|
44
|
+
|
|
45
|
+
var obsReader = hortis.csvReaderWithMap({
|
|
46
|
+
inputFile: fluid.module.resolvePath(pipeline.obsFile),
|
|
47
|
+
mapColumns: obsMap.columns
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
var patchReader = pipeline.patchFile ? hortis.csvReaderWithoutMap({
|
|
51
|
+
inputFile: fluid.module.resolvePath(pipeline.patchFile)
|
|
52
|
+
}) : {
|
|
53
|
+
completionPromise: fluid.promise().resolve([]),
|
|
54
|
+
rows: []
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
var outputDir = fluid.module.resolvePath(pipeline.outputDir);
|
|
58
|
+
fs.mkdirSync(outputDir, { recursive: true });
|
|
59
|
+
|
|
60
|
+
hortis.genusEx = /\((.*)\)/;
|
|
61
|
+
|
|
62
|
+
hortis.normalise = function (str) {
|
|
63
|
+
return str.replace(/\s+/g, " ").trim();
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
// Variant for summaries
|
|
67
|
+
hortis.extractGenus = function (name, outRow) {
|
|
68
|
+
var matches = hortis.genusEx.exec(name);
|
|
69
|
+
var togo;
|
|
70
|
+
if (matches) {
|
|
71
|
+
outRow.Subgenus = matches[1];
|
|
72
|
+
togo = name.replace(hortis.genusEx, "");
|
|
73
|
+
} else {
|
|
74
|
+
togo = name;
|
|
75
|
+
}
|
|
76
|
+
return hortis.normalise(togo);
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
// Variant for obs -> Darwin Core
|
|
80
|
+
hortis.axeFromName = ["sp.", "ssp.", "spp.", "complex", "sp.A", "sp.B", "B"];
|
|
81
|
+
|
|
82
|
+
hortis.qualForming = ["agg", "aff.", "s.lat.", "cf", "sp.nov.", "var."];
|
|
83
|
+
|
|
84
|
+
hortis.extractSubgenus = function (seg, outRow) {
|
|
85
|
+
if (seg.startsWith("(") && seg.endsWith(")")) {
|
|
86
|
+
outRow.subgenus = seg.substring(1, seg.length - 1);
|
|
87
|
+
return 1;
|
|
88
|
+
} else {
|
|
89
|
+
return 0;
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
hortis.findRank = function (row, ranks) {
|
|
94
|
+
return ranks.find(function (rank) {
|
|
95
|
+
return !!row[rank];
|
|
96
|
+
});
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
/*
|
|
100
|
+
* AS of 19/7/21
|
|
101
|
+
* so yes, in the 'qualifier' field I think the only relevant values from our dataset are: 'cf', 'species complex', 'sp.A', and 'sp.B'
|
|
102
|
+
* in the 'identificationRemarks' field we will add the critical note
|
|
103
|
+
* so let's discard 'sp.' and 'spp.' from the catalogue
|
|
104
|
+
* we will only use it in the curated summary / checklists
|
|
105
|
+
*/
|
|
106
|
+
|
|
107
|
+
hortis.extractQualifier = function (name, outRow) {
|
|
108
|
+
var words = name.split(" ");
|
|
109
|
+
// Unconditionally axe sp/spp/ssp from all words
|
|
110
|
+
var useWords = words.filter(function (word) {
|
|
111
|
+
return !hortis.axeFromName.includes(word);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
// Filter out even qualifier-forming terms so that we can populate broken-out fields and scientificName
|
|
115
|
+
var bareWords = useWords.filter(function (word) {
|
|
116
|
+
return !hortis.qualForming.includes(word);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
var sspPoint = 1 + hortis.extractSubgenus(bareWords[1] || "", outRow);
|
|
120
|
+
|
|
121
|
+
var lastBareWords = bareWords.slice(sspPoint);
|
|
122
|
+
var rank = hortis.findRank(outRow, hortis.reverseRanks);
|
|
123
|
+
|
|
124
|
+
if (rank === "genus" || rank === "species") {
|
|
125
|
+
outRow.genus = bareWords[0];
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if (lastBareWords.length === 0) {
|
|
129
|
+
outRow.taxonRank = rank;
|
|
130
|
+
} else if (lastBareWords.length === 1) {
|
|
131
|
+
outRow.specificEpithet = lastBareWords[0];
|
|
132
|
+
outRow.taxonRank = "species";
|
|
133
|
+
} else if (lastBareWords.length === 2) {
|
|
134
|
+
outRow.specificEpithet = lastBareWords[0];
|
|
135
|
+
outRow.infraspecificEpithet = lastBareWords[1];
|
|
136
|
+
outRow.taxonRank = "subspecies";
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
var qualPoint = useWords.findIndex(function (word) {
|
|
140
|
+
return hortis.qualForming.includes(word);
|
|
141
|
+
});
|
|
142
|
+
if (outRow.subgenus) {
|
|
143
|
+
lastBareWords.unshift("(" + outRow.subgenus + ")");
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
outRow.identificationQualifier = qualPoint === -1 ? "" : useWords.slice(qualPoint).join(" ");
|
|
147
|
+
outRow.scientificName = [outRow.genus, ...lastBareWords].join(" ");
|
|
148
|
+
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
// Reviewer recommendation 7
|
|
152
|
+
hortis.normaliseSex = function (outRow) {
|
|
153
|
+
var sexes = [];
|
|
154
|
+
var sex = outRow.sex;
|
|
155
|
+
if (sex.match(/(([^f][^e]|^)male)|M/)) {
|
|
156
|
+
sexes.push("male");
|
|
157
|
+
}
|
|
158
|
+
if (sex.match(/female|F/)) {
|
|
159
|
+
sexes.push("female");
|
|
160
|
+
}
|
|
161
|
+
var newSexes = sexes.join(", ");
|
|
162
|
+
if (sex !== newSexes) {
|
|
163
|
+
outRow.occurrenceRemarks = outRow.sex;
|
|
164
|
+
outRow.sex = newSexes;
|
|
165
|
+
}
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
// Variant for summaries
|
|
169
|
+
hortis.extractSsp = function (name, outRow) {
|
|
170
|
+
var words = name.split(" ");
|
|
171
|
+
|
|
172
|
+
if (words.length === 3) {
|
|
173
|
+
var maybeSsp = words[2];
|
|
174
|
+
if (maybeSsp.startsWith("complex")
|
|
175
|
+
|| maybeSsp.startsWith("agg")
|
|
176
|
+
|| maybeSsp.startsWith("s.lat.")
|
|
177
|
+
|| words[1].startsWith("cf")) {
|
|
178
|
+
outRow.Species = words[1] + " " + maybeSsp;
|
|
179
|
+
} else {
|
|
180
|
+
outRow.Species = words[1];
|
|
181
|
+
outRow.Subspecies = maybeSsp;
|
|
182
|
+
}
|
|
183
|
+
} else if (words.length === 2) {
|
|
184
|
+
outRow.Species = words[1];
|
|
185
|
+
} else {
|
|
186
|
+
fluid.fail("Unexpected species name " + name);
|
|
187
|
+
}
|
|
188
|
+
};
|
|
189
|
+
|
|
190
|
+
// Also in coordinatePatch.js, leafletMap.js
|
|
191
|
+
hortis.datasetIdFromObs = function (obsId) {
|
|
192
|
+
var colpos = obsId.indexOf(":");
|
|
193
|
+
return obsId.substring(0, colpos);
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
hortis.badDates = {};
|
|
197
|
+
|
|
198
|
+
hortis.formatDate = function (row, template) {
|
|
199
|
+
var togo = "";
|
|
200
|
+
var format = template.substring("!Date:".length);
|
|
201
|
+
// TODO: This should actually be applied in the data loader
|
|
202
|
+
var momentVal = moment.tz(row.dateObserved, "Canada/Pacific");
|
|
203
|
+
if (momentVal.isValid()) {
|
|
204
|
+
// RBCM records claim to have a time but they don't
|
|
205
|
+
var noTime = !row.dateObserved.includes("T") || row.dateObserved.includes("T00:00:00");
|
|
206
|
+
togo = noTime && format.includes("H") ? "" : momentVal.format(format);
|
|
207
|
+
} else {
|
|
208
|
+
var obsId = row.observationId;
|
|
209
|
+
if (!hortis.badDates[obsId]) {
|
|
210
|
+
console.log("WARNING: row " + row.observationId + " has invalid date " + row.dateObserved);
|
|
211
|
+
}
|
|
212
|
+
hortis.badDates[obsId] = true;
|
|
213
|
+
}
|
|
214
|
+
return togo;
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
// TODO: Of course we need to pipeline the whole of ARPHA export
|
|
218
|
+
hortis.quantiseDepth = function (outRow, places) {
|
|
219
|
+
var depth = outRow.verbatimDepth;
|
|
220
|
+
if (depth === "o-86") {
|
|
221
|
+
outRow.verbatimDepth = "0-86";
|
|
222
|
+
outRow.minimumDepthInMeters = "0";
|
|
223
|
+
outRow.maximumDepthInMeters = "86";
|
|
224
|
+
} else {
|
|
225
|
+
var togo = hortis.roundDecimals(depth, places);
|
|
226
|
+
if (togo && isNaN(togo)) {
|
|
227
|
+
console.log("WARNING: row " + outRow.occurrenceID + " has invalid depth " + depth);
|
|
228
|
+
}
|
|
229
|
+
outRow.verbatimDepth = togo;
|
|
230
|
+
}
|
|
231
|
+
};
|
|
232
|
+
|
|
233
|
+
// Table of bad "count" entries from PMLS data
|
|
234
|
+
hortis.badCountTable = {
|
|
235
|
+
"1 patc": "1 patch",
|
|
236
|
+
"1tiny": "1 tiny",
|
|
237
|
+
"2 (pair0": "2",
|
|
238
|
+
"2 larg": "2 large",
|
|
239
|
+
"?": "",
|
|
240
|
+
"`": "",
|
|
241
|
+
"i": "",
|
|
242
|
+
"-": "",
|
|
243
|
+
"snall": "",
|
|
244
|
+
"present": "",
|
|
245
|
+
"NA": "",
|
|
246
|
+
"white and cream": ""
|
|
247
|
+
};
|
|
248
|
+
|
|
249
|
+
// Technical reviewer recommendation 4
|
|
250
|
+
hortis.mapIndividualCount = function (outRow) {
|
|
251
|
+
var count = outRow.individualCount;
|
|
252
|
+
var lookup = hortis.badCountTable[count];
|
|
253
|
+
var mapped = lookup === undefined ? count : lookup;
|
|
254
|
+
if (mapped && !hortis.isInteger(mapped)) {
|
|
255
|
+
outRow.occurrenceRemarks = "Count: " + mapped;
|
|
256
|
+
outRow.individualCount = "";
|
|
257
|
+
} else {
|
|
258
|
+
outRow.individualCount = mapped;
|
|
259
|
+
}
|
|
260
|
+
};
|
|
261
|
+
|
|
262
|
+
hortis.countDecimals = function (value) {
|
|
263
|
+
var string = value.toString();
|
|
264
|
+
var dotpos = string.indexOf(".");
|
|
265
|
+
return dotpos === -1 ? dotpos : string.length - 1 - dotpos;
|
|
266
|
+
};
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
hortis.correctUncertainty = function (outRow) {
|
|
270
|
+
// 2nd round reviewer recommendation 9
|
|
271
|
+
var uncertainty = outRow.coordinateUncertaintyInMeters;
|
|
272
|
+
var val = hortis.parseFloat(uncertainty);
|
|
273
|
+
if (isNaN(val) || val < 1) {
|
|
274
|
+
outRow.coordinateUncertaintyInMeters = "";
|
|
275
|
+
}
|
|
276
|
+
// 3rd round reviewer recommendation 3
|
|
277
|
+
if (!outRow.coordinateUncertaintyInMeters) {
|
|
278
|
+
var decimals = Math.min(hortis.countDecimals(outRow.decimalLatitude), hortis.countDecimals(outRow.decimalLongitude));
|
|
279
|
+
if (decimals === -1) {
|
|
280
|
+
console.log("ERROR: unable to find coordinate resolution for row without coordinate uncertainty ", outRow);
|
|
281
|
+
} else { // 2 -> 4, 3 -> 3, 4 -> 2, 5-> 2
|
|
282
|
+
var scale = Math.min(Math.max(6 - decimals, 2), 4);
|
|
283
|
+
outRow.coordinateUncertaintyInMeters = Math.pow(10, scale);
|
|
284
|
+
var comment = "coordinate uncertainty inferred from coordinate resolution";
|
|
285
|
+
outRow.georeferenceRemarks = outRow.georeferenceRemarks ? outRow.georeferenceRemarks + "; " + comment : comment;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
};
|
|
289
|
+
|
|
290
|
+
// 2nd round reviewer recommendation 10
|
|
291
|
+
hortis.correctLocalityTable = {
|
|
292
|
+
"e96b99b5-e978-49b0-a151-b5e5eb0f0981": "Trincomali Channel",
|
|
293
|
+
"29392e13-0c6b-4ea0-9c3d-bec39ae428a3": "Trincomali Channel",
|
|
294
|
+
"dfead0da-98f0-4ed1-b6c2-d270339704ee": "Trincomali Channel",
|
|
295
|
+
"f8ebf88e-6e0a-4a6e-a888-5f837d5326b6": "Trincomali Channel",
|
|
296
|
+
"0c5923fe-91e6-423e-bedf-1d46a0b89273": "Trincomali Channel"
|
|
297
|
+
};
|
|
298
|
+
|
|
299
|
+
hortis.correctLocality = function (outRow) {
|
|
300
|
+
var lookup = hortis.correctLocalityTable[outRow.occurrenceID];
|
|
301
|
+
if (lookup !== undefined) {
|
|
302
|
+
outRow.locality = lookup;
|
|
303
|
+
}
|
|
304
|
+
};
|
|
305
|
+
|
|
306
|
+
// 2nd round reviewer recommendation 11
|
|
307
|
+
hortis.cleanIdRemarks = function (outRow) {
|
|
308
|
+
outRow.identificationRemarks = outRow.identificationRemarks.replace(/\"/g, "");
|
|
309
|
+
};
|
|
310
|
+
|
|
311
|
+
hortis.normaliseRecorders = function (recordedBy) {
|
|
312
|
+
// Technical reviewer recommendation 5
|
|
313
|
+
var separators = recordedBy.replace("; ", " | ").trim();
|
|
314
|
+
// Technical reviewer recommendation 6
|
|
315
|
+
var togo = separators === "anonymous" ? "" : separators;
|
|
316
|
+
return togo;
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
// 2nd round reviewer recommendation 13
|
|
320
|
+
hortis.badEventRemarksTable = {
|
|
321
|
+
"/": "",
|
|
322
|
+
"?": "",
|
|
323
|
+
"??": "",
|
|
324
|
+
"#NAME?": ""
|
|
325
|
+
};
|
|
326
|
+
|
|
327
|
+
hortis.cleanEventRemarks = function (outRow) {
|
|
328
|
+
var lookup = hortis.badEventRemarksTable[outRow.eventRemarks];
|
|
329
|
+
if (lookup !== undefined) {
|
|
330
|
+
outRow.eventRemarks = lookup;
|
|
331
|
+
}
|
|
332
|
+
};
|
|
333
|
+
|
|
334
|
+
/* Accepts a row of a summary, and the column structure inside "Taxa" */
|
|
335
|
+
|
|
336
|
+
hortis.mapTaxaRows = function (rows, columns, materialsMap) {
|
|
337
|
+
return fluid.transform(rows, function (row) {
|
|
338
|
+
var summaryRow = materialsMap.summaryIndex[row.iNaturalistTaxonId];
|
|
339
|
+
var togo = {};
|
|
340
|
+
fluid.each(columns, function (template, target) {
|
|
341
|
+
togo[target] = hortis.stringTemplate(template, row);
|
|
342
|
+
});
|
|
343
|
+
if (row.species !== "" || row.genus !== "") {
|
|
344
|
+
var degenified = hortis.extractGenus(row.taxonName, togo);
|
|
345
|
+
hortis.extractSsp(degenified, togo);
|
|
346
|
+
}
|
|
347
|
+
if (summaryRow) {
|
|
348
|
+
// Account for observations which have been hacked by assigning them to proxy iNat taxa - in theory we should
|
|
349
|
+
// copy more fields but we have managed to assign these to, e.g. proxies in the same family
|
|
350
|
+
var proxySummary = fluid.copy(summaryRow);
|
|
351
|
+
// Replicate this piece of workflow from mapMaterialsRows
|
|
352
|
+
hortis.extractQualifier(summaryRow.taxonName, proxySummary);
|
|
353
|
+
togo.Genus = proxySummary.genus;
|
|
354
|
+
}
|
|
355
|
+
return togo;
|
|
356
|
+
});
|
|
357
|
+
};
|
|
358
|
+
|
|
359
|
+
hortis.stashMismatchedRow = function (mismatches, patchIndex, obsRow, summaryRow) {
|
|
360
|
+
var key = obsRow.previousIdentifications + "|" + obsRow.scientificName;
|
|
361
|
+
var patchRow = patchIndex[key];
|
|
362
|
+
if (patchRow) {
|
|
363
|
+
if (patchRow.Disposition === "P") {
|
|
364
|
+
obsRow.scientificName = obsRow.previousIdentifications;
|
|
365
|
+
} else if (patchRow.Disposition === "S") {
|
|
366
|
+
var desp = obsRow.previousIdentifications.replace(" sp.", "");
|
|
367
|
+
obsRow.scientificName = desp + " sp.";
|
|
368
|
+
} else if (patchRow.Disposition === "X") {
|
|
369
|
+
console.log("!!! Unexpected use of patch with key " + key);
|
|
370
|
+
}
|
|
371
|
+
} else {
|
|
372
|
+
var existing = mismatches[key];
|
|
373
|
+
if (!existing) {
|
|
374
|
+
mismatches[key] = fluid.extend({
|
|
375
|
+
previousIdentifications: obsRow.previousIdentifications
|
|
376
|
+
}, summaryRow);
|
|
377
|
+
console.log("Stashing mismatched row ", mismatches[key]);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
};
|
|
381
|
+
|
|
382
|
+
hortis.mapMaterialsRows = function (rows, patchIndex, materialsMap, references, columns) {
|
|
383
|
+
return fluid.transform(rows, function (row) {
|
|
384
|
+
var togo = {};
|
|
385
|
+
var dataset = hortis.datasetIdFromObs(row.observationId);
|
|
386
|
+
var summaryRow = materialsMap.summaryIndex[row.iNaturalistTaxonId];
|
|
387
|
+
var termMap = fluid.extend({}, row, {
|
|
388
|
+
summary: summaryRow
|
|
389
|
+
});
|
|
390
|
+
// row.scientificName = summaryRow ? summaryRow.taxonName : "";
|
|
391
|
+
var refBlock = references[dataset];
|
|
392
|
+
fluid.each(columns, function (template, target) {
|
|
393
|
+
var outVal = "";
|
|
394
|
+
if (refBlock && refBlock[target]) {
|
|
395
|
+
outVal = refBlock[target];
|
|
396
|
+
}
|
|
397
|
+
if (template.startsWith("!references.")) {
|
|
398
|
+
var ref = template.substring("!references.".length);
|
|
399
|
+
outVal = refBlock && refBlock[ref] || "";
|
|
400
|
+
} else if (template.startsWith("!Date:")) {
|
|
401
|
+
outVal = hortis.formatDate(row, template);
|
|
402
|
+
} else {
|
|
403
|
+
outVal = hortis.stringTemplate(template, termMap) || outVal;
|
|
404
|
+
}
|
|
405
|
+
if (outVal === "Confidence: ") { // blatant special casing
|
|
406
|
+
outVal = "";
|
|
407
|
+
}
|
|
408
|
+
togo[target] = outVal;
|
|
409
|
+
});
|
|
410
|
+
if (!togo.occurrenceID) {
|
|
411
|
+
togo.occurrenceID = "imerss.org:" + row.observationId;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
if (row.coordinatesCorrected === "yes") {
|
|
415
|
+
togo.georeferencedBy = "Andrew Simon";
|
|
416
|
+
togo.georeferenceProtocol = "interpretation of locality, and/or inference based on local knowledge and species ecology";
|
|
417
|
+
togo.georeferenceVerificationStatus = "corrected";
|
|
418
|
+
togo.georeferenceRemarks = row.coordinatesCorrectedNote;
|
|
419
|
+
}
|
|
420
|
+
// Note that previousIdentifications is taken from the row's own "taxonName" field from the original obs
|
|
421
|
+
if (togo.scientificName !== togo.previousIdentifications) {
|
|
422
|
+
hortis.stashMismatchedRow(materialsMap.mismatches, patchIndex, togo, summaryRow);
|
|
423
|
+
}
|
|
424
|
+
hortis.extractQualifier(togo.scientificName, togo);
|
|
425
|
+
hortis.normaliseSex(togo);
|
|
426
|
+
if (summaryRow && summaryRow.criticalNotes) {
|
|
427
|
+
togo.identificationRemarks = summaryRow.criticalNotes;
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
// var filename = hortis.WoRMSTaxa.filenameFromTaxonName(WoRMSTaxonAPIFileBase, row.iNaturalistTaxonName);
|
|
431
|
+
// var wormsRec = hortis.readJSONSync(filename);
|
|
432
|
+
// togo.taxonID = "WoRMS:" + wormsRec.AphiaID;
|
|
433
|
+
togo.taxonID = "iNaturalist:" + row.iNaturalistTaxonId;
|
|
434
|
+
|
|
435
|
+
togo.recordedBy = hortis.normaliseRecorders(togo.recordedBy);
|
|
436
|
+
togo.georeferencedBy = hortis.normaliseRecorders(togo.georeferencedBy);
|
|
437
|
+
|
|
438
|
+
hortis.quantiseDepth(togo, 2);
|
|
439
|
+
hortis.mapIndividualCount(togo);
|
|
440
|
+
hortis.correctUncertainty(togo);
|
|
441
|
+
hortis.correctLocality(togo);
|
|
442
|
+
hortis.cleanIdRemarks(togo);
|
|
443
|
+
hortis.cleanEventRemarks(togo);
|
|
444
|
+
return togo;
|
|
445
|
+
});
|
|
446
|
+
};
|
|
447
|
+
|
|
448
|
+
hortis.writeSheet = function (workbook, sheetName, rows) {
|
|
449
|
+
var sheet = workbook.addWorksheet(sheetName);
|
|
450
|
+
var keys = Object.keys(rows[0]);
|
|
451
|
+
var header = sheet.getRow(1);
|
|
452
|
+
keys.forEach(function (key, index) {
|
|
453
|
+
header.getCell(index + 1).value = key;
|
|
454
|
+
});
|
|
455
|
+
rows.forEach(function (row, rowIndex) {
|
|
456
|
+
var sheetRow = sheet.getRow(rowIndex + 2);
|
|
457
|
+
keys.forEach(function (key, index) {
|
|
458
|
+
sheetRow.getCell(index + 1).value = row[key];
|
|
459
|
+
});
|
|
460
|
+
});
|
|
461
|
+
};
|
|
462
|
+
|
|
463
|
+
hortis.writeExcel = function (sheets, key, outputDir) {
|
|
464
|
+
if (sheets.Taxa.length === 0) {
|
|
465
|
+
console.log("Skipping key " + key + " since no rows were selected");
|
|
466
|
+
return fluid.promise().resolve();
|
|
467
|
+
}
|
|
468
|
+
var workbook = new ExcelJS.Workbook();
|
|
469
|
+
|
|
470
|
+
fluid.each(sheets, function (sheet, sheetName) {
|
|
471
|
+
hortis.writeSheet(workbook, sheetName, sheet);
|
|
472
|
+
});
|
|
473
|
+
|
|
474
|
+
var filename = outputDir + "/" + key + ".xlsx";
|
|
475
|
+
var togo = workbook.xlsx.writeFile(filename);
|
|
476
|
+
togo.then(function () {
|
|
477
|
+
var stats = fs.statSync(filename);
|
|
478
|
+
console.log("Written " + stats.size + " bytes to " + filename);
|
|
479
|
+
});
|
|
480
|
+
return togo;
|
|
481
|
+
};
|
|
482
|
+
|
|
483
|
+
hortis.indexSummary = function (summaryRows) {
|
|
484
|
+
var togo = {};
|
|
485
|
+
summaryRows.forEach(function (row) {
|
|
486
|
+
togo[row.iNaturalistTaxonId] = row;
|
|
487
|
+
if (row.subtaxonAuthority) { // All downstream steps expect only a single authority
|
|
488
|
+
row.authority = row.subtaxonAuthority;
|
|
489
|
+
}
|
|
490
|
+
});
|
|
491
|
+
return togo;
|
|
492
|
+
};
|
|
493
|
+
|
|
494
|
+
hortis.indexPatchRows = function (patchRows) {
|
|
495
|
+
var togo = {};
|
|
496
|
+
patchRows.forEach(function (row) {
|
|
497
|
+
togo[row.previousIdentifications + "|" + row.taxonName] = row;
|
|
498
|
+
});
|
|
499
|
+
return togo;
|
|
500
|
+
};
|
|
501
|
+
|
|
502
|
+
// TODO: Worry if obs and summaries diverge in taxonomy
|
|
503
|
+
hortis.filterArphaRows = function (rows, rec, rowCount) {
|
|
504
|
+
return rows.filter(function (row, index) {
|
|
505
|
+
var parsed = hortis.expr.parse(rec.filter);
|
|
506
|
+
var match = hortis.expr.evaluate(parsed, row);
|
|
507
|
+
if (match) {
|
|
508
|
+
++rowCount[index];
|
|
509
|
+
}
|
|
510
|
+
return match;
|
|
511
|
+
});
|
|
512
|
+
};
|
|
513
|
+
|
|
514
|
+
// Note - argument modified
|
|
515
|
+
hortis.sortRows = function (rows, sortBy) {
|
|
516
|
+
var comparator = function (ra, rb) {
|
|
517
|
+
return fluid.find(sortBy, function (column) {
|
|
518
|
+
return ra[column] > rb[column] ? 1 : (ra[column] < rb[column] ? -1 : undefined);
|
|
519
|
+
});
|
|
520
|
+
};
|
|
521
|
+
rows.sort(comparator);
|
|
522
|
+
};
|
|
523
|
+
|
|
524
|
+
hortis.verifyCounts = function (name, rowCount, rows) {
|
|
525
|
+
rowCount.forEach(function (count, index) {
|
|
526
|
+
if (count !== 1) {
|
|
527
|
+
console.log("Anomalous " + name + " count for row " + index + ": " + count);
|
|
528
|
+
console.log("Row contents: ", rows[index]);
|
|
529
|
+
}
|
|
530
|
+
});
|
|
531
|
+
};
|
|
532
|
+
|
|
533
|
+
hortis.eliminateEmptyColumns = function (rows) {
|
|
534
|
+
var hasValue = {};
|
|
535
|
+
rows.forEach(function (row) {
|
|
536
|
+
fluid.each(row, function (value, key) {
|
|
537
|
+
if (fluid.isValue(value) && value !== "") {
|
|
538
|
+
hasValue[key] = true;
|
|
539
|
+
}
|
|
540
|
+
});
|
|
541
|
+
});
|
|
542
|
+
var valueKeys = Object.keys(hasValue);
|
|
543
|
+
var togo = fluid.transform(rows, function (row) {
|
|
544
|
+
return fluid.filterKeys(row, valueKeys);
|
|
545
|
+
});
|
|
546
|
+
return togo;
|
|
547
|
+
};
|
|
548
|
+
|
|
549
|
+
hortis.checkDuplicates = function (rows, eliminate) {
|
|
550
|
+
var columns = ["occurrenceID"];
|
|
551
|
+
var byDataset = {};
|
|
552
|
+
|
|
553
|
+
columns.forEach(function (column) {
|
|
554
|
+
console.log("Checking near duplicates by omitting column " + column);
|
|
555
|
+
var buckets = {};
|
|
556
|
+
var emittedKeys = {};
|
|
557
|
+
var emitDuplicate = function (row) {
|
|
558
|
+
var key = row[column];
|
|
559
|
+
if (!emittedKeys[key]) {
|
|
560
|
+
fluid.pushArray(byDataset, row.institutionCode, row);
|
|
561
|
+
emittedKeys[key] = true;
|
|
562
|
+
}
|
|
563
|
+
};
|
|
564
|
+
var outRows = [];
|
|
565
|
+
var duplicates = 0;
|
|
566
|
+
rows.forEach(function (row) {
|
|
567
|
+
var filtered = fluid.censorKeys(row, [column]);
|
|
568
|
+
var omitted = row[column] || true;
|
|
569
|
+
var hash = Object.values(filtered).join("|");
|
|
570
|
+
var oldValue = buckets[hash];
|
|
571
|
+
if (oldValue) {
|
|
572
|
+
emitDuplicate(oldValue);
|
|
573
|
+
emitDuplicate(row);
|
|
574
|
+
fluid.log("ERROR: duplicate row with contents " + hash + " and key " + omitted);
|
|
575
|
+
++duplicates;
|
|
576
|
+
} else {
|
|
577
|
+
buckets[hash] = row;
|
|
578
|
+
outRows.push(row);
|
|
579
|
+
}
|
|
580
|
+
});
|
|
581
|
+
if (duplicates) {
|
|
582
|
+
console.log("Found " + duplicates + " duplicates");
|
|
583
|
+
}
|
|
584
|
+
if (eliminate && rows.length !== outRows.length) {
|
|
585
|
+
console.log("After filtering duplicates, " + rows.length + " rows reduced to " + outRows.length);
|
|
586
|
+
rows = outRows;
|
|
587
|
+
}
|
|
588
|
+
});
|
|
589
|
+
fluid.each(byDataset.iNaturalist, function (row) {
|
|
590
|
+
var id = row.occurrenceID;
|
|
591
|
+
var iNatObs = id.substring(id.lastIndexOf(":") + 1);
|
|
592
|
+
row.observationUrl = "https://www.inaturalist.org/observations/" + iNatObs;
|
|
593
|
+
});
|
|
594
|
+
fluid.each(byDataset, function (oneDataset, key) {
|
|
595
|
+
var outfile = "materials-duplicates-" + key + ".csv";
|
|
596
|
+
hortis.writeCSV(outfile, Object.keys(oneDataset[0]), oneDataset, fluid.promise());
|
|
597
|
+
});
|
|
598
|
+
return rows;
|
|
599
|
+
};
|
|
600
|
+
|
|
601
|
+
var completion = fluid.promise.sequence([summaryReader.completionPromise, obsReader.completionPromise, patchReader.completionPromise]);
|
|
602
|
+
|
|
603
|
+
completion.then(function () {
|
|
604
|
+
var summaryRows = summaryReader.rows;
|
|
605
|
+
console.log("Summary Input: " + summaryRows.length + " rows");
|
|
606
|
+
var summaryRowCount = fluid.generate(summaryRows.length, 0);
|
|
607
|
+
var obsRows = obsReader.rows;
|
|
608
|
+
console.log("Obs Input: " + obsRows.length + " rows");
|
|
609
|
+
var obsRowCount = fluid.generate(obsRows.length, 0);
|
|
610
|
+
var summaryIndex = hortis.indexSummary(summaryRows); // also overrides authority with subtaxonAuthority
|
|
611
|
+
var patchRows = patchReader.rows;
|
|
612
|
+
console.log("Patch Input: " + patchRows.length + " rows");
|
|
613
|
+
var patchIndex = hortis.indexPatchRows(patchRows);
|
|
614
|
+
var materialsMap = {
|
|
615
|
+
summaryIndex: summaryIndex,
|
|
616
|
+
mismatches: {}
|
|
617
|
+
};
|
|
618
|
+
var now = Date.now();
|
|
619
|
+
var allMaterials = [];
|
|
620
|
+
var outs = fluid.transform(pipeline.files, function (rec, key) {
|
|
621
|
+
var outSummaryRows = hortis.filterArphaRows(summaryRows, rec, summaryRowCount);
|
|
622
|
+
console.log("Extracted " + outSummaryRows.length + " summary rows via filter " + key);
|
|
623
|
+
|
|
624
|
+
var Taxa = pipeline.sheets.Taxa;
|
|
625
|
+
var taxaRows = hortis.mapTaxaRows(outSummaryRows, Taxa.columns, materialsMap);
|
|
626
|
+
hortis.sortRows(taxaRows, Taxa.sortBy);
|
|
627
|
+
|
|
628
|
+
var outObsRows = hortis.filterArphaRows(obsRows, rec, obsRowCount);
|
|
629
|
+
console.log("Extracted " + outObsRows.length + " obs rows via filter " + key);
|
|
630
|
+
|
|
631
|
+
var materialsRows = hortis.mapMaterialsRows(outObsRows, patchIndex, materialsMap, pipeline.references, pipeline.sheets.Materials.columns);
|
|
632
|
+
|
|
633
|
+
// This is the export file destined for GBIF
|
|
634
|
+
allMaterials = allMaterials.concat(materialsRows);
|
|
635
|
+
|
|
636
|
+
return {
|
|
637
|
+
Taxa: taxaRows,
|
|
638
|
+
Materials: materialsRows,
|
|
639
|
+
ExternalLinks: [fluid.copy(pipeline.sheets.ExternalLinks.columns)]
|
|
640
|
+
};
|
|
641
|
+
});
|
|
642
|
+
console.log("Total extracted obs rows: " + fluid.flatten(fluid.getMembers(outs, "Materials")).length);
|
|
643
|
+
console.log("Filtered obs in " + (Date.now() - now) + " ms");
|
|
644
|
+
hortis.verifyCounts("summary", summaryRowCount, summaryRows);
|
|
645
|
+
hortis.verifyCounts("obs", obsRowCount, obsRows);
|
|
646
|
+
var mismatches = Object.values(materialsMap.mismatches);
|
|
647
|
+
if (mismatches.length > 0) {
|
|
648
|
+
console.log("Writing " + mismatches.length + " mismatched rows to arphaMismatches.csv");
|
|
649
|
+
hortis.writeCSV("arphaMismatches.csv", ["previousIdentifications", "taxonName"].concat(Object.keys(fluid.censorKeys(summaryRows[0], ["taxonName"]))), mismatches, fluid.promise());
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
hortis.sortRows(allMaterials, pipeline.sheets.Materials.sortBy);
|
|
653
|
+
var filteredMaterials = hortis.eliminateEmptyColumns(allMaterials);
|
|
654
|
+
filteredMaterials = hortis.checkDuplicates(filteredMaterials, true);
|
|
655
|
+
hortis.writeCSV(outputDir + "/Materials.csv", Object.keys(filteredMaterials[0]), filteredMaterials, fluid.promise());
|
|
656
|
+
|
|
657
|
+
fluid.each(outs, function (sheets, key) {
|
|
658
|
+
hortis.writeExcel(sheets, key, outputDir);
|
|
659
|
+
});
|
|
660
|
+
}, function (err) {
|
|
661
|
+
console.log("Error ", err);
|
|
662
|
+
});
|