imerss-bioinfo 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +156 -0
- package/build.js +196 -0
- package/docs/css/imerss-viz-all.css +1227 -0
- package/docs/css/imerss-viz-wp-overrides.css +70 -0
- package/docs/css/pepiowelh.css +76 -0
- package/docs/css/xetthecum-external.css +47 -0
- package/docs/css/xetthecum-shared.css +80 -0
- package/docs/css/xetthecum.css +526 -0
- package/docs/js/imerss-viz-all.js +58 -0
- package/docs/js/imerss-viz-all.js.map +1 -0
- package/docs/js/jquery.js +10881 -0
- package/index.html +81 -0
- package/index.js +7 -0
- package/indexWithJustMap.html +92 -0
- package/indexWithMap.html +79 -0
- package/indexWithMapHoweSound.html +91 -0
- package/indexWithMapPepiowelh.html +119 -0
- package/indexWithMapValdes.html +90 -0
- package/indexWithMapXetthecum.html +90 -0
- package/package.json +50 -0
- package/src/WoRMS/taxonAPI.js +13 -0
- package/src/arphify.js +662 -0
- package/src/assignNames.js +103 -0
- package/src/auxBuild/restoreJQuery.js +1 -0
- package/src/buildSource/Comprehensive Lists.html +42 -0
- package/src/buildSource/Data Paper Part I Visualisation.html +50 -0
- package/src/buildSource/Galiano Life List.html +43 -0
- package/src/buildSource/Metchosin/index.html +22 -0
- package/src/buildSource/Pe'pi'ow'elh.html +72 -0
- package/src/buildSource/Squamish Life List.html +48 -0
- package/src/buildSource/Valdes Island Biodiversity.html +64 -0
- package/src/buildSource/Xetthecum.html +40 -0
- package/src/buildSource/dataPaperSunburstAndMap.html +30 -0
- package/src/buildSource/index.html +224 -0
- package/src/client/css/imerss-viz-wp-overrides.css +70 -0
- package/src/client/css/imerss-viz.css +531 -0
- package/src/client/css/pepiowelh.css +76 -0
- package/src/client/css/xetthecum-external.css +47 -0
- package/src/client/css/xetthecum-shared.css +80 -0
- package/src/client/css/xetthecum.css +526 -0
- package/src/client/html/imerss-viz-map-only.html +75 -0
- package/src/client/html/imerss-viz-map-pepi.html +77 -0
- package/src/client/html/imerss-viz-map-xetthecum.html +90 -0
- package/src/client/html/imerss-viz-map.html +75 -0
- package/src/client/html/imerss-viz.html +58 -0
- package/src/client/html/importing Xetthecum patterns.txt +10 -0
- package/src/client/html/xetthecum-patterns.html +27 -0
- package/src/client/img/Animalia_2.svg +1 -0
- package/src/client/img/Animalia_Annelida.svg +655 -0
- package/src/client/img/Animalia_Chaetognatha.svg +72 -0
- package/src/client/img/Animalia_Cnidaria.svg +173 -0
- package/src/client/img/Animalia_Ctenophora.svg +298 -0
- package/src/client/img/Animalia_Nemertea.svg +54 -0
- package/src/client/img/Animalia_Phoronida.svg +271 -0
- package/src/client/img/Animalia_Platyhelminthes.svg +58 -0
- package/src/client/img/Animalia_Porifera.svg +220 -0
- package/src/client/img/Animalia_Sipuncula.svg +155 -0
- package/src/client/img/Arachnoidiscus_schmidtii_web_1-2-scaled.webp +0 -0
- package/src/client/img/Arrow-down.svg +11 -0
- package/src/client/img/Arrow-up.svg +6 -0
- package/src/client/img/Chromista.svg +1 -0
- package/src/client/img/Pepiowelh/BlackNoWe2-wb-bb.png +0 -0
- package/src/client/img/Pepiowelh/BlackNoWe2-wb.png +0 -0
- package/src/client/img/Pepiowelh/BlackNoWe2.png +0 -0
- package/src/client/img/Pepiowelh/Ecocultural Mapping Banner.svg +1 -0
- package/src/client/img/Plankton.jpg +0 -0
- package/src/client/img/Plantae.svg +1 -0
- package/src/client/img/Plantae_2.svg +1 -0
- package/src/client/img/Protista.svg +1 -0
- package/src/client/img/Valdes/Valdes Island Conservancy.jpg +0 -0
- package/src/client/img/Xetthecum/Agricultural.jpg +0 -0
- package/src/client/img/Xetthecum/Anthropogenic_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Broad_Leaf.jpg +0 -0
- package/src/client/img/Xetthecum/Caves.jpg +0 -0
- package/src/client/img/Xetthecum/Coastal_Bluffs.jpg +0 -0
- package/src/client/img/Xetthecum/Conifer_Pole_Sapling.jpg +0 -0
- package/src/client/img/Xetthecum/Developed.jpg +0 -0
- package/src/client/img/Xetthecum/Dock.jpg +0 -0
- package/src/client/img/Xetthecum/Eelgrass.jpg +0 -0
- package/src/client/img/Xetthecum/Forests_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Freshwater_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Greig_Creek.jpg +0 -0
- package/src/client/img/Xetthecum/Intertidal.jpg +0 -0
- package/src/client/img/Xetthecum/Laughlin_Lake.jpg +0 -0
- package/src/client/img/Xetthecum/Marine_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Mature_Conifer_Forest.jpg +0 -0
- package/src/client/img/Xetthecum/Porlier_Pass_Road.jpg +0 -0
- package/src/client/img/Xetthecum/Riparian.jpg +0 -0
- package/src/client/img/Xetthecum/Rural.jpg +0 -0
- package/src/client/img/Xetthecum/Subtidal.jpg +0 -0
- package/src/client/img/Xetthecum/Wetland.jpg +0 -0
- package/src/client/img/Xetthecum/Woodland.jpg +0 -0
- package/src/client/img/Xetthecum/Woodland_banner.jpg +0 -0
- package/src/client/img/Xetthecum/Xetthecum.jpg +0 -0
- package/src/client/img/Xetthecum/Young_Conifer_Forest.jpg +0 -0
- package/src/client/img/Xetthecum/Young_Mixed_Forest.jpg +0 -0
- package/src/client/img/Xetthecum/food.svg +38 -0
- package/src/client/img/Xetthecum/indicator.svg +52 -0
- package/src/client/img/Xetthecum/material.svg +186 -0
- package/src/client/img/Xetthecum/medicinal.svg +62 -0
- package/src/client/img/Xetthecum/missing-value.svg +3 -0
- package/src/client/img/Xetthecum/spiritual.svg +13 -0
- package/src/client/img/Xetthecum/trade.svg +74 -0
- package/src/client/img/audio.svg +5 -0
- package/src/client/img/external-link.svg +3 -0
- package/src/client/img/selected-eye.svg +4 -0
- package/src/client/img/selected-tick.svg +3 -0
- package/src/client/img/thumbs/BCLeps.jpg +0 -0
- package/src/client/img/thumbs/Checklist.jpg +0 -0
- package/src/client/img/thumbs/Data-Paper-Figma.jpg +0 -0
- package/src/client/img/thumbs/Galiano-Data-Paper.jpg +0 -0
- package/src/client/img/thumbs/Galiano.jpg +0 -0
- package/src/client/img/thumbs/Maxwell.jpg +0 -0
- package/src/client/img/thumbs/Squamish.jpg +0 -0
- package/src/client/img/thumbs/Valdes.jpg +0 -0
- package/src/client/img/thumbs/Xetthecum.jpg +0 -0
- package/src/client/img/up-arrow.svg +4 -0
- package/src/client/js/autocomplete.js +53 -0
- package/src/client/js/checklist.js +179 -0
- package/src/client/js/colour.js +139 -0
- package/src/client/js/datasetControls.js +212 -0
- package/src/client/js/imerss-viz.js +1397 -0
- package/src/client/js/leafletMap.js +357 -0
- package/src/client/js/leafletMapWithGrid.js +337 -0
- package/src/client/js/leafletMapWithRegions.js +381 -0
- package/src/client/js/renderSVG.js +108 -0
- package/src/client/js/tabs.js +53 -0
- package/src/client/js/xetthecum.js +106 -0
- package/src/client/json/emptyPhyloMap.json +1 -0
- package/src/client/json/kingdomPhyloMap.json +17 -0
- package/src/client/json/phyloMap.json +58 -0
- package/src/compare.js +97 -0
- package/src/compilecsv.js +47 -0
- package/src/dataProcessing/LZ4.js +27 -0
- package/src/dataProcessing/coordinatePatch.js +151 -0
- package/src/dataProcessing/readCSV.js +107 -0
- package/src/dataProcessing/readCSVwithMap.js +90 -0
- package/src/dataProcessing/readCSVwithoutMap.js +25 -0
- package/src/dataProcessing/readJSON.js +23 -0
- package/src/dataProcessing/sqlite.js +174 -0
- package/src/dataProcessing/summarise.js +122 -0
- package/src/dataProcessing/taxonScrape.js +14 -0
- package/src/dataProcessing/worker.js +31 -0
- package/src/dataProcessing/writeCSV.js +34 -0
- package/src/dataProcessing/writeJSON.js +15 -0
- package/src/dbtaxa.js +96 -0
- package/src/deAuthorise.js +48 -0
- package/src/deqgis.js +148 -0
- package/src/expressions/expr.js +65 -0
- package/src/expressions/jsep.js +925 -0
- package/src/filterFirst.js +51 -0
- package/src/geom/geoJSON.js +121 -0
- package/src/iNaturalist/examples/api.inaturalist.org-v1-taxa!q=Acmispon americanus americanus.json +200 -0
- package/src/iNaturalist/examples/api.inaturalist.org-v1-taxa!q=Flabellina japonica.json +61 -0
- package/src/iNaturalist/examples/www.inaturalist.org-taxa-877359.json +132 -0
- package/src/iNaturalist/iNatUrls.js +17 -0
- package/src/iNaturalist/obsAPI.js +150 -0
- package/src/iNaturalist/taxonAPI.js +460 -0
- package/src/inatObs.js +120 -0
- package/src/inattify.js +123 -0
- package/src/intersect.js +80 -0
- package/src/intersectFilter.js +48 -0
- package/src/landlocked.js +137 -0
- package/src/lib/jquery.binarytransport.js +51 -0
- package/src/lib/lz4-commentary.txt +9 -0
- package/src/lib/lz4.js +73 -0
- package/src/lib/point-in-polygon.js +36 -0
- package/src/lz4.js +40 -0
- package/src/marmalise.js +333 -0
- package/src/materialise.js +68 -0
- package/src/reintegrate.js +71 -0
- package/src/taxonomise.js +615 -0
- package/src/unscrewGBIF.js +44 -0
- package/src/utils/dataSource.js +173 -0
- package/src/utils/settleStructure.js +62 -0
- package/src/utils/utils.js +92 -0
- package/src/viz/plethify.js +39 -0
- package/src/wormify.js +154 -0
- package/src/xetthecum/xetthecumFilters.js +126 -0
- package/test/testDataSource.js +182 -0
- package/test/testiNatObsSource.js +38 -0
- package/test/testiNatSource.js +50 -0
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/* eslint-env node */
|
|
2
|
+
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
const fluid = require("infusion");
|
|
6
|
+
const hortis = fluid.registerNamespace("hortis");
|
|
7
|
+
|
|
8
|
+
const sqlite = require("sqlite");
|
|
9
|
+
const sqlite3 = require("sqlite3");
|
|
10
|
+
const pako = require("pako");
|
|
11
|
+
|
|
12
|
+
fluid.registerNamespace("hortis.codecs.zlib");
|
|
13
|
+
|
|
14
|
+
hortis.codecs.zlib.encode = function (obj) {
|
|
15
|
+
const string = JSON.stringify(obj);
|
|
16
|
+
const togo = string === undefined ? undefined : pako.deflate(string);
|
|
17
|
+
return togo;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
hortis.codecs.zlib.decode = function (buffer) {
|
|
21
|
+
if (!buffer) {
|
|
22
|
+
return buffer;
|
|
23
|
+
} else {
|
|
24
|
+
const string = pako.inflate(buffer, {to: "string"});
|
|
25
|
+
return JSON.parse(string);
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
fluid.defaults("hortis.sqliteDB", {
|
|
30
|
+
gradeNames: ["fluid.modelComponent", "fluid.resourceLoader"],
|
|
31
|
+
// dbFile: String
|
|
32
|
+
model: {
|
|
33
|
+
db: "{that}.resources.db.parsed"
|
|
34
|
+
},
|
|
35
|
+
invokers: {
|
|
36
|
+
beginTransaction: "hortis.sqliteDB.beginTransaction({that}.model.db)",
|
|
37
|
+
endTransaction: "hortis.sqliteDB.endTransaction({that}.model.db)"
|
|
38
|
+
},
|
|
39
|
+
resources: {
|
|
40
|
+
db: {
|
|
41
|
+
promiseFunc: "hortis.sqliteDB.openDB",
|
|
42
|
+
promiseArgs: ["{that}.options.dbFile"]
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
hortis.sqliteDB.beginTransaction = function (db) {
|
|
48
|
+
return db.run("BEGIN");
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
hortis.sqliteDB.endTransaction = function (db) {
|
|
52
|
+
return db.run("COMMIT");
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
/** Opens an SQLite database held at the supplied module-relative path
|
|
56
|
+
* @param {String }dbFile - The (possibly module-qualified) filesystem path holding the database
|
|
57
|
+
* @return {Promise<Database>} A promise for an "sqlite" wrapper for the opened SQLite database
|
|
58
|
+
*/
|
|
59
|
+
hortis.sqliteDB.openDB = async function (dbFile) {
|
|
60
|
+
const path = fluid.module.resolvePath(dbFile);
|
|
61
|
+
try {
|
|
62
|
+
return sqlite.open({
|
|
63
|
+
filename: path,
|
|
64
|
+
driver: sqlite3.Database
|
|
65
|
+
});
|
|
66
|
+
} catch (err) {
|
|
67
|
+
console.log("Error opening database ", err);
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
fluid.defaults("hortis.sqliteSource", {
|
|
72
|
+
gradeNames: ["fluid.dataSource", "fluid.dataSource.noencoding", "fluid.dataSource.writable", "fluid.resourceLoader"],
|
|
73
|
+
// createString: String
|
|
74
|
+
// readQuery: Object
|
|
75
|
+
// writeQuery: Object
|
|
76
|
+
// columnCodecs: Object
|
|
77
|
+
model: {
|
|
78
|
+
db: "{that}.db.model.db",
|
|
79
|
+
tableCreated: "{that}.resources.table.parsed"
|
|
80
|
+
},
|
|
81
|
+
components: {
|
|
82
|
+
db: "{hortis.sqliteDB}"
|
|
83
|
+
},
|
|
84
|
+
resources: {
|
|
85
|
+
table: {
|
|
86
|
+
promiseFunc: "hortis.sqliteSource.createTable",
|
|
87
|
+
promiseArgs: ["{that}.db", "{that}.options.createString"]
|
|
88
|
+
}
|
|
89
|
+
},
|
|
90
|
+
listeners: {
|
|
91
|
+
"onRead.impl": {
|
|
92
|
+
func: "hortis.sqliteSource.read", // payload, options
|
|
93
|
+
args: ["{that}", "{that}.options.readQuery", "{arguments}.0", "{arguments}.1"] // can't be {that}.model.db because of FLUID-6752
|
|
94
|
+
},
|
|
95
|
+
"onWrite.impl": {
|
|
96
|
+
func: "hortis.sqliteSource.write", // payload
|
|
97
|
+
args: ["{that}", "{that}.options.writeQuery", "{arguments}.0"]
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
fluid.defaults("hortis.listableSqliteSource", {
|
|
103
|
+
gradeNames: ["hortis.sqliteSource", "fluid.dataSource.listable"],
|
|
104
|
+
// listQuery: Object
|
|
105
|
+
invokers: {
|
|
106
|
+
list: "hortis.sqliteSource.list({that}.model.db, {that}.options.listQuery)"
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
hortis.sqliteSource.createTable = function (dbThat, createString) {
|
|
111
|
+
console.log("Beginning to create table");
|
|
112
|
+
const togo = fluid.promise();
|
|
113
|
+
// Note that we can't expect the database component itself to construct until the entire tree constructs, we need
|
|
114
|
+
// to wait for the individual resource. Note also that the promise itself doesn't resolve to much useful.
|
|
115
|
+
// What do we expect about this in the future? That every "observation" of a model value is promisified? Or that it
|
|
116
|
+
// just has a reducer syntax.
|
|
117
|
+
dbThat.resources.db.promise.then(function () {
|
|
118
|
+
const db = dbThat.model.db;
|
|
119
|
+
console.log("Database opened");
|
|
120
|
+
db.run(createString).then(function () {
|
|
121
|
+
console.log("Table created");
|
|
122
|
+
togo.resolve(true);
|
|
123
|
+
}, function (err) {
|
|
124
|
+
togo.reject("Error creating table " + err);
|
|
125
|
+
});
|
|
126
|
+
}, function (err) {
|
|
127
|
+
togo.reject("Error waiting for database " + err);
|
|
128
|
+
});
|
|
129
|
+
return togo;
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
hortis.sqliteSource.prepareQueryArgs = function (args, directModel) {
|
|
133
|
+
return fluid.transform(args, function (arg) {
|
|
134
|
+
return typeof(arg) === "string" && arg.charAt(0) === "%" ?
|
|
135
|
+
fluid.get(directModel, arg.slice(1)) : arg;
|
|
136
|
+
});
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
hortis.deprefixColumn = function (column) {
|
|
140
|
+
return column.startsWith("$") ? column.substring(1) : column;
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
hortis.transcodeColumns = function (direction, columns, columnCodecs) {
|
|
144
|
+
return fluid.transform(columns, function (value, key) {
|
|
145
|
+
const column = hortis.deprefixColumn(key);
|
|
146
|
+
const codec = columnCodecs[column];
|
|
147
|
+
if (codec) {
|
|
148
|
+
const holder = hortis.codecs[codec];
|
|
149
|
+
if (!holder) {
|
|
150
|
+
fluid.fail("Unknown column codec ", codec, " allowable options are ", Object.keys(hortis.codecs).join(", "));
|
|
151
|
+
}
|
|
152
|
+
return holder[direction](value);
|
|
153
|
+
} else {
|
|
154
|
+
return value;
|
|
155
|
+
}
|
|
156
|
+
});
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
hortis.sqliteSource.read = async function (that, readQuery, value, options) {
|
|
160
|
+
const db = that.model.db; // for FLUID-6752
|
|
161
|
+
const result = await db.get(readQuery.query, hortis.sqliteSource.prepareQueryArgs(readQuery.args, options.directModel));
|
|
162
|
+
return hortis.transcodeColumns("decode", result, that.options.columnCodecs);
|
|
163
|
+
};
|
|
164
|
+
|
|
165
|
+
hortis.sqliteSource.write = function (that, writeQuery, value) {
|
|
166
|
+
const db = that.model.db;
|
|
167
|
+
const prepared = hortis.sqliteSource.prepareQueryArgs(writeQuery.args, value);
|
|
168
|
+
const encoded = hortis.transcodeColumns("encode", prepared, that.options.columnCodecs);
|
|
169
|
+
return db.run(writeQuery.query, encoded);
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
hortis.sqliteSource.list = function (db, writeQuery) {
|
|
173
|
+
return db.all(writeQuery.query);
|
|
174
|
+
};
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/* eslint-env node */
|
|
2
|
+
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
const fluid = require("infusion");
|
|
6
|
+
|
|
7
|
+
const hortis = fluid.registerNamespace("hortis");
|
|
8
|
+
|
|
9
|
+
fluid.defaults("hortis.summarise", {
|
|
10
|
+
gradeNames: "fluid.component",
|
|
11
|
+
summarise: true,
|
|
12
|
+
members: {
|
|
13
|
+
uniqueRows: {},
|
|
14
|
+
discardedRows: {}
|
|
15
|
+
},
|
|
16
|
+
summaryOutMapExtraColumns: { // TODO: In case we need an obs file with, e.g. Hulq names - propagate this through from fusion
|
|
17
|
+
},
|
|
18
|
+
fields: {
|
|
19
|
+
date: "dateObserved",
|
|
20
|
+
dateResolution: "dateObservedResolution",
|
|
21
|
+
unique: "iNaturalistTaxonName", // Field which identifies observations as being of the same taxon
|
|
22
|
+
obsCount: "observationCount",
|
|
23
|
+
obsId: "observationId",
|
|
24
|
+
coords: "coords"
|
|
25
|
+
},
|
|
26
|
+
invokers: {
|
|
27
|
+
storeRow: "hortis.summarise.storeRow({that}, {arguments}.0)"
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
// TODO: Wrong place for all of this processing - coordinates should be processed as part of the main
|
|
32
|
+
// parsing of the obs file
|
|
33
|
+
|
|
34
|
+
// Taken from https://stackoverflow.com/a/1140335 with several fixes
|
|
35
|
+
hortis.summarise.convertDMStoDD = function (degrees, minutes, seconds, direction) {
|
|
36
|
+
let dd = +degrees + minutes / 60 + seconds / (60 * 60);
|
|
37
|
+
|
|
38
|
+
if (direction === "S" || direction === "W") {
|
|
39
|
+
dd = -1 * Math.abs(dd);
|
|
40
|
+
}
|
|
41
|
+
return dd;
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
hortis.summarise.parseDMS = function (string) {
|
|
45
|
+
const parts = string.split(/[^\d\w]+/);
|
|
46
|
+
return hortis.summarise.convertDMStoDD(parts[0], parts[1], parts[2], parts[3]);
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
hortis.summarise.parseCoordinate = function (value) {
|
|
50
|
+
if (typeof(value) === "string" && value.match(/[NSEW]/)) {
|
|
51
|
+
return hortis.summarise.parseDMS(value);
|
|
52
|
+
} else {
|
|
53
|
+
return hortis.parseFloat(value);
|
|
54
|
+
}
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
hortis.summarise.parseCoordinates = function (row) {
|
|
58
|
+
const round = function (coord) {
|
|
59
|
+
return +hortis.roundDecimals(coord, 6);
|
|
60
|
+
};
|
|
61
|
+
const latitude = hortis.summarise.parseCoordinate(row.latitude);
|
|
62
|
+
const longitude = hortis.summarise.parseCoordinate(row.longitude);
|
|
63
|
+
return !isNaN(latitude) && !isNaN(longitude) ? [round(latitude), round(longitude)] : null;
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
hortis.summarise.copyObsFields = function (target, source, prefix, fields) {
|
|
67
|
+
fields.forEach(function (field) {
|
|
68
|
+
const targetField = prefix + hortis.capitalize(field);
|
|
69
|
+
target[targetField] = source[field];
|
|
70
|
+
});
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
hortis.summarise.updateRowRange = function (that, summaryRow, obsRow) {
|
|
74
|
+
const obsCountField = that.options.fields.obsCount;
|
|
75
|
+
if (!summaryRow) {
|
|
76
|
+
const commonKeys = Object.keys(hortis.summariseCommonOutMap.columns);
|
|
77
|
+
const extraKeys = Object.keys(that.options.summaryOutMapExtraColumns);
|
|
78
|
+
summaryRow = fluid.filterKeys(obsRow, commonKeys.concat(extraKeys));
|
|
79
|
+
summaryRow.firstTimestamp = Infinity;
|
|
80
|
+
summaryRow.lastTimestamp = -Infinity;
|
|
81
|
+
summaryRow[obsCountField] = 1;
|
|
82
|
+
} else {
|
|
83
|
+
summaryRow[obsCountField]++;
|
|
84
|
+
}
|
|
85
|
+
const fields = Object.keys(hortis.obsToSummaryFields);
|
|
86
|
+
if (obsRow.timestamp < summaryRow.firstTimestamp) {
|
|
87
|
+
hortis.summarise.copyObsFields(summaryRow, obsRow, "first", fields);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
if (obsRow.timestamp > summaryRow.lastTimestamp) {
|
|
91
|
+
hortis.summarise.copyObsFields(summaryRow, obsRow, "last", fields);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
return summaryRow;
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
hortis.summarise.storeRow = function (that, row) {
|
|
98
|
+
const fields = that.options.fields;
|
|
99
|
+
const coordsField = fields.coords;
|
|
100
|
+
row.timestamp = Date.parse(row[fields.date]);
|
|
101
|
+
const uniqueVal = row[fields.unique];
|
|
102
|
+
const existing = that.uniqueRows[uniqueVal];
|
|
103
|
+
|
|
104
|
+
const summaryRow = hortis.summarise.updateRowRange(that, existing, row);
|
|
105
|
+
|
|
106
|
+
that.uniqueRows[uniqueVal] = summaryRow;
|
|
107
|
+
|
|
108
|
+
if (that.options.summarise) {
|
|
109
|
+
const obsId = row[fields.obsId];
|
|
110
|
+
if (obsId === undefined) {
|
|
111
|
+
fluid.fail("Unable to find unique observation field for row ", row);
|
|
112
|
+
}
|
|
113
|
+
// TODO: Wrong place for this logic - these should be parsed as part of main filtering workflow, and before
|
|
114
|
+
// hortis.roundCoordinates
|
|
115
|
+
const coords = hortis.summarise.parseCoordinates(row);
|
|
116
|
+
fluid.set(summaryRow, [coordsField, obsId], coords);
|
|
117
|
+
if (coords) {
|
|
118
|
+
row.latitude = coords[0];
|
|
119
|
+
row.longitude = coords[1];
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/* eslint-env node */
|
|
2
|
+
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
var fluid = require("infusion");
|
|
6
|
+
var fs = require("fs");
|
|
7
|
+
|
|
8
|
+
var hortis = fluid.registerNamespace("hortis");
|
|
9
|
+
|
|
10
|
+
hortis.writeTaxonDoc = function (filename, doc) {
|
|
11
|
+
var formatted = JSON.stringify(doc, null, 4);
|
|
12
|
+
fs.writeFileSync(filename, formatted);
|
|
13
|
+
console.log("Written " + formatted.length + " bytes to " + filename);
|
|
14
|
+
};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/* eslint-env node */
|
|
2
|
+
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
var fluid = require("infusion");
|
|
6
|
+
|
|
7
|
+
var hortis = fluid.registerNamespace("hortis");
|
|
8
|
+
|
|
9
|
+
// Work in progress factoring common engine from wormify.js and inattify.js
|
|
10
|
+
|
|
11
|
+
fluid.defaults("hortis.worker", {
|
|
12
|
+
gradeNames: "fluid.component",
|
|
13
|
+
members: {
|
|
14
|
+
queue: "{that}.options.queue",
|
|
15
|
+
fetched: [],
|
|
16
|
+
cache: {}
|
|
17
|
+
},
|
|
18
|
+
invokers: {
|
|
19
|
+
oneWork: "hortis.worker.oneWork({that})",
|
|
20
|
+
// Returns {name, filename}
|
|
21
|
+
namesFromItem: "fluid.notImplemented()"
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
hortis.worker.oneWork = function (that) {
|
|
26
|
+
if (that.queue.length) {
|
|
27
|
+
var head = that.queue.shift();
|
|
28
|
+
var names = that.namesFromItem(head);
|
|
29
|
+
that.oneWork(head, names);
|
|
30
|
+
}
|
|
31
|
+
};
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/* eslint-env node */
|
|
2
|
+
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
var fluid = require("infusion");
|
|
6
|
+
var csvWriterModule = require("csv-writer");
|
|
7
|
+
|
|
8
|
+
var hortis = fluid.registerNamespace("hortis");
|
|
9
|
+
|
|
10
|
+
require("./readCSVWithmap.js");
|
|
11
|
+
require("./readCSVWithoutMap.js");
|
|
12
|
+
require("../utils/utils.js");
|
|
13
|
+
|
|
14
|
+
hortis.writeCSV = function (fileName, headersOrMap, rows, completionPromise) {
|
|
15
|
+
var isHeaders = fluid.isArrayable(headersOrMap);
|
|
16
|
+
var headers = isHeaders ? headersOrMap : Object.values(headersOrMap);
|
|
17
|
+
var duplicates = hortis.findDuplicates(headers);
|
|
18
|
+
if (duplicates.length > 0) {
|
|
19
|
+
fluid.fail("Error in hortis.writeCSV: duplicate column names: ", duplicates.join(", "));
|
|
20
|
+
}
|
|
21
|
+
var csvWriter = csvWriterModule.createArrayCsvWriter({
|
|
22
|
+
header: headers,
|
|
23
|
+
path: fileName
|
|
24
|
+
});
|
|
25
|
+
var arrayRows = [];
|
|
26
|
+
fluid.each(rows, function (row) {
|
|
27
|
+
arrayRows.push(hortis[isHeaders ? "rowToArray" : "mapToArray"](row, headersOrMap));
|
|
28
|
+
});
|
|
29
|
+
var promise = csvWriter.writeRecords(arrayRows);
|
|
30
|
+
promise.then(function () {
|
|
31
|
+
fluid.log("Written " + arrayRows.length + " lines to CSV file ", fileName);
|
|
32
|
+
});
|
|
33
|
+
fluid.promise.follow(promise, completionPromise);
|
|
34
|
+
};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/* eslint-env node */
|
|
2
|
+
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
var fluid = require("infusion");
|
|
6
|
+
var fs = require("fs");
|
|
7
|
+
|
|
8
|
+
var hortis = fluid.registerNamespace("hortis");
|
|
9
|
+
|
|
10
|
+
hortis.writeJSONSync = function (inFilename, doc) {
|
|
11
|
+
var filename = fluid.module.resolvePath(inFilename);
|
|
12
|
+
var formatted = JSON.stringify(doc, null, 4) + "\n";
|
|
13
|
+
fs.writeFileSync(filename, formatted);
|
|
14
|
+
console.log("Written " + formatted.length + " bytes to " + filename);
|
|
15
|
+
};
|
package/src/dbtaxa.js
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/* eslint-env node */
|
|
2
|
+
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
const fluid = require("infusion");
|
|
6
|
+
const fs = require("fs");
|
|
7
|
+
|
|
8
|
+
require("./dataProcessing/readJSON.js");
|
|
9
|
+
require("./iNaturalist/taxonAPI.js");
|
|
10
|
+
|
|
11
|
+
const glob = require("glob");
|
|
12
|
+
|
|
13
|
+
const taxonAPIFileBase = "data/iNaturalist/taxonAPI";
|
|
14
|
+
|
|
15
|
+
const hortis = fluid.registerNamespace("hortis");
|
|
16
|
+
|
|
17
|
+
const taxonAPIs = hortis.iNatTaxonAPI.dbTaxonAPIs();
|
|
18
|
+
taxonAPIs.events.onCreate.then((function (taxonAPIs) {
|
|
19
|
+
return traverseDB(taxonAPIs.byId, taxonAPIs.db);
|
|
20
|
+
}), function (err) {
|
|
21
|
+
console.log("Got error instantiating APIs ", err);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
const now = Date.now();
|
|
25
|
+
|
|
26
|
+
const dataFiles = glob.sync(taxonAPIFileBase + "/*.json");
|
|
27
|
+
|
|
28
|
+
console.log("Found " + dataFiles.length + " taxon files in " + (Date.now() - now) + "ms");
|
|
29
|
+
|
|
30
|
+
hortis.idFromFilename = function (filename) {
|
|
31
|
+
const lastslash = filename.lastIndexOf("/");
|
|
32
|
+
const lastdot = filename.lastIndexOf(".");
|
|
33
|
+
return filename.substring(lastslash + 1, lastdot);
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
async function traverseDB(source, db) {
|
|
37
|
+
|
|
38
|
+
console.log("Beginning insert");
|
|
39
|
+
const beforeWrite = Date.now();
|
|
40
|
+
|
|
41
|
+
const writeAll = async function (dataFiles) {
|
|
42
|
+
await db.beginTransaction();
|
|
43
|
+
for (let i = 0; i < dataFiles.length; ++i) {
|
|
44
|
+
const dataFile = dataFiles[i];
|
|
45
|
+
const contents = hortis.readJSONSync(dataFile);
|
|
46
|
+
const stats = fs.statSync(dataFile);
|
|
47
|
+
const fetched_at = stats.mtime.toISOString();
|
|
48
|
+
await source.set({id: contents.id}, {
|
|
49
|
+
id: contents.id,
|
|
50
|
+
fetched_at: fetched_at,
|
|
51
|
+
doc: contents
|
|
52
|
+
});
|
|
53
|
+
if (i % 1000 === 0) {
|
|
54
|
+
process.stdout.write(i + " ... ");
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
await db.endTransaction();
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
try {
|
|
61
|
+
await writeAll(dataFiles);
|
|
62
|
+
} catch (err) {
|
|
63
|
+
console.log("Error writing files ", err);
|
|
64
|
+
throw err;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
console.log("\nWritten " + dataFiles.length + " taxon files in " + (Date.now() - beforeWrite) + "ms");
|
|
68
|
+
|
|
69
|
+
const allIds = await source.list();
|
|
70
|
+
|
|
71
|
+
console.log("Got " + allIds.length + " ids: ", allIds[0]);
|
|
72
|
+
|
|
73
|
+
const beforeRead = Date.now();
|
|
74
|
+
|
|
75
|
+
const readAll = async function () {
|
|
76
|
+
let index = 0;
|
|
77
|
+
let first = true;
|
|
78
|
+
for (const {id: key} of allIds) {
|
|
79
|
+
|
|
80
|
+
if (index % 100 === 0) {
|
|
81
|
+
process.stdout.write("\"" + key + "\" ");
|
|
82
|
+
}
|
|
83
|
+
const row = await source.get({id: key});
|
|
84
|
+
const doc = row.doc;
|
|
85
|
+
if (first) {
|
|
86
|
+
console.log("First doc: ", doc);
|
|
87
|
+
first = false;
|
|
88
|
+
}
|
|
89
|
+
++index;
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
await readAll();
|
|
94
|
+
console.log("\nRead " + allIds.length + " taxon files in " + (Date.now() - beforeRead) + "ms");
|
|
95
|
+
|
|
96
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/* eslint-env node */
|
|
2
|
+
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
const fluid = require("infusion");
|
|
6
|
+
const minimist = require("minimist");
|
|
7
|
+
fluid.require("%imerss-bioinfo");
|
|
8
|
+
|
|
9
|
+
require("./dataProcessing/readCSV.js");
|
|
10
|
+
require("./dataProcessing/readCSVwithMap.js");
|
|
11
|
+
require("./dataProcessing/writeCSV.js");
|
|
12
|
+
|
|
13
|
+
const hortis = fluid.registerNamespace("hortis");
|
|
14
|
+
|
|
15
|
+
fluid.setLogging(true);
|
|
16
|
+
|
|
17
|
+
// old one "%imerss-bioinfo/data/Comprehensive Lists/sji_master_flora.csv"); used Full Species
|
|
18
|
+
|
|
19
|
+
const parsedArgs = minimist(process.argv.slice(2));
|
|
20
|
+
|
|
21
|
+
const outputFile = parsedArgs.o || "deauthorized.csv";
|
|
22
|
+
const inputFile = parsedArgs._[0] || fluid.module.resolvePath("%imerss-bioinfo/data/Squamish/GBIF_2022_Plantae_DwC.csv");
|
|
23
|
+
|
|
24
|
+
const reader = hortis.csvReaderWithoutMap({
|
|
25
|
+
inputFile: inputFile
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
// One-off script to parse out authorities from files such as those received from Dunwiddie
|
|
29
|
+
|
|
30
|
+
reader.completionPromise.then(async function () {
|
|
31
|
+
const mapped = [];
|
|
32
|
+
for (let i = 0; i < reader.rows.length; ++i) {
|
|
33
|
+
console.log("Processing row ", i);
|
|
34
|
+
const row = reader.rows[i];
|
|
35
|
+
const taxon = row["scientificName"];
|
|
36
|
+
const parsed = taxon.split(" ");
|
|
37
|
+
row.taxonName = parsed[0] + " " + parsed[1];
|
|
38
|
+
const infra = row["Infra taxa"] || row["infraspecificEpithet"];
|
|
39
|
+
if (infra) {
|
|
40
|
+
row.infraTaxonName = row.taxonName + " " + infra;
|
|
41
|
+
} else {
|
|
42
|
+
row.infraTaxonName = "";
|
|
43
|
+
}
|
|
44
|
+
row.authority = parsed.slice(2).join(" ");
|
|
45
|
+
mapped.push(row);
|
|
46
|
+
}
|
|
47
|
+
hortis.writeCSV(fluid.module.resolvePath(outputFile), Object.keys(mapped[0]), mapped, fluid.promise());
|
|
48
|
+
});
|
package/src/deqgis.js
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/* eslint-env node */
|
|
2
|
+
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
const fluid = require("infusion");
|
|
6
|
+
const fs = require("fs");
|
|
7
|
+
const minimist = require("minimist");
|
|
8
|
+
const glob = require("glob");
|
|
9
|
+
|
|
10
|
+
fluid.require("%imerss-bioinfo");
|
|
11
|
+
|
|
12
|
+
require("./dataProcessing/readJSON.js");
|
|
13
|
+
require("./dataProcessing/writeJSON.js");
|
|
14
|
+
require("./dataProcessing/readCSV.js");
|
|
15
|
+
require("./dataProcessing/readCSVWithoutMap.js");
|
|
16
|
+
|
|
17
|
+
const hortis = fluid.registerNamespace("hortis");
|
|
18
|
+
|
|
19
|
+
fluid.setLogging(true);
|
|
20
|
+
|
|
21
|
+
const parsedArgs = minimist(process.argv.slice(2));
|
|
22
|
+
|
|
23
|
+
const inputFile = parsedArgs._[0] || "%imerss-bioinfo/data/Xetthecum/deqgis.json5";
|
|
24
|
+
const outputFile = fluid.module.resolvePath(parsedArgs.o || "%imerss-bioinfo/data/Xetthecum/flatFeatures.json");
|
|
25
|
+
|
|
26
|
+
const config = hortis.readJSONSync(fluid.module.resolvePath(inputFile));
|
|
27
|
+
|
|
28
|
+
const baseDir = fluid.module.resolvePath(config.baseDirectory);
|
|
29
|
+
|
|
30
|
+
const dataFiles = glob.sync(baseDir + "/*.js");
|
|
31
|
+
|
|
32
|
+
console.log("Got data files ", dataFiles);
|
|
33
|
+
|
|
34
|
+
hortis.seColumns = ["Tagline", "What", "Where", "Importance", "Protection", "Source"];
|
|
35
|
+
|
|
36
|
+
hortis.applySensitiveEcosystems = function (togo, rows) {
|
|
37
|
+
rows.forEach(function (row, index) {
|
|
38
|
+
const labels = row.LABELS.split(",").map(s => s.trim());
|
|
39
|
+
labels.forEach(function (clazz) {
|
|
40
|
+
if (togo.classes[clazz]) {
|
|
41
|
+
hortis.seColumns.forEach(function (column) {
|
|
42
|
+
const cell = row[column];
|
|
43
|
+
if (cell) {
|
|
44
|
+
togo.classes[clazz]["sE-" + column] = cell;
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
console.log("Warnings: Sensitive ecosystems column " + column + " did not match row " + index, row);
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
} else if (clazz) {
|
|
51
|
+
console.log("Warning: Sensitive ecosystems column " + clazz + " did not match any from QGIS");
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
hortis.applyCulturalValues = function (togo, rows) {
|
|
58
|
+
rows.forEach(function (row) {
|
|
59
|
+
const communityName = row.COMMUNITIES;
|
|
60
|
+
if (togo.communities[communityName]) {
|
|
61
|
+
const community = togo.communities[communityName];
|
|
62
|
+
community.culturalValues = row["CULTURAL VALUES"];
|
|
63
|
+
community.culturalValuesSources = row.SOURCES;
|
|
64
|
+
} else {
|
|
65
|
+
console.log("Warning: Cultural values column " + communityName + " did not match any from QGIS");
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
const features = [];
|
|
71
|
+
|
|
72
|
+
hortis.indexCommunities = function (communities) {
|
|
73
|
+
const classes = {};
|
|
74
|
+
fluid.each(communities, function (community, communityKey) {
|
|
75
|
+
fluid.each(community.classes, function (clazz, classKey) {
|
|
76
|
+
clazz.community = communityKey;
|
|
77
|
+
classes[classKey] = clazz;
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
return classes;
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
hortis.filterCommunities = function (communities) {
|
|
84
|
+
return fluid.transform(communities, function (community) {
|
|
85
|
+
return fluid.extend(fluid.censorKeys(community, ["classes"]), {
|
|
86
|
+
classes: fluid.transform(community.classes, function () {
|
|
87
|
+
return {};
|
|
88
|
+
})
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
const labels = {};
|
|
94
|
+
|
|
95
|
+
hortis.indexOneFeature = function (dataFileName, feature, features, classes) {
|
|
96
|
+
const classname = feature.properties.LEGEND_LAB || feature.properties.LABEL;
|
|
97
|
+
fluid.set(labels, [feature.properties.COMMUNITY, classname], true);
|
|
98
|
+
|
|
99
|
+
feature.properties.clazz = classname;
|
|
100
|
+
const clazz = classes[classname];
|
|
101
|
+
if (!clazz) {
|
|
102
|
+
console.log("Warning: unknown feature with name " + classname + " in file " + dataFileName);
|
|
103
|
+
} else {
|
|
104
|
+
features.push(feature);
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
const classes = hortis.indexCommunities(config.communities);
|
|
109
|
+
|
|
110
|
+
const CSVs = fluid.transform(config.CSVs, function (oneFile) {
|
|
111
|
+
return hortis.csvReaderWithoutMap({
|
|
112
|
+
inputFile: fluid.module.resolvePath(oneFile.path)
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
const promises = Object.values(fluid.getMembers(CSVs, "completionPromise"));
|
|
117
|
+
|
|
118
|
+
fluid.promise.sequence(promises).then(function () {
|
|
119
|
+
console.log("Loaded " + Object.keys(CSVs).length + " CSV files");
|
|
120
|
+
const allRows = fluid.getMembers(CSVs, "rows");
|
|
121
|
+
|
|
122
|
+
dataFiles.forEach(function (dataFileName) {
|
|
123
|
+
const string = fs.readFileSync(dataFileName, "utf8");
|
|
124
|
+
const firstp = string.indexOf("{");
|
|
125
|
+
const lastp = string.lastIndexOf("}");
|
|
126
|
+
const data = JSON.parse(string.substring(firstp, lastp + 1));
|
|
127
|
+
console.log("Processing qgis2web data with name " + data.name);
|
|
128
|
+
data.features.forEach(function (feature) {
|
|
129
|
+
hortis.indexOneFeature(dataFileName, feature, features, classes);
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
const togo = {
|
|
134
|
+
communities: hortis.filterCommunities(config.communities),
|
|
135
|
+
classes: classes,
|
|
136
|
+
features: features
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
if (allRows.sensitiveEcosystems) {
|
|
140
|
+
hortis.applySensitiveEcosystems(togo, allRows.sensitiveEcosystems);
|
|
141
|
+
}
|
|
142
|
+
if (allRows.culturalValues) {
|
|
143
|
+
hortis.applyCulturalValues(togo, allRows.culturalValues);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
console.log("Encountered feature hierarchy: ", JSON.stringify(labels, null, 2));
|
|
147
|
+
hortis.writeJSONSync(outputFile, togo);
|
|
148
|
+
});
|