imerss-bioinfo 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/README.md +156 -0
  2. package/build.js +196 -0
  3. package/docs/css/imerss-viz-all.css +1227 -0
  4. package/docs/css/imerss-viz-wp-overrides.css +70 -0
  5. package/docs/css/pepiowelh.css +76 -0
  6. package/docs/css/xetthecum-external.css +47 -0
  7. package/docs/css/xetthecum-shared.css +80 -0
  8. package/docs/css/xetthecum.css +526 -0
  9. package/docs/js/imerss-viz-all.js +58 -0
  10. package/docs/js/imerss-viz-all.js.map +1 -0
  11. package/docs/js/jquery.js +10881 -0
  12. package/index.html +81 -0
  13. package/index.js +7 -0
  14. package/indexWithJustMap.html +92 -0
  15. package/indexWithMap.html +79 -0
  16. package/indexWithMapHoweSound.html +91 -0
  17. package/indexWithMapPepiowelh.html +119 -0
  18. package/indexWithMapValdes.html +90 -0
  19. package/indexWithMapXetthecum.html +90 -0
  20. package/package.json +50 -0
  21. package/src/WoRMS/taxonAPI.js +13 -0
  22. package/src/arphify.js +662 -0
  23. package/src/assignNames.js +103 -0
  24. package/src/auxBuild/restoreJQuery.js +1 -0
  25. package/src/buildSource/Comprehensive Lists.html +42 -0
  26. package/src/buildSource/Data Paper Part I Visualisation.html +50 -0
  27. package/src/buildSource/Galiano Life List.html +43 -0
  28. package/src/buildSource/Metchosin/index.html +22 -0
  29. package/src/buildSource/Pe'pi'ow'elh.html +72 -0
  30. package/src/buildSource/Squamish Life List.html +48 -0
  31. package/src/buildSource/Valdes Island Biodiversity.html +64 -0
  32. package/src/buildSource/Xetthecum.html +40 -0
  33. package/src/buildSource/dataPaperSunburstAndMap.html +30 -0
  34. package/src/buildSource/index.html +224 -0
  35. package/src/client/css/imerss-viz-wp-overrides.css +70 -0
  36. package/src/client/css/imerss-viz.css +531 -0
  37. package/src/client/css/pepiowelh.css +76 -0
  38. package/src/client/css/xetthecum-external.css +47 -0
  39. package/src/client/css/xetthecum-shared.css +80 -0
  40. package/src/client/css/xetthecum.css +526 -0
  41. package/src/client/html/imerss-viz-map-only.html +75 -0
  42. package/src/client/html/imerss-viz-map-pepi.html +77 -0
  43. package/src/client/html/imerss-viz-map-xetthecum.html +90 -0
  44. package/src/client/html/imerss-viz-map.html +75 -0
  45. package/src/client/html/imerss-viz.html +58 -0
  46. package/src/client/html/importing Xetthecum patterns.txt +10 -0
  47. package/src/client/html/xetthecum-patterns.html +27 -0
  48. package/src/client/img/Animalia_2.svg +1 -0
  49. package/src/client/img/Animalia_Annelida.svg +655 -0
  50. package/src/client/img/Animalia_Chaetognatha.svg +72 -0
  51. package/src/client/img/Animalia_Cnidaria.svg +173 -0
  52. package/src/client/img/Animalia_Ctenophora.svg +298 -0
  53. package/src/client/img/Animalia_Nemertea.svg +54 -0
  54. package/src/client/img/Animalia_Phoronida.svg +271 -0
  55. package/src/client/img/Animalia_Platyhelminthes.svg +58 -0
  56. package/src/client/img/Animalia_Porifera.svg +220 -0
  57. package/src/client/img/Animalia_Sipuncula.svg +155 -0
  58. package/src/client/img/Arachnoidiscus_schmidtii_web_1-2-scaled.webp +0 -0
  59. package/src/client/img/Arrow-down.svg +11 -0
  60. package/src/client/img/Arrow-up.svg +6 -0
  61. package/src/client/img/Chromista.svg +1 -0
  62. package/src/client/img/Pepiowelh/BlackNoWe2-wb-bb.png +0 -0
  63. package/src/client/img/Pepiowelh/BlackNoWe2-wb.png +0 -0
  64. package/src/client/img/Pepiowelh/BlackNoWe2.png +0 -0
  65. package/src/client/img/Pepiowelh/Ecocultural Mapping Banner.svg +1 -0
  66. package/src/client/img/Plankton.jpg +0 -0
  67. package/src/client/img/Plantae.svg +1 -0
  68. package/src/client/img/Plantae_2.svg +1 -0
  69. package/src/client/img/Protista.svg +1 -0
  70. package/src/client/img/Valdes/Valdes Island Conservancy.jpg +0 -0
  71. package/src/client/img/Xetthecum/Agricultural.jpg +0 -0
  72. package/src/client/img/Xetthecum/Anthropogenic_banner.jpg +0 -0
  73. package/src/client/img/Xetthecum/Broad_Leaf.jpg +0 -0
  74. package/src/client/img/Xetthecum/Caves.jpg +0 -0
  75. package/src/client/img/Xetthecum/Coastal_Bluffs.jpg +0 -0
  76. package/src/client/img/Xetthecum/Conifer_Pole_Sapling.jpg +0 -0
  77. package/src/client/img/Xetthecum/Developed.jpg +0 -0
  78. package/src/client/img/Xetthecum/Dock.jpg +0 -0
  79. package/src/client/img/Xetthecum/Eelgrass.jpg +0 -0
  80. package/src/client/img/Xetthecum/Forests_banner.jpg +0 -0
  81. package/src/client/img/Xetthecum/Freshwater_banner.jpg +0 -0
  82. package/src/client/img/Xetthecum/Greig_Creek.jpg +0 -0
  83. package/src/client/img/Xetthecum/Intertidal.jpg +0 -0
  84. package/src/client/img/Xetthecum/Laughlin_Lake.jpg +0 -0
  85. package/src/client/img/Xetthecum/Marine_banner.jpg +0 -0
  86. package/src/client/img/Xetthecum/Mature_Conifer_Forest.jpg +0 -0
  87. package/src/client/img/Xetthecum/Porlier_Pass_Road.jpg +0 -0
  88. package/src/client/img/Xetthecum/Riparian.jpg +0 -0
  89. package/src/client/img/Xetthecum/Rural.jpg +0 -0
  90. package/src/client/img/Xetthecum/Subtidal.jpg +0 -0
  91. package/src/client/img/Xetthecum/Wetland.jpg +0 -0
  92. package/src/client/img/Xetthecum/Woodland.jpg +0 -0
  93. package/src/client/img/Xetthecum/Woodland_banner.jpg +0 -0
  94. package/src/client/img/Xetthecum/Xetthecum.jpg +0 -0
  95. package/src/client/img/Xetthecum/Young_Conifer_Forest.jpg +0 -0
  96. package/src/client/img/Xetthecum/Young_Mixed_Forest.jpg +0 -0
  97. package/src/client/img/Xetthecum/food.svg +38 -0
  98. package/src/client/img/Xetthecum/indicator.svg +52 -0
  99. package/src/client/img/Xetthecum/material.svg +186 -0
  100. package/src/client/img/Xetthecum/medicinal.svg +62 -0
  101. package/src/client/img/Xetthecum/missing-value.svg +3 -0
  102. package/src/client/img/Xetthecum/spiritual.svg +13 -0
  103. package/src/client/img/Xetthecum/trade.svg +74 -0
  104. package/src/client/img/audio.svg +5 -0
  105. package/src/client/img/external-link.svg +3 -0
  106. package/src/client/img/selected-eye.svg +4 -0
  107. package/src/client/img/selected-tick.svg +3 -0
  108. package/src/client/img/thumbs/BCLeps.jpg +0 -0
  109. package/src/client/img/thumbs/Checklist.jpg +0 -0
  110. package/src/client/img/thumbs/Data-Paper-Figma.jpg +0 -0
  111. package/src/client/img/thumbs/Galiano-Data-Paper.jpg +0 -0
  112. package/src/client/img/thumbs/Galiano.jpg +0 -0
  113. package/src/client/img/thumbs/Maxwell.jpg +0 -0
  114. package/src/client/img/thumbs/Squamish.jpg +0 -0
  115. package/src/client/img/thumbs/Valdes.jpg +0 -0
  116. package/src/client/img/thumbs/Xetthecum.jpg +0 -0
  117. package/src/client/img/up-arrow.svg +4 -0
  118. package/src/client/js/autocomplete.js +53 -0
  119. package/src/client/js/checklist.js +179 -0
  120. package/src/client/js/colour.js +139 -0
  121. package/src/client/js/datasetControls.js +212 -0
  122. package/src/client/js/imerss-viz.js +1397 -0
  123. package/src/client/js/leafletMap.js +357 -0
  124. package/src/client/js/leafletMapWithGrid.js +337 -0
  125. package/src/client/js/leafletMapWithRegions.js +381 -0
  126. package/src/client/js/renderSVG.js +108 -0
  127. package/src/client/js/tabs.js +53 -0
  128. package/src/client/js/xetthecum.js +106 -0
  129. package/src/client/json/emptyPhyloMap.json +1 -0
  130. package/src/client/json/kingdomPhyloMap.json +17 -0
  131. package/src/client/json/phyloMap.json +58 -0
  132. package/src/compare.js +97 -0
  133. package/src/compilecsv.js +47 -0
  134. package/src/dataProcessing/LZ4.js +27 -0
  135. package/src/dataProcessing/coordinatePatch.js +151 -0
  136. package/src/dataProcessing/readCSV.js +107 -0
  137. package/src/dataProcessing/readCSVwithMap.js +90 -0
  138. package/src/dataProcessing/readCSVwithoutMap.js +25 -0
  139. package/src/dataProcessing/readJSON.js +23 -0
  140. package/src/dataProcessing/sqlite.js +174 -0
  141. package/src/dataProcessing/summarise.js +122 -0
  142. package/src/dataProcessing/taxonScrape.js +14 -0
  143. package/src/dataProcessing/worker.js +31 -0
  144. package/src/dataProcessing/writeCSV.js +34 -0
  145. package/src/dataProcessing/writeJSON.js +15 -0
  146. package/src/dbtaxa.js +96 -0
  147. package/src/deAuthorise.js +48 -0
  148. package/src/deqgis.js +148 -0
  149. package/src/expressions/expr.js +65 -0
  150. package/src/expressions/jsep.js +925 -0
  151. package/src/filterFirst.js +51 -0
  152. package/src/geom/geoJSON.js +121 -0
  153. package/src/iNaturalist/examples/api.inaturalist.org-v1-taxa!q=Acmispon americanus americanus.json +200 -0
  154. package/src/iNaturalist/examples/api.inaturalist.org-v1-taxa!q=Flabellina japonica.json +61 -0
  155. package/src/iNaturalist/examples/www.inaturalist.org-taxa-877359.json +132 -0
  156. package/src/iNaturalist/iNatUrls.js +17 -0
  157. package/src/iNaturalist/obsAPI.js +150 -0
  158. package/src/iNaturalist/taxonAPI.js +460 -0
  159. package/src/inatObs.js +120 -0
  160. package/src/inattify.js +123 -0
  161. package/src/intersect.js +80 -0
  162. package/src/intersectFilter.js +48 -0
  163. package/src/landlocked.js +137 -0
  164. package/src/lib/jquery.binarytransport.js +51 -0
  165. package/src/lib/lz4-commentary.txt +9 -0
  166. package/src/lib/lz4.js +73 -0
  167. package/src/lib/point-in-polygon.js +36 -0
  168. package/src/lz4.js +40 -0
  169. package/src/marmalise.js +333 -0
  170. package/src/materialise.js +68 -0
  171. package/src/reintegrate.js +71 -0
  172. package/src/taxonomise.js +615 -0
  173. package/src/unscrewGBIF.js +44 -0
  174. package/src/utils/dataSource.js +173 -0
  175. package/src/utils/settleStructure.js +62 -0
  176. package/src/utils/utils.js +92 -0
  177. package/src/viz/plethify.js +39 -0
  178. package/src/wormify.js +154 -0
  179. package/src/xetthecum/xetthecumFilters.js +126 -0
  180. package/test/testDataSource.js +182 -0
  181. package/test/testiNatObsSource.js +38 -0
  182. package/test/testiNatSource.js +50 -0
@@ -0,0 +1,174 @@
1
+ /* eslint-env node */
2
+
3
+ "use strict";
4
+
5
+ const fluid = require("infusion");
6
+ const hortis = fluid.registerNamespace("hortis");
7
+
8
+ const sqlite = require("sqlite");
9
+ const sqlite3 = require("sqlite3");
10
+ const pako = require("pako");
11
+
12
+ fluid.registerNamespace("hortis.codecs.zlib");
13
+
14
+ hortis.codecs.zlib.encode = function (obj) {
15
+ const string = JSON.stringify(obj);
16
+ const togo = string === undefined ? undefined : pako.deflate(string);
17
+ return togo;
18
+ };
19
+
20
+ hortis.codecs.zlib.decode = function (buffer) {
21
+ if (!buffer) {
22
+ return buffer;
23
+ } else {
24
+ const string = pako.inflate(buffer, {to: "string"});
25
+ return JSON.parse(string);
26
+ }
27
+ };
28
+
29
+ fluid.defaults("hortis.sqliteDB", {
30
+ gradeNames: ["fluid.modelComponent", "fluid.resourceLoader"],
31
+ // dbFile: String
32
+ model: {
33
+ db: "{that}.resources.db.parsed"
34
+ },
35
+ invokers: {
36
+ beginTransaction: "hortis.sqliteDB.beginTransaction({that}.model.db)",
37
+ endTransaction: "hortis.sqliteDB.endTransaction({that}.model.db)"
38
+ },
39
+ resources: {
40
+ db: {
41
+ promiseFunc: "hortis.sqliteDB.openDB",
42
+ promiseArgs: ["{that}.options.dbFile"]
43
+ }
44
+ }
45
+ });
46
+
47
+ hortis.sqliteDB.beginTransaction = function (db) {
48
+ return db.run("BEGIN");
49
+ };
50
+
51
+ hortis.sqliteDB.endTransaction = function (db) {
52
+ return db.run("COMMIT");
53
+ };
54
+
55
+ /** Opens an SQLite database held at the supplied module-relative path
56
+ * @param {String }dbFile - The (possibly module-qualified) filesystem path holding the database
57
+ * @return {Promise<Database>} A promise for an "sqlite" wrapper for the opened SQLite database
58
+ */
59
+ hortis.sqliteDB.openDB = async function (dbFile) {
60
+ const path = fluid.module.resolvePath(dbFile);
61
+ try {
62
+ return sqlite.open({
63
+ filename: path,
64
+ driver: sqlite3.Database
65
+ });
66
+ } catch (err) {
67
+ console.log("Error opening database ", err);
68
+ }
69
+ };
70
+
71
+ fluid.defaults("hortis.sqliteSource", {
72
+ gradeNames: ["fluid.dataSource", "fluid.dataSource.noencoding", "fluid.dataSource.writable", "fluid.resourceLoader"],
73
+ // createString: String
74
+ // readQuery: Object
75
+ // writeQuery: Object
76
+ // columnCodecs: Object
77
+ model: {
78
+ db: "{that}.db.model.db",
79
+ tableCreated: "{that}.resources.table.parsed"
80
+ },
81
+ components: {
82
+ db: "{hortis.sqliteDB}"
83
+ },
84
+ resources: {
85
+ table: {
86
+ promiseFunc: "hortis.sqliteSource.createTable",
87
+ promiseArgs: ["{that}.db", "{that}.options.createString"]
88
+ }
89
+ },
90
+ listeners: {
91
+ "onRead.impl": {
92
+ func: "hortis.sqliteSource.read", // payload, options
93
+ args: ["{that}", "{that}.options.readQuery", "{arguments}.0", "{arguments}.1"] // can't be {that}.model.db because of FLUID-6752
94
+ },
95
+ "onWrite.impl": {
96
+ func: "hortis.sqliteSource.write", // payload
97
+ args: ["{that}", "{that}.options.writeQuery", "{arguments}.0"]
98
+ }
99
+ }
100
+ });
101
+
102
+ fluid.defaults("hortis.listableSqliteSource", {
103
+ gradeNames: ["hortis.sqliteSource", "fluid.dataSource.listable"],
104
+ // listQuery: Object
105
+ invokers: {
106
+ list: "hortis.sqliteSource.list({that}.model.db, {that}.options.listQuery)"
107
+ }
108
+ });
109
+
110
+ hortis.sqliteSource.createTable = function (dbThat, createString) {
111
+ console.log("Beginning to create table");
112
+ const togo = fluid.promise();
113
+ // Note that we can't expect the database component itself to construct until the entire tree constructs, we need
114
+ // to wait for the individual resource. Note also that the promise itself doesn't resolve to much useful.
115
+ // What do we expect about this in the future? That every "observation" of a model value is promisified? Or that it
116
+ // just has a reducer syntax.
117
+ dbThat.resources.db.promise.then(function () {
118
+ const db = dbThat.model.db;
119
+ console.log("Database opened");
120
+ db.run(createString).then(function () {
121
+ console.log("Table created");
122
+ togo.resolve(true);
123
+ }, function (err) {
124
+ togo.reject("Error creating table " + err);
125
+ });
126
+ }, function (err) {
127
+ togo.reject("Error waiting for database " + err);
128
+ });
129
+ return togo;
130
+ };
131
+
132
+ hortis.sqliteSource.prepareQueryArgs = function (args, directModel) {
133
+ return fluid.transform(args, function (arg) {
134
+ return typeof(arg) === "string" && arg.charAt(0) === "%" ?
135
+ fluid.get(directModel, arg.slice(1)) : arg;
136
+ });
137
+ };
138
+
139
+ hortis.deprefixColumn = function (column) {
140
+ return column.startsWith("$") ? column.substring(1) : column;
141
+ };
142
+
143
+ hortis.transcodeColumns = function (direction, columns, columnCodecs) {
144
+ return fluid.transform(columns, function (value, key) {
145
+ const column = hortis.deprefixColumn(key);
146
+ const codec = columnCodecs[column];
147
+ if (codec) {
148
+ const holder = hortis.codecs[codec];
149
+ if (!holder) {
150
+ fluid.fail("Unknown column codec ", codec, " allowable options are ", Object.keys(hortis.codecs).join(", "));
151
+ }
152
+ return holder[direction](value);
153
+ } else {
154
+ return value;
155
+ }
156
+ });
157
+ };
158
+
159
+ hortis.sqliteSource.read = async function (that, readQuery, value, options) {
160
+ const db = that.model.db; // for FLUID-6752
161
+ const result = await db.get(readQuery.query, hortis.sqliteSource.prepareQueryArgs(readQuery.args, options.directModel));
162
+ return hortis.transcodeColumns("decode", result, that.options.columnCodecs);
163
+ };
164
+
165
+ hortis.sqliteSource.write = function (that, writeQuery, value) {
166
+ const db = that.model.db;
167
+ const prepared = hortis.sqliteSource.prepareQueryArgs(writeQuery.args, value);
168
+ const encoded = hortis.transcodeColumns("encode", prepared, that.options.columnCodecs);
169
+ return db.run(writeQuery.query, encoded);
170
+ };
171
+
172
+ hortis.sqliteSource.list = function (db, writeQuery) {
173
+ return db.all(writeQuery.query);
174
+ };
@@ -0,0 +1,122 @@
1
+ /* eslint-env node */
2
+
3
+ "use strict";
4
+
5
+ const fluid = require("infusion");
6
+
7
+ const hortis = fluid.registerNamespace("hortis");
8
+
9
+ fluid.defaults("hortis.summarise", {
10
+ gradeNames: "fluid.component",
11
+ summarise: true,
12
+ members: {
13
+ uniqueRows: {},
14
+ discardedRows: {}
15
+ },
16
+ summaryOutMapExtraColumns: { // TODO: In case we need an obs file with, e.g. Hulq names - propagate this through from fusion
17
+ },
18
+ fields: {
19
+ date: "dateObserved",
20
+ dateResolution: "dateObservedResolution",
21
+ unique: "iNaturalistTaxonName", // Field which identifies observations as being of the same taxon
22
+ obsCount: "observationCount",
23
+ obsId: "observationId",
24
+ coords: "coords"
25
+ },
26
+ invokers: {
27
+ storeRow: "hortis.summarise.storeRow({that}, {arguments}.0)"
28
+ }
29
+ });
30
+
31
+ // TODO: Wrong place for all of this processing - coordinates should be processed as part of the main
32
+ // parsing of the obs file
33
+
34
+ // Taken from https://stackoverflow.com/a/1140335 with several fixes
35
+ hortis.summarise.convertDMStoDD = function (degrees, minutes, seconds, direction) {
36
+ let dd = +degrees + minutes / 60 + seconds / (60 * 60);
37
+
38
+ if (direction === "S" || direction === "W") {
39
+ dd = -1 * Math.abs(dd);
40
+ }
41
+ return dd;
42
+ };
43
+
44
+ hortis.summarise.parseDMS = function (string) {
45
+ const parts = string.split(/[^\d\w]+/);
46
+ return hortis.summarise.convertDMStoDD(parts[0], parts[1], parts[2], parts[3]);
47
+ };
48
+
49
+ hortis.summarise.parseCoordinate = function (value) {
50
+ if (typeof(value) === "string" && value.match(/[NSEW]/)) {
51
+ return hortis.summarise.parseDMS(value);
52
+ } else {
53
+ return hortis.parseFloat(value);
54
+ }
55
+ };
56
+
57
+ hortis.summarise.parseCoordinates = function (row) {
58
+ const round = function (coord) {
59
+ return +hortis.roundDecimals(coord, 6);
60
+ };
61
+ const latitude = hortis.summarise.parseCoordinate(row.latitude);
62
+ const longitude = hortis.summarise.parseCoordinate(row.longitude);
63
+ return !isNaN(latitude) && !isNaN(longitude) ? [round(latitude), round(longitude)] : null;
64
+ };
65
+
66
+ hortis.summarise.copyObsFields = function (target, source, prefix, fields) {
67
+ fields.forEach(function (field) {
68
+ const targetField = prefix + hortis.capitalize(field);
69
+ target[targetField] = source[field];
70
+ });
71
+ };
72
+
73
+ hortis.summarise.updateRowRange = function (that, summaryRow, obsRow) {
74
+ const obsCountField = that.options.fields.obsCount;
75
+ if (!summaryRow) {
76
+ const commonKeys = Object.keys(hortis.summariseCommonOutMap.columns);
77
+ const extraKeys = Object.keys(that.options.summaryOutMapExtraColumns);
78
+ summaryRow = fluid.filterKeys(obsRow, commonKeys.concat(extraKeys));
79
+ summaryRow.firstTimestamp = Infinity;
80
+ summaryRow.lastTimestamp = -Infinity;
81
+ summaryRow[obsCountField] = 1;
82
+ } else {
83
+ summaryRow[obsCountField]++;
84
+ }
85
+ const fields = Object.keys(hortis.obsToSummaryFields);
86
+ if (obsRow.timestamp < summaryRow.firstTimestamp) {
87
+ hortis.summarise.copyObsFields(summaryRow, obsRow, "first", fields);
88
+ }
89
+
90
+ if (obsRow.timestamp > summaryRow.lastTimestamp) {
91
+ hortis.summarise.copyObsFields(summaryRow, obsRow, "last", fields);
92
+ }
93
+
94
+ return summaryRow;
95
+ };
96
+
97
+ hortis.summarise.storeRow = function (that, row) {
98
+ const fields = that.options.fields;
99
+ const coordsField = fields.coords;
100
+ row.timestamp = Date.parse(row[fields.date]);
101
+ const uniqueVal = row[fields.unique];
102
+ const existing = that.uniqueRows[uniqueVal];
103
+
104
+ const summaryRow = hortis.summarise.updateRowRange(that, existing, row);
105
+
106
+ that.uniqueRows[uniqueVal] = summaryRow;
107
+
108
+ if (that.options.summarise) {
109
+ const obsId = row[fields.obsId];
110
+ if (obsId === undefined) {
111
+ fluid.fail("Unable to find unique observation field for row ", row);
112
+ }
113
+ // TODO: Wrong place for this logic - these should be parsed as part of main filtering workflow, and before
114
+ // hortis.roundCoordinates
115
+ const coords = hortis.summarise.parseCoordinates(row);
116
+ fluid.set(summaryRow, [coordsField, obsId], coords);
117
+ if (coords) {
118
+ row.latitude = coords[0];
119
+ row.longitude = coords[1];
120
+ }
121
+ }
122
+ };
@@ -0,0 +1,14 @@
1
+ /* eslint-env node */
2
+
3
+ "use strict";
4
+
5
+ var fluid = require("infusion");
6
+ var fs = require("fs");
7
+
8
+ var hortis = fluid.registerNamespace("hortis");
9
+
10
+ hortis.writeTaxonDoc = function (filename, doc) {
11
+ var formatted = JSON.stringify(doc, null, 4);
12
+ fs.writeFileSync(filename, formatted);
13
+ console.log("Written " + formatted.length + " bytes to " + filename);
14
+ };
@@ -0,0 +1,31 @@
1
+ /* eslint-env node */
2
+
3
+ "use strict";
4
+
5
+ var fluid = require("infusion");
6
+
7
+ var hortis = fluid.registerNamespace("hortis");
8
+
9
+ // Work in progress factoring common engine from wormify.js and inattify.js
10
+
11
+ fluid.defaults("hortis.worker", {
12
+ gradeNames: "fluid.component",
13
+ members: {
14
+ queue: "{that}.options.queue",
15
+ fetched: [],
16
+ cache: {}
17
+ },
18
+ invokers: {
19
+ oneWork: "hortis.worker.oneWork({that})",
20
+ // Returns {name, filename}
21
+ namesFromItem: "fluid.notImplemented()"
22
+ }
23
+ });
24
+
25
+ hortis.worker.oneWork = function (that) {
26
+ if (that.queue.length) {
27
+ var head = that.queue.shift();
28
+ var names = that.namesFromItem(head);
29
+ that.oneWork(head, names);
30
+ }
31
+ };
@@ -0,0 +1,34 @@
1
+ /* eslint-env node */
2
+
3
+ "use strict";
4
+
5
+ var fluid = require("infusion");
6
+ var csvWriterModule = require("csv-writer");
7
+
8
+ var hortis = fluid.registerNamespace("hortis");
9
+
10
+ require("./readCSVWithmap.js");
11
+ require("./readCSVWithoutMap.js");
12
+ require("../utils/utils.js");
13
+
14
+ hortis.writeCSV = function (fileName, headersOrMap, rows, completionPromise) {
15
+ var isHeaders = fluid.isArrayable(headersOrMap);
16
+ var headers = isHeaders ? headersOrMap : Object.values(headersOrMap);
17
+ var duplicates = hortis.findDuplicates(headers);
18
+ if (duplicates.length > 0) {
19
+ fluid.fail("Error in hortis.writeCSV: duplicate column names: ", duplicates.join(", "));
20
+ }
21
+ var csvWriter = csvWriterModule.createArrayCsvWriter({
22
+ header: headers,
23
+ path: fileName
24
+ });
25
+ var arrayRows = [];
26
+ fluid.each(rows, function (row) {
27
+ arrayRows.push(hortis[isHeaders ? "rowToArray" : "mapToArray"](row, headersOrMap));
28
+ });
29
+ var promise = csvWriter.writeRecords(arrayRows);
30
+ promise.then(function () {
31
+ fluid.log("Written " + arrayRows.length + " lines to CSV file ", fileName);
32
+ });
33
+ fluid.promise.follow(promise, completionPromise);
34
+ };
@@ -0,0 +1,15 @@
1
+ /* eslint-env node */
2
+
3
+ "use strict";
4
+
5
+ var fluid = require("infusion");
6
+ var fs = require("fs");
7
+
8
+ var hortis = fluid.registerNamespace("hortis");
9
+
10
+ hortis.writeJSONSync = function (inFilename, doc) {
11
+ var filename = fluid.module.resolvePath(inFilename);
12
+ var formatted = JSON.stringify(doc, null, 4) + "\n";
13
+ fs.writeFileSync(filename, formatted);
14
+ console.log("Written " + formatted.length + " bytes to " + filename);
15
+ };
package/src/dbtaxa.js ADDED
@@ -0,0 +1,96 @@
1
+ /* eslint-env node */
2
+
3
+ "use strict";
4
+
5
+ const fluid = require("infusion");
6
+ const fs = require("fs");
7
+
8
+ require("./dataProcessing/readJSON.js");
9
+ require("./iNaturalist/taxonAPI.js");
10
+
11
+ const glob = require("glob");
12
+
13
+ const taxonAPIFileBase = "data/iNaturalist/taxonAPI";
14
+
15
+ const hortis = fluid.registerNamespace("hortis");
16
+
17
+ const taxonAPIs = hortis.iNatTaxonAPI.dbTaxonAPIs();
18
+ taxonAPIs.events.onCreate.then((function (taxonAPIs) {
19
+ return traverseDB(taxonAPIs.byId, taxonAPIs.db);
20
+ }), function (err) {
21
+ console.log("Got error instantiating APIs ", err);
22
+ });
23
+
24
+ const now = Date.now();
25
+
26
+ const dataFiles = glob.sync(taxonAPIFileBase + "/*.json");
27
+
28
+ console.log("Found " + dataFiles.length + " taxon files in " + (Date.now() - now) + "ms");
29
+
30
+ hortis.idFromFilename = function (filename) {
31
+ const lastslash = filename.lastIndexOf("/");
32
+ const lastdot = filename.lastIndexOf(".");
33
+ return filename.substring(lastslash + 1, lastdot);
34
+ };
35
+
36
+ async function traverseDB(source, db) {
37
+
38
+ console.log("Beginning insert");
39
+ const beforeWrite = Date.now();
40
+
41
+ const writeAll = async function (dataFiles) {
42
+ await db.beginTransaction();
43
+ for (let i = 0; i < dataFiles.length; ++i) {
44
+ const dataFile = dataFiles[i];
45
+ const contents = hortis.readJSONSync(dataFile);
46
+ const stats = fs.statSync(dataFile);
47
+ const fetched_at = stats.mtime.toISOString();
48
+ await source.set({id: contents.id}, {
49
+ id: contents.id,
50
+ fetched_at: fetched_at,
51
+ doc: contents
52
+ });
53
+ if (i % 1000 === 0) {
54
+ process.stdout.write(i + " ... ");
55
+ }
56
+ }
57
+ await db.endTransaction();
58
+ };
59
+
60
+ try {
61
+ await writeAll(dataFiles);
62
+ } catch (err) {
63
+ console.log("Error writing files ", err);
64
+ throw err;
65
+ }
66
+
67
+ console.log("\nWritten " + dataFiles.length + " taxon files in " + (Date.now() - beforeWrite) + "ms");
68
+
69
+ const allIds = await source.list();
70
+
71
+ console.log("Got " + allIds.length + " ids: ", allIds[0]);
72
+
73
+ const beforeRead = Date.now();
74
+
75
+ const readAll = async function () {
76
+ let index = 0;
77
+ let first = true;
78
+ for (const {id: key} of allIds) {
79
+
80
+ if (index % 100 === 0) {
81
+ process.stdout.write("\"" + key + "\" ");
82
+ }
83
+ const row = await source.get({id: key});
84
+ const doc = row.doc;
85
+ if (first) {
86
+ console.log("First doc: ", doc);
87
+ first = false;
88
+ }
89
+ ++index;
90
+ }
91
+ };
92
+
93
+ await readAll();
94
+ console.log("\nRead " + allIds.length + " taxon files in " + (Date.now() - beforeRead) + "ms");
95
+
96
+ }
@@ -0,0 +1,48 @@
1
+ /* eslint-env node */
2
+
3
+ "use strict";
4
+
5
+ const fluid = require("infusion");
6
+ const minimist = require("minimist");
7
+ fluid.require("%imerss-bioinfo");
8
+
9
+ require("./dataProcessing/readCSV.js");
10
+ require("./dataProcessing/readCSVwithMap.js");
11
+ require("./dataProcessing/writeCSV.js");
12
+
13
+ const hortis = fluid.registerNamespace("hortis");
14
+
15
+ fluid.setLogging(true);
16
+
17
+ // old one "%imerss-bioinfo/data/Comprehensive Lists/sji_master_flora.csv"); used Full Species
18
+
19
+ const parsedArgs = minimist(process.argv.slice(2));
20
+
21
+ const outputFile = parsedArgs.o || "deauthorized.csv";
22
+ const inputFile = parsedArgs._[0] || fluid.module.resolvePath("%imerss-bioinfo/data/Squamish/GBIF_2022_Plantae_DwC.csv");
23
+
24
+ const reader = hortis.csvReaderWithoutMap({
25
+ inputFile: inputFile
26
+ });
27
+
28
+ // One-off script to parse out authorities from files such as those received from Dunwiddie
29
+
30
+ reader.completionPromise.then(async function () {
31
+ const mapped = [];
32
+ for (let i = 0; i < reader.rows.length; ++i) {
33
+ console.log("Processing row ", i);
34
+ const row = reader.rows[i];
35
+ const taxon = row["scientificName"];
36
+ const parsed = taxon.split(" ");
37
+ row.taxonName = parsed[0] + " " + parsed[1];
38
+ const infra = row["Infra taxa"] || row["infraspecificEpithet"];
39
+ if (infra) {
40
+ row.infraTaxonName = row.taxonName + " " + infra;
41
+ } else {
42
+ row.infraTaxonName = "";
43
+ }
44
+ row.authority = parsed.slice(2).join(" ");
45
+ mapped.push(row);
46
+ }
47
+ hortis.writeCSV(fluid.module.resolvePath(outputFile), Object.keys(mapped[0]), mapped, fluid.promise());
48
+ });
package/src/deqgis.js ADDED
@@ -0,0 +1,148 @@
1
+ /* eslint-env node */
2
+
3
+ "use strict";
4
+
5
+ const fluid = require("infusion");
6
+ const fs = require("fs");
7
+ const minimist = require("minimist");
8
+ const glob = require("glob");
9
+
10
+ fluid.require("%imerss-bioinfo");
11
+
12
+ require("./dataProcessing/readJSON.js");
13
+ require("./dataProcessing/writeJSON.js");
14
+ require("./dataProcessing/readCSV.js");
15
+ require("./dataProcessing/readCSVWithoutMap.js");
16
+
17
+ const hortis = fluid.registerNamespace("hortis");
18
+
19
+ fluid.setLogging(true);
20
+
21
+ const parsedArgs = minimist(process.argv.slice(2));
22
+
23
+ const inputFile = parsedArgs._[0] || "%imerss-bioinfo/data/Xetthecum/deqgis.json5";
24
+ const outputFile = fluid.module.resolvePath(parsedArgs.o || "%imerss-bioinfo/data/Xetthecum/flatFeatures.json");
25
+
26
+ const config = hortis.readJSONSync(fluid.module.resolvePath(inputFile));
27
+
28
+ const baseDir = fluid.module.resolvePath(config.baseDirectory);
29
+
30
+ const dataFiles = glob.sync(baseDir + "/*.js");
31
+
32
+ console.log("Got data files ", dataFiles);
33
+
34
+ hortis.seColumns = ["Tagline", "What", "Where", "Importance", "Protection", "Source"];
35
+
36
+ hortis.applySensitiveEcosystems = function (togo, rows) {
37
+ rows.forEach(function (row, index) {
38
+ const labels = row.LABELS.split(",").map(s => s.trim());
39
+ labels.forEach(function (clazz) {
40
+ if (togo.classes[clazz]) {
41
+ hortis.seColumns.forEach(function (column) {
42
+ const cell = row[column];
43
+ if (cell) {
44
+ togo.classes[clazz]["sE-" + column] = cell;
45
+ }
46
+ else {
47
+ console.log("Warnings: Sensitive ecosystems column " + column + " did not match row " + index, row);
48
+ }
49
+ });
50
+ } else if (clazz) {
51
+ console.log("Warning: Sensitive ecosystems column " + clazz + " did not match any from QGIS");
52
+ }
53
+ });
54
+ });
55
+ };
56
+
57
+ hortis.applyCulturalValues = function (togo, rows) {
58
+ rows.forEach(function (row) {
59
+ const communityName = row.COMMUNITIES;
60
+ if (togo.communities[communityName]) {
61
+ const community = togo.communities[communityName];
62
+ community.culturalValues = row["CULTURAL VALUES"];
63
+ community.culturalValuesSources = row.SOURCES;
64
+ } else {
65
+ console.log("Warning: Cultural values column " + communityName + " did not match any from QGIS");
66
+ }
67
+ });
68
+ };
69
+
70
+ const features = [];
71
+
72
+ hortis.indexCommunities = function (communities) {
73
+ const classes = {};
74
+ fluid.each(communities, function (community, communityKey) {
75
+ fluid.each(community.classes, function (clazz, classKey) {
76
+ clazz.community = communityKey;
77
+ classes[classKey] = clazz;
78
+ });
79
+ });
80
+ return classes;
81
+ };
82
+
83
+ hortis.filterCommunities = function (communities) {
84
+ return fluid.transform(communities, function (community) {
85
+ return fluid.extend(fluid.censorKeys(community, ["classes"]), {
86
+ classes: fluid.transform(community.classes, function () {
87
+ return {};
88
+ })
89
+ });
90
+ });
91
+ };
92
+
93
+ const labels = {};
94
+
95
+ hortis.indexOneFeature = function (dataFileName, feature, features, classes) {
96
+ const classname = feature.properties.LEGEND_LAB || feature.properties.LABEL;
97
+ fluid.set(labels, [feature.properties.COMMUNITY, classname], true);
98
+
99
+ feature.properties.clazz = classname;
100
+ const clazz = classes[classname];
101
+ if (!clazz) {
102
+ console.log("Warning: unknown feature with name " + classname + " in file " + dataFileName);
103
+ } else {
104
+ features.push(feature);
105
+ }
106
+ };
107
+
108
+ const classes = hortis.indexCommunities(config.communities);
109
+
110
+ const CSVs = fluid.transform(config.CSVs, function (oneFile) {
111
+ return hortis.csvReaderWithoutMap({
112
+ inputFile: fluid.module.resolvePath(oneFile.path)
113
+ });
114
+ });
115
+
116
+ const promises = Object.values(fluid.getMembers(CSVs, "completionPromise"));
117
+
118
+ fluid.promise.sequence(promises).then(function () {
119
+ console.log("Loaded " + Object.keys(CSVs).length + " CSV files");
120
+ const allRows = fluid.getMembers(CSVs, "rows");
121
+
122
+ dataFiles.forEach(function (dataFileName) {
123
+ const string = fs.readFileSync(dataFileName, "utf8");
124
+ const firstp = string.indexOf("{");
125
+ const lastp = string.lastIndexOf("}");
126
+ const data = JSON.parse(string.substring(firstp, lastp + 1));
127
+ console.log("Processing qgis2web data with name " + data.name);
128
+ data.features.forEach(function (feature) {
129
+ hortis.indexOneFeature(dataFileName, feature, features, classes);
130
+ });
131
+ });
132
+
133
+ const togo = {
134
+ communities: hortis.filterCommunities(config.communities),
135
+ classes: classes,
136
+ features: features
137
+ };
138
+
139
+ if (allRows.sensitiveEcosystems) {
140
+ hortis.applySensitiveEcosystems(togo, allRows.sensitiveEcosystems);
141
+ }
142
+ if (allRows.culturalValues) {
143
+ hortis.applyCulturalValues(togo, allRows.culturalValues);
144
+ }
145
+
146
+ console.log("Encountered feature hierarchy: ", JSON.stringify(labels, null, 2));
147
+ hortis.writeJSONSync(outputFile, togo);
148
+ });