@larsgw/formica 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -0
- package/lib/bin/process-resources-index.js +10 -4
- package/lib/bin/process-resources.js +214 -142
- package/lib/bin/validate-resources-text.js +0 -0
- package/lib/index.d.ts +1 -0
- package/lib/index.js +2 -1
- package/lib/resources/parse-text.js +26 -10
- package/lib/taxon-names/index.d.ts +2 -0
- package/lib/taxon-names/index.js +79 -0
- package/package.json +4 -1
- package/src/bin/process-resources-index.ts +11 -4
- package/src/bin/process-resources.ts +154 -99
- package/src/index.ts +1 -0
- package/src/module.d.ts +70 -50
- package/src/resources/parse-text.ts +28 -11
- package/src/taxon-names/index.ts +79 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# [0.3.0](https://github.com/identification-resources/formica/compare/v0.2.1...v0.3.0) (2023-08-20)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Features
|
|
5
|
+
|
|
6
|
+
* **resources:** add accepted ids to dwc and index ([1b5aba8](https://github.com/identification-resources/formica/commit/1b5aba8ff07bba32fc1bbf49927fef7a880eb35e))
|
|
7
|
+
* **resources:** allow updates of just mappings ([0ea62c1](https://github.com/identification-resources/formica/commit/0ea62c143a49129709960de123c0037973356487))
|
|
8
|
+
* **resources:** improve prefix selection heuristics ([7c2e0c9](https://github.com/identification-resources/formica/commit/7c2e0c99d9de7490314f971e8b4e9e5e190a81c2))
|
|
9
|
+
* **resources:** improve taxon name matching ([78ff480](https://github.com/identification-resources/formica/commit/78ff480485ca42bd1d1f2893f230e61db6cb6be8)), closes [#2](https://github.com/identification-resources/formica/issues/2)
|
|
10
|
+
* **resources:** test for rank mismatch ([0e69b7f](https://github.com/identification-resources/formica/commit/0e69b7f0a6654fa5255e797be7516cd06edb4df3)), closes [#2](https://github.com/identification-resources/formica/issues/2)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
## [0.2.1](https://github.com/identification-resources/formica/compare/v0.2.0...v0.2.1) (2023-08-10)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
### Bug Fixes
|
|
18
|
+
|
|
19
|
+
* **resources:** do not validate old versions ([1a8dd2e](https://github.com/identification-resources/formica/commit/1a8dd2e0e373489e287d1fd89dbb7443245c5214))
|
|
20
|
+
* **resources:** fix check for missing leaf taxa ([035f1ad](https://github.com/identification-resources/formica/commit/035f1ad20efa80b388ec3828d4ace4518c7b344e))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# [0.2.0](https://github.com/identification-resources/formica/compare/v0.1.1...v0.2.0) (2023-08-09)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
### Features
|
|
28
|
+
|
|
29
|
+
* **catalog:** add taxon_scope column ([6a06c37](https://github.com/identification-resources/formica/commit/6a06c37ec0640a8ffbc258ad6a4d81d84d35fee9))
|
|
30
|
+
* **resources:** check for missing leaf taxa ([387c47d](https://github.com/identification-resources/formica/commit/387c47d10bb7f1fb09a68c10dfa869bb924b7633))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
## [0.1.1](https://github.com/identification-resources/formica/compare/v0.1.0...v0.1.1) (2023-05-10)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
### Bug Fixes
|
|
38
|
+
|
|
39
|
+
* **processor:** add Node hashbang ([51dee57](https://github.com/identification-resources/formica/commit/51dee57b8547afba449b2b846bd7d65e7927368f))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
# [0.1.0](https://github.com/identification-resources/formica/compare/fed91fd6f350c47bd067d221a4d0e2278a199dae...v0.1.0) (2023-03-01)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
### Bug Fixes
|
|
47
|
+
|
|
48
|
+
* **resources:** fix typo in taxon name pattern ([8f1b6cf](https://github.com/identification-resources/formica/commit/8f1b6cfa3858edb83c0aee589b84ad760ab815af))
|
|
49
|
+
* **resources:** improve name parsing heuristics ([a40642e](https://github.com/identification-resources/formica/commit/a40642e9bdaeaae363584712e45f218809a5754b))
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
### Features
|
|
53
|
+
|
|
54
|
+
* **catalog:** add SDK for catalog data ([fed91fd](https://github.com/identification-resources/formica/commit/fed91fd6f350c47bd067d221a4d0e2278a199dae))
|
|
55
|
+
* **csv:** allow custom delims in output ([b523614](https://github.com/identification-resources/formica/commit/b523614171b0ff96347cb876a08106df413c4032))
|
|
56
|
+
* **processor:** processor for DwC creation ([0e6f21d](https://github.com/identification-resources/formica/commit/0e6f21dfe00a2892348a4ab04fcaf3d6c8324a91))
|
|
57
|
+
* **processor:** processor for DwC indexing ([279773b](https://github.com/identification-resources/formica/commit/279773b085676cf7cb3eb4d81496ad219ac85bce))
|
|
58
|
+
* **resources:** add SDK for resource text files ([0acd36d](https://github.com/identification-resources/formica/commit/0acd36dde04cc1240cb8be97d48812b201b2685a))
|
|
59
|
+
* **resources:** add support for cluster markings ([112315c](https://github.com/identification-resources/formica/commit/112315c7ebd7151c4e2cc77ab20e0cec3f579db6))
|
|
60
|
+
* **resources:** add support for diffs as input ([9751627](https://github.com/identification-resources/formica/commit/9751627a916f47552661596048f1f0d44d89b102))
|
|
61
|
+
* **resources:** add support for hybrids ([e49e96a](https://github.com/identification-resources/formica/commit/e49e96af9c3958bab2a3b0508b1fc8a018b80716))
|
|
62
|
+
* **validate:** add catalog validation script ([4cdf785](https://github.com/identification-resources/formica/commit/4cdf785afb2c77320cd9b5b76483e3464c0f1cff))
|
|
63
|
+
* **validate:** add script to validate resources ([246fb5d](https://github.com/identification-resources/formica/commit/246fb5dd8da03e736884850874cc51a11be9985c))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
|
|
@@ -62,6 +62,13 @@ function sortObject(object) {
|
|
|
62
62
|
return sorted;
|
|
63
63
|
}
|
|
64
64
|
/* eslint-enable @typescript-eslint/no-explicit-any */
|
|
65
|
+
function addTaxon(gbifIndex, gbifId, taxon) {
|
|
66
|
+
if (!(gbifId in gbifIndex)) {
|
|
67
|
+
gbifIndex[gbifId] = [];
|
|
68
|
+
}
|
|
69
|
+
gbifIndex[gbifId].push(taxon[0]);
|
|
70
|
+
gbifIndex[gbifId].sort(util_1.numericSort);
|
|
71
|
+
}
|
|
65
72
|
function main(args) {
|
|
66
73
|
return __awaiter(this, void 0, void 0, function () {
|
|
67
74
|
var REPO_ROOT, files, gbifIndex, resourceIndex;
|
|
@@ -106,11 +113,10 @@ function main(args) {
|
|
|
106
113
|
taxon = dwc_1[_i];
|
|
107
114
|
gbifId = taxon[25];
|
|
108
115
|
if (gbifId) {
|
|
109
|
-
|
|
110
|
-
|
|
116
|
+
addTaxon(gbifIndex, gbifId, taxon);
|
|
117
|
+
if (taxon[27] !== taxon[25]) {
|
|
118
|
+
addTaxon(gbifIndex, taxon[27], taxon);
|
|
111
119
|
}
|
|
112
|
-
gbifIndex[gbifId].push(taxon[0]);
|
|
113
|
-
gbifIndex[gbifId].sort(util_1.numericSort);
|
|
114
120
|
}
|
|
115
121
|
amendedResource.taxonCount += 1;
|
|
116
122
|
}
|
|
@@ -80,7 +80,9 @@ var DWC_FIELDS = [
|
|
|
80
80
|
'subgenus',
|
|
81
81
|
'higherClassification',
|
|
82
82
|
'colTaxonID',
|
|
83
|
-
'gbifTaxonID'
|
|
83
|
+
'gbifTaxonID',
|
|
84
|
+
'colAcceptedTaxonID',
|
|
85
|
+
'gbifAcceptedTaxonID'
|
|
84
86
|
];
|
|
85
87
|
var DISPLAY_FIELDS = [
|
|
86
88
|
'scientificNameID',
|
|
@@ -102,16 +104,9 @@ var GBIF_RANKS = [
|
|
|
102
104
|
'subspecies',
|
|
103
105
|
'variety'
|
|
104
106
|
];
|
|
105
|
-
var VALID_COMMON_PREFIXES = [
|
|
106
|
-
'Plantae|Tracheophyta',
|
|
107
|
-
'Fungi',
|
|
108
|
-
'Fungi|Ascomycota',
|
|
109
|
-
'Fungi|Basidiomycota',
|
|
110
|
-
'Fungi|Zygomycota'
|
|
111
|
-
];
|
|
112
107
|
function runGnverifier(names) {
|
|
113
108
|
return new Promise(function (resolve, reject) {
|
|
114
|
-
var proc = (0, child_process_1.spawn)('gnverifier', ['-s', '1,11', '-M']);
|
|
109
|
+
var proc = (0, child_process_1.spawn)('gnverifier', ['-s', '1,11', '-f', 'compact', '-M']);
|
|
115
110
|
var stdout = '';
|
|
116
111
|
proc.stdout.on('data', function (data) { stdout += data; });
|
|
117
112
|
proc.stderr.pipe(process.stdout);
|
|
@@ -241,6 +236,34 @@ var ResourceProcessor = /** @class */ (function () {
|
|
|
241
236
|
});
|
|
242
237
|
});
|
|
243
238
|
};
|
|
239
|
+
ResourceProcessor.prototype.runMappingsUpdate = function () {
|
|
240
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
241
|
+
var input, ids, _i, ids_2, id;
|
|
242
|
+
return __generator(this, function (_a) {
|
|
243
|
+
switch (_a.label) {
|
|
244
|
+
case 0: return [4 /*yield*/, fs_1.promises.readdir(this.DIR_TXT)];
|
|
245
|
+
case 1:
|
|
246
|
+
input = _a.sent();
|
|
247
|
+
ids = input
|
|
248
|
+
.map(function (file) { return path.basename(file, '.txt'); })
|
|
249
|
+
.sort(function (a, b) { return parseInt(a.slice(1)) - parseInt(b.slice(1)); });
|
|
250
|
+
_i = 0, ids_2 = ids;
|
|
251
|
+
_a.label = 2;
|
|
252
|
+
case 2:
|
|
253
|
+
if (!(_i < ids_2.length)) return [3 /*break*/, 5];
|
|
254
|
+
id = ids_2[_i];
|
|
255
|
+
return [4 /*yield*/, this.processWork(id, true)];
|
|
256
|
+
case 3:
|
|
257
|
+
_a.sent();
|
|
258
|
+
_a.label = 4;
|
|
259
|
+
case 4:
|
|
260
|
+
_i++;
|
|
261
|
+
return [3 /*break*/, 2];
|
|
262
|
+
case 5: return [2 /*return*/];
|
|
263
|
+
}
|
|
264
|
+
});
|
|
265
|
+
});
|
|
266
|
+
};
|
|
244
267
|
ResourceProcessor.prototype.processWork = function (id, update) {
|
|
245
268
|
return __awaiter(this, void 0, void 0, function () {
|
|
246
269
|
var resources;
|
|
@@ -271,79 +294,61 @@ var ResourceProcessor = /** @class */ (function () {
|
|
|
271
294
|
};
|
|
272
295
|
ResourceProcessor.prototype.processResources = function (id, update) {
|
|
273
296
|
return __awaiter(this, void 0, void 0, function () {
|
|
274
|
-
var resources, amendedResources, _i, resources_1, resource,
|
|
275
|
-
return __generator(this, function (
|
|
276
|
-
switch (
|
|
297
|
+
var resources, amendedResources, _i, resources_1, resource, results, skip, correct, choice, _a, reason;
|
|
298
|
+
return __generator(this, function (_b) {
|
|
299
|
+
switch (_b.label) {
|
|
277
300
|
case 0: return [4 /*yield*/, this.processResourceText(id, update)];
|
|
278
301
|
case 1:
|
|
279
|
-
resources =
|
|
302
|
+
resources = _b.sent();
|
|
280
303
|
amendedResources = [];
|
|
281
304
|
_i = 0, resources_1 = resources;
|
|
282
|
-
|
|
305
|
+
_b.label = 2;
|
|
283
306
|
case 2:
|
|
284
|
-
if (!(_i < resources_1.length)) return [3 /*break*/,
|
|
307
|
+
if (!(_i < resources_1.length)) return [3 /*break*/, 11];
|
|
285
308
|
resource = resources_1[_i];
|
|
286
309
|
return [4 /*yield*/, this.processResourceDwc(resource)];
|
|
287
310
|
case 3:
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
_c = [];
|
|
291
|
-
for (_d in _b)
|
|
292
|
-
_c.push(_d);
|
|
293
|
-
_e = 0;
|
|
294
|
-
_g.label = 4;
|
|
311
|
+
results = _b.sent();
|
|
312
|
+
return [4 /*yield*/, this.shouldBeSkipped(resource.id)];
|
|
295
313
|
case 4:
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
if (!(_d in _b)) return [3 /*break*/, 6];
|
|
299
|
-
source = _d;
|
|
300
|
-
return [4 /*yield*/, this.checkPrefix(resource, classifications, source)];
|
|
301
|
-
case 5:
|
|
302
|
-
_g.sent();
|
|
303
|
-
_g.label = 6;
|
|
304
|
-
case 6:
|
|
305
|
-
_e++;
|
|
306
|
-
return [3 /*break*/, 4];
|
|
307
|
-
case 7: return [4 /*yield*/, this.shouldBeSkipped(resource.id)];
|
|
308
|
-
case 8:
|
|
309
|
-
skip = _g.sent();
|
|
310
|
-
if (!!skip) return [3 /*break*/, 13];
|
|
314
|
+
skip = _b.sent();
|
|
315
|
+
if (!!skip) return [3 /*break*/, 9];
|
|
311
316
|
correct = this.checkResults(results);
|
|
312
|
-
if (!!correct) return [3 /*break*/,
|
|
317
|
+
if (!!correct) return [3 /*break*/, 9];
|
|
313
318
|
return [4 /*yield*/, (0, util_1.promptForAnswers)("".concat(resource.workId, ": problems found in ").concat(resource.id, ". Skip or retry (s/r)? "), ['s', 'S', 'r', 'R'])];
|
|
314
|
-
case
|
|
315
|
-
choice =
|
|
316
|
-
|
|
317
|
-
switch (
|
|
318
|
-
case 's': return [3 /*break*/,
|
|
319
|
-
case 'S': return [3 /*break*/,
|
|
320
|
-
case 'r': return [3 /*break*/,
|
|
321
|
-
case 'R': return [3 /*break*/,
|
|
319
|
+
case 5:
|
|
320
|
+
choice = _b.sent();
|
|
321
|
+
_a = choice;
|
|
322
|
+
switch (_a) {
|
|
323
|
+
case 's': return [3 /*break*/, 6];
|
|
324
|
+
case 'S': return [3 /*break*/, 6];
|
|
325
|
+
case 'r': return [3 /*break*/, 8];
|
|
326
|
+
case 'R': return [3 /*break*/, 8];
|
|
322
327
|
}
|
|
323
|
-
return [3 /*break*/,
|
|
324
|
-
case
|
|
325
|
-
case
|
|
326
|
-
reason =
|
|
328
|
+
return [3 /*break*/, 9];
|
|
329
|
+
case 6: return [4 /*yield*/, (0, util_1.prompt)('Reason for skipping? ')];
|
|
330
|
+
case 7:
|
|
331
|
+
reason = _b.sent();
|
|
327
332
|
fs_1.promises.appendFile(this.FILE_PROBLEMS, index_1.csv.formatCsv([[
|
|
328
333
|
resource.workId,
|
|
329
334
|
resource.id,
|
|
330
335
|
reason
|
|
331
336
|
]]));
|
|
332
337
|
console.log("".concat(resource.workId, ": skipping ").concat(resource.id));
|
|
333
|
-
return [3 /*break*/,
|
|
334
|
-
case
|
|
338
|
+
return [3 /*break*/, 9];
|
|
339
|
+
case 8:
|
|
335
340
|
{
|
|
336
341
|
console.log("".concat(resource.workId, ": retrying ").concat(resource.id));
|
|
337
342
|
return [2 /*return*/, this.processResources(id, update)];
|
|
338
343
|
}
|
|
339
|
-
|
|
340
|
-
case
|
|
344
|
+
_b.label = 9;
|
|
345
|
+
case 9:
|
|
341
346
|
amendedResources.push(results);
|
|
342
|
-
|
|
343
|
-
case
|
|
347
|
+
_b.label = 10;
|
|
348
|
+
case 10:
|
|
344
349
|
_i++;
|
|
345
350
|
return [3 /*break*/, 2];
|
|
346
|
-
case
|
|
351
|
+
case 11: return [2 /*return*/, amendedResources];
|
|
347
352
|
}
|
|
348
353
|
});
|
|
349
354
|
});
|
|
@@ -415,119 +420,174 @@ var ResourceProcessor = /** @class */ (function () {
|
|
|
415
420
|
};
|
|
416
421
|
ResourceProcessor.prototype.processResourceDwc = function (resource) {
|
|
417
422
|
return __awaiter(this, void 0, void 0, function () {
|
|
418
|
-
var
|
|
419
|
-
return __generator(this, function (
|
|
420
|
-
switch (
|
|
423
|
+
var filteredResults, taxonNames, names, id, name_1, result, _i, _a, results, _b, name_2, matches, _c, matches_1, match, source, currentRank, _d, _e, loirId, taxon, _f, amendResource, groupNameMatches, groupedNameMatches, amendedResource, _g, _h, _j, _k, source, matches;
|
|
424
|
+
return __generator(this, function (_l) {
|
|
425
|
+
switch (_l.label) {
|
|
421
426
|
case 0:
|
|
422
427
|
console.log("".concat(resource.workId, ": matching ").concat(resource.id));
|
|
423
|
-
|
|
424
|
-
|
|
428
|
+
filteredResults = {};
|
|
429
|
+
taxonNames = {};
|
|
430
|
+
names = new Set();
|
|
425
431
|
for (id in resource.taxa) {
|
|
426
432
|
name_1 = resource.taxa[id].scientificName;
|
|
427
|
-
if (!
|
|
428
|
-
|
|
433
|
+
if (!taxonNames[name_1]) {
|
|
434
|
+
taxonNames[name_1] = [];
|
|
429
435
|
}
|
|
430
|
-
|
|
431
|
-
names.
|
|
436
|
+
taxonNames[name_1].push(id);
|
|
437
|
+
names.add(name_1);
|
|
438
|
+
filteredResults[id] = [];
|
|
432
439
|
}
|
|
433
|
-
return [4 /*yield*/, runGnverifier(names.join('\n'))];
|
|
440
|
+
return [4 /*yield*/, runGnverifier(Array.from(names).join('\n'))];
|
|
434
441
|
case 1:
|
|
435
|
-
result =
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
if (
|
|
447
|
-
|
|
448
|
-
|
|
442
|
+
result = _l.sent();
|
|
443
|
+
for (_i = 0, _a = result.trim().split('\n'); _i < _a.length; _i++) {
|
|
444
|
+
results = _a[_i];
|
|
445
|
+
_b = JSON.parse(results), name_2 = _b.name, matches = _b.results;
|
|
446
|
+
if (!matches) {
|
|
447
|
+
continue;
|
|
448
|
+
}
|
|
449
|
+
for (_c = 0, matches_1 = matches; _c < matches_1.length; _c++) {
|
|
450
|
+
match = matches_1[_c];
|
|
451
|
+
source = match.dataSourceId;
|
|
452
|
+
currentRank = match.classificationRanks.split('|').pop();
|
|
453
|
+
if (match.scoreDetails.cardinalityScore === 0) {
|
|
454
|
+
// Rank mismatch
|
|
455
|
+
continue;
|
|
456
|
+
}
|
|
457
|
+
else if (source === 11 && currentRank === 'species' && match.classificationPath.endsWith(' spec')) {
|
|
458
|
+
// GBIF species like "Nomada spec"
|
|
459
|
+
continue;
|
|
449
460
|
}
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
461
|
+
for (_d = 0, _e = taxonNames[name_2]; _d < _e.length; _d++) {
|
|
462
|
+
loirId = _e[_d];
|
|
463
|
+
taxon = resource.taxa[loirId];
|
|
464
|
+
if (source === 11 && !GBIF_RANKS.includes(taxon.taxonRank)) {
|
|
465
|
+
// Exclude GBIF matches for ranks that are not in GBIF
|
|
466
|
+
continue;
|
|
467
|
+
}
|
|
468
|
+
else if (source === 11 && !match.isSynonym && currentRank !== taxon.taxonRank) {
|
|
469
|
+
// Exclude matches with rank mismatches (only possible
|
|
470
|
+
// for non-synonyms).
|
|
471
|
+
continue;
|
|
472
|
+
}
|
|
473
|
+
if (!filteredResults[loirId]) {
|
|
474
|
+
filteredResults[loirId] = [];
|
|
475
|
+
}
|
|
476
|
+
filteredResults[loirId].push({
|
|
477
|
+
source: source,
|
|
478
|
+
id: match.recordId,
|
|
479
|
+
currentId: match.currentRecordId,
|
|
480
|
+
classificationPath: match.classificationPath.split('|')
|
|
481
|
+
});
|
|
453
482
|
}
|
|
454
483
|
}
|
|
455
484
|
}
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
485
|
+
return [4 /*yield*/, Promise.resolve().then(function () { return require('../index'); })];
|
|
486
|
+
case 2:
|
|
487
|
+
_f = (_l.sent()).taxonNames, amendResource = _f.amendResource, groupNameMatches = _f.groupNameMatches;
|
|
488
|
+
groupedNameMatches = groupNameMatches(filteredResults);
|
|
489
|
+
amendedResource = __assign(__assign({}, resource), { taxa: __assign({}, resource.taxa) });
|
|
490
|
+
_g = groupedNameMatches;
|
|
491
|
+
_h = [];
|
|
492
|
+
for (_j in _g)
|
|
493
|
+
_h.push(_j);
|
|
494
|
+
_k = 0;
|
|
495
|
+
_l.label = 3;
|
|
496
|
+
case 3:
|
|
497
|
+
if (!(_k < _h.length)) return [3 /*break*/, 6];
|
|
498
|
+
_j = _h[_k];
|
|
499
|
+
if (!(_j in _g)) return [3 /*break*/, 5];
|
|
500
|
+
source = _j;
|
|
501
|
+
return [4 /*yield*/, this.selectPrefixes(resource, groupedNameMatches, source)];
|
|
502
|
+
case 4:
|
|
503
|
+
matches = _l.sent();
|
|
504
|
+
amendResource(amendedResource, source, matches);
|
|
505
|
+
_l.label = 5;
|
|
506
|
+
case 5:
|
|
507
|
+
_k++;
|
|
508
|
+
return [3 /*break*/, 3];
|
|
509
|
+
case 6: return [2 /*return*/, amendedResource];
|
|
461
510
|
}
|
|
462
511
|
});
|
|
463
512
|
});
|
|
464
513
|
};
|
|
465
|
-
ResourceProcessor.prototype.
|
|
514
|
+
ResourceProcessor.prototype.selectPrefixes = function (resource, groupedNameMatches, source) {
|
|
466
515
|
return __awaiter(this, void 0, void 0, function () {
|
|
467
|
-
var
|
|
516
|
+
var prefixes, mappedTaxa, _i, prefixes_1, prefix, taxon, missedTaxonCount, choice, i, prefix, taxa, taxonIds, j, taxonId, taxon, match, matches, _a, _b, i, prefix, taxa, id;
|
|
468
517
|
return __generator(this, function (_c) {
|
|
469
518
|
switch (_c.label) {
|
|
470
519
|
case 0:
|
|
471
|
-
|
|
472
|
-
if (
|
|
473
|
-
return [2 /*return
|
|
520
|
+
prefixes = Object.keys(groupedNameMatches[source]);
|
|
521
|
+
if (prefixes.length === 0) {
|
|
522
|
+
return [2 /*return*/, {}];
|
|
474
523
|
}
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
if (
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
524
|
+
else if (prefixes.length === 1) {
|
|
525
|
+
return [2 /*return*/, groupedNameMatches[source][prefixes[0]]];
|
|
526
|
+
}
|
|
527
|
+
mappedTaxa = {};
|
|
528
|
+
for (_i = 0, prefixes_1 = prefixes; _i < prefixes_1.length; _i++) {
|
|
529
|
+
prefix = prefixes_1[_i];
|
|
530
|
+
for (taxon in groupedNameMatches[source][prefix]) {
|
|
531
|
+
mappedTaxa[taxon] = true;
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
missedTaxonCount = Object.keys(mappedTaxa).length - Object.keys(groupedNameMatches[source][prefixes[0]]).length;
|
|
535
|
+
if (missedTaxonCount === 0) {
|
|
536
|
+
// Multiple prefixes but the first one maps all taxa (not counting that are unmapped in all prefixes)
|
|
537
|
+
return [2 /*return*/, groupedNameMatches[source][prefixes[0]]];
|
|
538
|
+
}
|
|
539
|
+
console.error("".concat(resource.workId, ": source ").concat(source, " results in multiple prefixes"));
|
|
540
|
+
if (!(missedTaxonCount <= 5)) return [3 /*break*/, 1];
|
|
541
|
+
console.error(" Most common prefix misses ".concat(missedTaxonCount, " taxa: automatically selecting most common prefix..."));
|
|
542
|
+
choice = '1';
|
|
490
543
|
return [3 /*break*/, 6];
|
|
491
|
-
case
|
|
492
|
-
if (!
|
|
493
|
-
|
|
544
|
+
case 1:
|
|
545
|
+
if (!(source === '1')) return [3 /*break*/, 2];
|
|
546
|
+
console.error(" Catalogue of Life: automatically selecting most common prefix...");
|
|
547
|
+
choice = '1';
|
|
494
548
|
return [3 /*break*/, 6];
|
|
549
|
+
case 2:
|
|
550
|
+
for (i = 0; i < prefixes.length; i++) {
|
|
551
|
+
prefix = prefixes[i];
|
|
552
|
+
taxa = groupedNameMatches[source][prefix];
|
|
553
|
+
taxonIds = Object.keys(taxa);
|
|
554
|
+
console.error(" [".concat(i + 1, "] ").concat(prefix, " (").concat(taxonIds.length, " taxa)"));
|
|
555
|
+
for (j = 0; j < Math.min(9, taxonIds.length); j++) {
|
|
556
|
+
taxonId = taxonIds[j];
|
|
557
|
+
taxon = resource.taxa[taxonId];
|
|
558
|
+
match = taxa[taxonId];
|
|
559
|
+
console.error(" taxon: ".concat(taxonId, " \"").concat(taxon.scientificName, "\" - ").concat(match.classificationPath.join('|')));
|
|
560
|
+
}
|
|
561
|
+
if (taxonIds.length > 9) {
|
|
562
|
+
console.error(" ...");
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
_c.label = 3;
|
|
566
|
+
case 3: return [4 /*yield*/, (0, util_1.prompt)(" Select prefixes (1-".concat(prefixes.length, ")? "))];
|
|
495
567
|
case 4:
|
|
496
|
-
console.log("".concat(resource.workId, ": source ").concat(source, " results in short prefix \"").concat(prefix.slice(0, i).join('|'), "\" (").concat(i, " taxa)"));
|
|
497
|
-
console.log(" taxon: ".concat(taxon.scientificNameID, " \"").concat(taxon.scientificName, "\""));
|
|
498
|
-
console.log(" class: ".concat(parts.join('|')));
|
|
499
|
-
console.log(" prefx: ".concat(prefix.join('|')));
|
|
500
|
-
return [4 /*yield*/, (0, util_1.promptForAnswers)(" Keep or delete (k/d)? ", ['k', 'K', 'd', 'D'])];
|
|
501
|
-
case 5:
|
|
502
568
|
choice = _c.sent();
|
|
569
|
+
_c.label = 5;
|
|
570
|
+
case 5:
|
|
571
|
+
if (!/^(|\d+(,\d+)*)$/.test(choice)) return [3 /*break*/, 3];
|
|
503
572
|
_c.label = 6;
|
|
504
573
|
case 6:
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
else if (source === '11') {
|
|
518
|
-
delete taxon.gbifTaxonID;
|
|
574
|
+
console.error(" Applying selection...");
|
|
575
|
+
if (choice === '') {
|
|
576
|
+
return [2 /*return*/, {}];
|
|
577
|
+
}
|
|
578
|
+
matches = {};
|
|
579
|
+
for (_a = 0, _b = choice.split(','); _a < _b.length; _a++) {
|
|
580
|
+
i = _b[_a];
|
|
581
|
+
prefix = prefixes[parseInt(i) - 1];
|
|
582
|
+
taxa = groupedNameMatches[source][prefix];
|
|
583
|
+
for (id in taxa) {
|
|
584
|
+
if (id in matches) {
|
|
585
|
+
continue;
|
|
519
586
|
}
|
|
520
|
-
|
|
587
|
+
matches[id] = taxa[id];
|
|
521
588
|
}
|
|
522
589
|
}
|
|
523
|
-
return [
|
|
524
|
-
case 7:
|
|
525
|
-
i++;
|
|
526
|
-
return [3 /*break*/, 2];
|
|
527
|
-
case 8:
|
|
528
|
-
_i++;
|
|
529
|
-
return [3 /*break*/, 1];
|
|
530
|
-
case 9: return [2 /*return*/];
|
|
590
|
+
return [2 /*return*/, matches];
|
|
531
591
|
}
|
|
532
592
|
});
|
|
533
593
|
});
|
|
@@ -578,6 +638,9 @@ function main() {
|
|
|
578
638
|
update: {
|
|
579
639
|
type: 'boolean',
|
|
580
640
|
short: 'u'
|
|
641
|
+
},
|
|
642
|
+
'update-mappings': {
|
|
643
|
+
type: 'boolean'
|
|
581
644
|
}
|
|
582
645
|
},
|
|
583
646
|
allowPositionals: true
|
|
@@ -586,7 +649,16 @@ function main() {
|
|
|
586
649
|
process.on('exit', function () {
|
|
587
650
|
process.stdout.write('\n');
|
|
588
651
|
});
|
|
589
|
-
var task
|
|
652
|
+
var task;
|
|
653
|
+
if (args.values.update) {
|
|
654
|
+
task = processor.runUpdate();
|
|
655
|
+
}
|
|
656
|
+
else if (args.values['update-mappings']) {
|
|
657
|
+
task = processor.runMappingsUpdate();
|
|
658
|
+
}
|
|
659
|
+
else {
|
|
660
|
+
task = processor.run();
|
|
661
|
+
}
|
|
590
662
|
task.catch(function (error) {
|
|
591
663
|
console.error(error);
|
|
592
664
|
process.exit(1);
|
|
File without changes
|
package/lib/index.d.ts
CHANGED
package/lib/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.csv = exports.resources = exports.catalog = void 0;
|
|
3
|
+
exports.csv = exports.taxonNames = exports.resources = exports.catalog = void 0;
|
|
4
4
|
exports.catalog = require("./catalog/index");
|
|
5
5
|
exports.resources = require("./resources/index");
|
|
6
|
+
exports.taxonNames = require("./taxon-names/index");
|
|
6
7
|
exports.csv = require("./csv");
|