@larsgw/formica 0.2.1 → 0.3.0

package/CHANGELOG.md

@@ -1,3 +1,16 @@
+# [0.3.0](https://github.com/identification-resources/formica/compare/v0.2.1...v0.3.0) (2023-08-20)
+
+
+### Features
+
+* **resources:** add accepted ids to dwc and index ([1b5aba8](https://github.com/identification-resources/formica/commit/1b5aba8ff07bba32fc1bbf49927fef7a880eb35e))
+* **resources:** allow updates of just mappings ([0ea62c1](https://github.com/identification-resources/formica/commit/0ea62c143a49129709960de123c0037973356487))
+* **resources:** improve prefix selection heuristics ([7c2e0c9](https://github.com/identification-resources/formica/commit/7c2e0c99d9de7490314f971e8b4e9e5e190a81c2))
+* **resources:** improve taxon name matching ([78ff480](https://github.com/identification-resources/formica/commit/78ff480485ca42bd1d1f2893f230e61db6cb6be8)), closes [#2](https://github.com/identification-resources/formica/issues/2)
+* **resources:** test for rank mismatch ([0e69b7f](https://github.com/identification-resources/formica/commit/0e69b7f0a6654fa5255e797be7516cd06edb4df3)), closes [#2](https://github.com/identification-resources/formica/issues/2)
+
+
+
 ## [0.2.1](https://github.com/identification-resources/formica/compare/v0.2.0...v0.2.1) (2023-08-10)
 
 

package/lib/bin/process-resources-index.js

@@ -62,6 +62,13 @@ function sortObject(object) {
     return sorted;
 }
 /* eslint-enable @typescript-eslint/no-explicit-any */
+function addTaxon(gbifIndex, gbifId, taxon) {
+    if (!(gbifId in gbifIndex)) {
+        gbifIndex[gbifId] = [];
+    }
+    gbifIndex[gbifId].push(taxon[0]);
+    gbifIndex[gbifId].sort(util_1.numericSort);
+}
 function main(args) {
     return __awaiter(this, void 0, void 0, function () {
         var REPO_ROOT, files, gbifIndex, resourceIndex;
@@ -106,11 +113,10 @@ function main(args) {
                                                                         taxon = dwc_1[_i];
                                                                         gbifId = taxon[25];
                                                                         if (gbifId) {
-                                                                            if (!(gbifId in gbifIndex)) {
-                                                                                gbifIndex[gbifId] = [];
+                                                                            addTaxon(gbifIndex, gbifId, taxon);
+                                                                            if (taxon[27] !== taxon[25]) {
+                                                                                addTaxon(gbifIndex, taxon[27], taxon);
                                                                             }
-                                                                            gbifIndex[gbifId].push(taxon[0]);
-                                                                            gbifIndex[gbifId].sort(util_1.numericSort);
                                                                         }
                                                                         amendedResource.taxonCount += 1;
                                                                     }

package/lib/bin/process-resources.js

@@ -80,7 +80,9 @@ var DWC_FIELDS = [
     'subgenus',
     'higherClassification',
     'colTaxonID',
-    'gbifTaxonID'
+    'gbifTaxonID',
+    'colAcceptedTaxonID',
+    'gbifAcceptedTaxonID'
 ];
 var DISPLAY_FIELDS = [
     'scientificNameID',
@@ -102,16 +104,9 @@ var GBIF_RANKS = [
     'subspecies',
     'variety'
 ];
-var VALID_COMMON_PREFIXES = [
-    'Plantae|Tracheophyta',
-    'Fungi',
-    'Fungi|Ascomycota',
-    'Fungi|Basidiomycota',
-    'Fungi|Zygomycota'
-];
 function runGnverifier(names) {
     return new Promise(function (resolve, reject) {
-        var proc = (0, child_process_1.spawn)('gnverifier', ['-s', '1,11', '-M']);
+        var proc = (0, child_process_1.spawn)('gnverifier', ['-s', '1,11', '-f', 'compact', '-M']);
         var stdout = '';
         proc.stdout.on('data', function (data) { stdout += data; });
         proc.stderr.pipe(process.stdout);
@@ -241,6 +236,34 @@ var ResourceProcessor = /** @class */ (function () {
             });
         });
     };
+    ResourceProcessor.prototype.runMappingsUpdate = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var input, ids, _i, ids_2, id;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, fs_1.promises.readdir(this.DIR_TXT)];
+                    case 1:
+                        input = _a.sent();
+                        ids = input
+                            .map(function (file) { return path.basename(file, '.txt'); })
+                            .sort(function (a, b) { return parseInt(a.slice(1)) - parseInt(b.slice(1)); });
+                        _i = 0, ids_2 = ids;
+                        _a.label = 2;
+                    case 2:
+                        if (!(_i < ids_2.length)) return [3 /*break*/, 5];
+                        id = ids_2[_i];
+                        return [4 /*yield*/, this.processWork(id, true)];
+                    case 3:
+                        _a.sent();
+                        _a.label = 4;
+                    case 4:
+                        _i++;
+                        return [3 /*break*/, 2];
+                    case 5: return [2 /*return*/];
+                }
+            });
+        });
+    };
     ResourceProcessor.prototype.processWork = function (id, update) {
         return __awaiter(this, void 0, void 0, function () {
             var resources;
@@ -271,79 +294,61 @@ var ResourceProcessor = /** @class */ (function () {
     };
     ResourceProcessor.prototype.processResources = function (id, update) {
         return __awaiter(this, void 0, void 0, function () {
-            var resources, amendedResources, _i, resources_1, resource, _a, results, classifications, _b, _c, _d, _e, source, skip, correct, choice, _f, reason;
-            return __generator(this, function (_g) {
-                switch (_g.label) {
+            var resources, amendedResources, _i, resources_1, resource, results, skip, correct, choice, _a, reason;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
                     case 0: return [4 /*yield*/, this.processResourceText(id, update)];
                     case 1:
-                        resources = _g.sent();
+                        resources = _b.sent();
                         amendedResources = [];
                         _i = 0, resources_1 = resources;
-                        _g.label = 2;
+                        _b.label = 2;
                     case 2:
-                        if (!(_i < resources_1.length)) return [3 /*break*/, 15];
+                        if (!(_i < resources_1.length)) return [3 /*break*/, 11];
                         resource = resources_1[_i];
                         return [4 /*yield*/, this.processResourceDwc(resource)];
                     case 3:
-                        _a = _g.sent(), results = _a[0], classifications = _a[1];
-                        _b = classifications;
-                        _c = [];
-                        for (_d in _b)
-                            _c.push(_d);
-                        _e = 0;
-                        _g.label = 4;
+                        results = _b.sent();
+                        return [4 /*yield*/, this.shouldBeSkipped(resource.id)];
                     case 4:
-                        if (!(_e < _c.length)) return [3 /*break*/, 7];
-                        _d = _c[_e];
-                        if (!(_d in _b)) return [3 /*break*/, 6];
-                        source = _d;
-                        return [4 /*yield*/, this.checkPrefix(resource, classifications, source)];
-                    case 5:
-                        _g.sent();
-                        _g.label = 6;
-                    case 6:
-                        _e++;
-                        return [3 /*break*/, 4];
-                    case 7: return [4 /*yield*/, this.shouldBeSkipped(resource.id)];
-                    case 8:
-                        skip = _g.sent();
-                        if (!!skip) return [3 /*break*/, 13];
+                        skip = _b.sent();
+                        if (!!skip) return [3 /*break*/, 9];
                         correct = this.checkResults(results);
-                        if (!!correct) return [3 /*break*/, 13];
+                        if (!!correct) return [3 /*break*/, 9];
                         return [4 /*yield*/, (0, util_1.promptForAnswers)("".concat(resource.workId, ": problems found in ").concat(resource.id, ". Skip or retry (s/r)? "), ['s', 'S', 'r', 'R'])];
-                    case 9:
-                        choice = _g.sent();
-                        _f = choice;
-                        switch (_f) {
-                            case 's': return [3 /*break*/, 10];
-                            case 'S': return [3 /*break*/, 10];
-                            case 'r': return [3 /*break*/, 12];
-                            case 'R': return [3 /*break*/, 12];
+                    case 5:
+                        choice = _b.sent();
+                        _a = choice;
+                        switch (_a) {
+                            case 's': return [3 /*break*/, 6];
+                            case 'S': return [3 /*break*/, 6];
+                            case 'r': return [3 /*break*/, 8];
+                            case 'R': return [3 /*break*/, 8];
                         }
-                        return [3 /*break*/, 13];
-                    case 10: return [4 /*yield*/, (0, util_1.prompt)('Reason for skipping? ')];
-                    case 11:
-                        reason = _g.sent();
+                        return [3 /*break*/, 9];
+                    case 6: return [4 /*yield*/, (0, util_1.prompt)('Reason for skipping? ')];
+                    case 7:
+                        reason = _b.sent();
                         fs_1.promises.appendFile(this.FILE_PROBLEMS, index_1.csv.formatCsv([[
                                 resource.workId,
                                 resource.id,
                                 reason
                             ]]));
                         console.log("".concat(resource.workId, ": skipping ").concat(resource.id));
-                        return [3 /*break*/, 13];
-                    case 12:
+                        return [3 /*break*/, 9];
+                    case 8:
                         {
                             console.log("".concat(resource.workId, ": retrying ").concat(resource.id));
                             return [2 /*return*/, this.processResources(id, update)];
                         }
-                        _g.label = 13;
-                    case 13:
+                        _b.label = 9;
+                    case 9:
                         amendedResources.push(results);
-                        _g.label = 14;
-                    case 14:
+                        _b.label = 10;
+                    case 10:
                         _i++;
                         return [3 /*break*/, 2];
-                    case 15: return [2 /*return*/, amendedResources];
+                    case 11: return [2 /*return*/, amendedResources];
                 }
             });
         });
@@ -415,119 +420,174 @@ var ResourceProcessor = /** @class */ (function () {
     };
     ResourceProcessor.prototype.processResourceDwc = function (resource) {
         return __awaiter(this, void 0, void 0, function () {
-            var taxa, names, id, name_1, result, classifications, _a, header, matches, _i, matches_1, match, name_2, source, id, classification, loirId, taxon, results;
-            return __generator(this, function (_b) {
-                switch (_b.label) {
+            var filteredResults, taxonNames, names, id, name_1, result, _i, _a, results, _b, name_2, matches, _c, matches_1, match, source, currentRank, _d, _e, loirId, taxon, _f, amendResource, groupNameMatches, groupedNameMatches, amendedResource, _g, _h, _j, _k, source, matches;
+            return __generator(this, function (_l) {
+                switch (_l.label) {
                     case 0:
                         console.log("".concat(resource.workId, ": matching ").concat(resource.id));
-                        taxa = {};
-                        names = [];
+                        filteredResults = {};
+                        taxonNames = {};
+                        names = new Set();
                         for (id in resource.taxa) {
                             name_1 = resource.taxa[id].scientificName;
-                            if (!taxa[name_1]) {
-                                taxa[name_1] = {};
+                            if (!taxonNames[name_1]) {
+                                taxonNames[name_1] = [];
                             }
-                            taxa[name_1][id] = __assign({}, resource.taxa[id]);
-                            names.push(name_1);
+                            taxonNames[name_1].push(id);
+                            names.add(name_1);
+                            filteredResults[id] = [];
                         }
-                        return [4 /*yield*/, runGnverifier(names.join('\n'))];
+                        return [4 /*yield*/, runGnverifier(Array.from(names).join('\n'))];
                     case 1:
-                        result = _b.sent();
-                        classifications = { '1': [], '11': [] };
-                        _a = index_1.csv.parseCsv(result), header = _a[0], matches = _a.slice(1);
-                        for (_i = 0, matches_1 = matches; _i < matches_1.length; _i++) {
-                            match = matches_1[_i];
-                            name_2 = match[header.indexOf('ScientificName')];
-                            source = match[header.indexOf('DataSourceId')];
-                            id = match[header.indexOf('TaxonId')];
-                            classification = match[header.indexOf('ClassificationPath')];
-                            for (loirId in taxa[name_2]) {
-                                taxon = taxa[name_2][loirId];
-                                if (source === '1' && !taxon.colTaxonID) {
-                                    taxon.colTaxonID = id;
-                                    classifications[source].push([taxon, classification]);
+                        result = _l.sent();
+                        for (_i = 0, _a = result.trim().split('\n'); _i < _a.length; _i++) {
+                            results = _a[_i];
+                            _b = JSON.parse(results), name_2 = _b.name, matches = _b.results;
+                            if (!matches) {
+                                continue;
+                            }
+                            for (_c = 0, matches_1 = matches; _c < matches_1.length; _c++) {
+                                match = matches_1[_c];
+                                source = match.dataSourceId;
+                                currentRank = match.classificationRanks.split('|').pop();
+                                if (match.scoreDetails.cardinalityScore === 0) {
+                                    // Rank mismatch
+                                    continue;
+                                }
+                                else if (source === 11 && currentRank === 'species' && match.classificationPath.endsWith(' spec')) {
+                                    // GBIF species like "Nomada spec"
+                                    continue;
                                 }
-                                if (source === '11' && GBIF_RANKS.includes(taxon.taxonRank) && !taxon.gbifTaxonID) {
-                                    taxon.gbifTaxonID = id;
-                                    classifications[source].push([taxon, classification]);
+                                for (_d = 0, _e = taxonNames[name_2]; _d < _e.length; _d++) {
+                                    loirId = _e[_d];
+                                    taxon = resource.taxa[loirId];
+                                    if (source === 11 && !GBIF_RANKS.includes(taxon.taxonRank)) {
+                                        // Exclude GBIF matches for ranks that are not in GBIF
+                                        continue;
+                                    }
+                                    else if (source === 11 && !match.isSynonym && currentRank !== taxon.taxonRank) {
+                                        // Exclude matches with rank mismatches (only possible
+                                        // for non-synonyms).
+                                        continue;
+                                    }
+                                    if (!filteredResults[loirId]) {
+                                        filteredResults[loirId] = [];
+                                    }
+                                    filteredResults[loirId].push({
+                                        source: source,
+                                        id: match.recordId,
+                                        currentId: match.currentRecordId,
+                                        classificationPath: match.classificationPath.split('|')
+                                    });
                                 }
                             }
                         }
-                        results = __assign(__assign({}, resource), { taxa: Object.fromEntries(Object.values(resource.taxa).map(function (taxon) { return [
-                                taxon.scientificNameID,
-                                taxa[taxon.scientificName][taxon.scientificNameID]
-                            ]; })) });
-                        return [2 /*return*/, [results, classifications]];
+                        return [4 /*yield*/, Promise.resolve().then(function () { return require('../index'); })];
+                    case 2:
+                        _f = (_l.sent()).taxonNames, amendResource = _f.amendResource, groupNameMatches = _f.groupNameMatches;
+                        groupedNameMatches = groupNameMatches(filteredResults);
+                        amendedResource = __assign(__assign({}, resource), { taxa: __assign({}, resource.taxa) });
+                        _g = groupedNameMatches;
+                        _h = [];
+                        for (_j in _g)
+                            _h.push(_j);
+                        _k = 0;
+                        _l.label = 3;
+                    case 3:
+                        if (!(_k < _h.length)) return [3 /*break*/, 6];
+                        _j = _h[_k];
+                        if (!(_j in _g)) return [3 /*break*/, 5];
+                        source = _j;
+                        return [4 /*yield*/, this.selectPrefixes(resource, groupedNameMatches, source)];
+                    case 4:
+                        matches = _l.sent();
+                        amendResource(amendedResource, source, matches);
+                        _l.label = 5;
+                    case 5:
+                        _k++;
+                        return [3 /*break*/, 3];
+                    case 6: return [2 /*return*/, amendedResource];
                 }
             });
         });
     };
-    ResourceProcessor.prototype.checkPrefix = function (resource, classifications, source) {
+    ResourceProcessor.prototype.selectPrefixes = function (resource, groupedNameMatches, source) {
         return __awaiter(this, void 0, void 0, function () {
-            var lists, prefix, _i, _a, _b, taxon, list, parts, i, choice;
+            var prefixes, mappedTaxa, _i, prefixes_1, prefix, taxon, missedTaxonCount, choice, i, prefix, taxa, taxonIds, j, taxonId, taxon, match, matches, _a, _b, i, prefix, taxa, id;
             return __generator(this, function (_c) {
                 switch (_c.label) {
                     case 0:
-                        lists = classifications[source];
-                        if (!lists.length) {
-                            return [2 /*return*/];
+                        prefixes = Object.keys(groupedNameMatches[source]);
+                        if (prefixes.length === 0) {
+                            return [2 /*return*/, {}];
                         }
-                        prefix = lists[0][1].split('|');
-                        _i = 0, _a = lists.slice(1);
-                        _c.label = 1;
-                    case 1:
-                        if (!(_i < _a.length)) return [3 /*break*/, 9];
-                        _b = _a[_i], taxon = _b[0], list = _b[1];
-                        parts = list.split('|');
-                        i = 0;
-                        _c.label = 2;
-                    case 2:
-                        if (!(i < parts.length)) return [3 /*break*/, 8];
-                        if (!(parts[i] !== prefix[i] && i < 3)) return [3 /*break*/, 7];
-                        choice = void 0;
-                        if (!(source === '1' || taxon.taxonomicStatus !== 'accepted')) return [3 /*break*/, 3];
-                        choice = 'd';
+                        else if (prefixes.length === 1) {
+                            return [2 /*return*/, groupedNameMatches[source][prefixes[0]]];
+                        }
+                        mappedTaxa = {};
+                        for (_i = 0, prefixes_1 = prefixes; _i < prefixes_1.length; _i++) {
+                            prefix = prefixes_1[_i];
+                            for (taxon in groupedNameMatches[source][prefix]) {
+                                mappedTaxa[taxon] = true;
+                            }
+                        }
+                        missedTaxonCount = Object.keys(mappedTaxa).length - Object.keys(groupedNameMatches[source][prefixes[0]]).length;
+                        if (missedTaxonCount === 0) {
+                            // Multiple prefixes but the first one maps all taxa (not counting that are unmapped in all prefixes)
+                            return [2 /*return*/, groupedNameMatches[source][prefixes[0]]];
+                        }
+                        console.error("".concat(resource.workId, ": source ").concat(source, " results in multiple prefixes"));
+                        if (!(missedTaxonCount <= 5)) return [3 /*break*/, 1];
+                        console.error("  Most common prefix misses ".concat(missedTaxonCount, " taxa: automatically selecting most common prefix..."));
+                        choice = '1';
                         return [3 /*break*/, 6];
-                    case 3:
-                        if (!VALID_COMMON_PREFIXES.includes(prefix.slice(0, i).join('|'))) return [3 /*break*/, 4];
-                        choice = 'k';
+                    case 1:
+                        if (!(source === '1')) return [3 /*break*/, 2];
+                        console.error("  Catalogue of Life: automatically selecting most common prefix...");
+                        choice = '1';
                         return [3 /*break*/, 6];
+                    case 2:
+                        for (i = 0; i < prefixes.length; i++) {
+                            prefix = prefixes[i];
+                            taxa = groupedNameMatches[source][prefix];
+                            taxonIds = Object.keys(taxa);
+                            console.error("  [".concat(i + 1, "] ").concat(prefix, " (").concat(taxonIds.length, " taxa)"));
+                            for (j = 0; j < Math.min(9, taxonIds.length); j++) {
+                                taxonId = taxonIds[j];
+                                taxon = resource.taxa[taxonId];
+                                match = taxa[taxonId];
+                                console.error("      taxon: ".concat(taxonId, " \"").concat(taxon.scientificName, "\" - ").concat(match.classificationPath.join('|')));
+                            }
+                            if (taxonIds.length > 9) {
+                                console.error("      ...");
+                            }
+                        }
+                        _c.label = 3;
+                    case 3: return [4 /*yield*/, (0, util_1.prompt)("  Select prefixes (1-".concat(prefixes.length, ")? "))];
                     case 4:
-                        console.log("".concat(resource.workId, ": source ").concat(source, " results in short prefix \"").concat(prefix.slice(0, i).join('|'), "\" (").concat(i, " taxa)"));
-                        console.log("  taxon: ".concat(taxon.scientificNameID, " \"").concat(taxon.scientificName, "\""));
-                        console.log("  class: ".concat(parts.join('|')));
-                        console.log("  prefx: ".concat(prefix.join('|')));
-                        return [4 /*yield*/, (0, util_1.promptForAnswers)("  Keep or delete (k/d)? ", ['k', 'K', 'd', 'D'])];
-                    case 5:
                         choice = _c.sent();
+                        _c.label = 5;
+                    case 5:
+                        if (!/^(|\d+(,\d+)*)$/.test(choice)) return [3 /*break*/, 3];
                         _c.label = 6;
                     case 6:
-                        switch (choice) {
-                            case 'k':
-                            case 'K': {
-                                console.log("  keeping...");
-                                break;
-                            }
-                            case 'd':
-                            case 'D': {
-                                console.log("  deleting...");
-                                if (source === '1') {
-                                    delete taxon.colTaxonID;
-                                }
-                                else if (source === '11') {
-                                    delete taxon.gbifTaxonID;
+                        console.error("  Applying selection...");
+                        if (choice === '') {
+                            return [2 /*return*/, {}];
+                        }
+                        matches = {};
+                        for (_a = 0, _b = choice.split(','); _a < _b.length; _a++) {
+                            i = _b[_a];
+                            prefix = prefixes[parseInt(i) - 1];
+                            taxa = groupedNameMatches[source][prefix];
+                            for (id in taxa) {
+                                if (id in matches) {
+                                    continue;
                                 }
-                                break;
+                                matches[id] = taxa[id];
                             }
                         }
-                        return [3 /*break*/, 8];
-                    case 7:
-                        i++;
-                        return [3 /*break*/, 2];
-                    case 8:
-                        _i++;
-                        return [3 /*break*/, 1];
-                    case 9: return [2 /*return*/];
+                        return [2 /*return*/, matches];
                 }
             });
         });
@@ -578,6 +638,9 @@ function main() {
             update: {
                 type: 'boolean',
                 short: 'u'
+            },
+            'update-mappings': {
+                type: 'boolean'
             }
         },
         allowPositionals: true
@@ -586,7 +649,16 @@ function main() {
     process.on('exit', function () {
         process.stdout.write('\n');
     });
-    var task = args.values.update ? processor.runUpdate() : processor.run();
+    var task;
+    if (args.values.update) {
+        task = processor.runUpdate();
+    }
+    else if (args.values['update-mappings']) {
+        task = processor.runMappingsUpdate();
+    }
+    else {
+        task = processor.run();
+    }
     task.catch(function (error) {
         console.error(error);
         process.exit(1);

package/lib/index.d.ts

@@ -1,3 +1,4 @@
 export * as catalog from './catalog/index';
 export * as resources from './resources/index';
+export * as taxonNames from './taxon-names/index';
 export * as csv from './csv';

package/lib/index.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.csv = exports.resources = exports.catalog = void 0;
+exports.csv = exports.taxonNames = exports.resources = exports.catalog = void 0;
 exports.catalog = require("./catalog/index");
 exports.resources = require("./resources/index");
+exports.taxonNames = require("./taxon-names/index");
 exports.csv = require("./csv");

package/lib/taxon-names/index.d.ts

@@ -0,0 +1,2 @@
+export declare function groupNameMatches(results: Record<TaxonId, TaxonMatch[]>): GroupedNameMatches;
+export declare function amendResource(resource: AmendedResource, source: string, matches: Record<TaxonId, TaxonMatch>): void;

package/lib/taxon-names/index.js

@@ -0,0 +1,79 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.amendResource = exports.groupNameMatches = void 0;
+var MINIMUM_PREFIX_LENGTH = 3;
+var VALID_COMMON_PREFIXES = new Set([
+    'Plantae|Tracheophyta',
+    'Fungi',
+    'Fungi|Ascomycota',
+    'Fungi|Basidiomycota',
+    'Fungi|Zygomycota'
+]);
+function getCommonPrefix(a, b) {
+    for (var i = 0; i < Math.max(a.length, b.length); i++) {
+        if (a[i] !== b[i]) {
+            return a.slice(0, i);
+        }
+    }
+    return a.slice();
+}
+function isValidPrefix(a, b) {
+    var prefix = getCommonPrefix(a, b);
+    return VALID_COMMON_PREFIXES.has(prefix.join('|')) || prefix.length >= MINIMUM_PREFIX_LENGTH;
+}
+function groupNameMatches(results) {
+    var prefixes = {};
+    for (var scientificNameID in results) {
+        var _loop_1 = function (result) {
+            if (!prefixes[result.source]) {
+                prefixes[result.source] = [];
+            }
+            var prefix = prefixes[result.source].find(function (prefix) { return isValidPrefix(prefix[0], result.classificationPath); });
+            if (!prefix) {
+                prefix = [result.classificationPath, {}];
+                prefixes[result.source].push(prefix);
+            }
+            else {
+                prefix[0] = getCommonPrefix(prefix[0], result.classificationPath);
+            }
+            if (scientificNameID in prefix[1]) {
+                return "continue";
+            }
+            prefix[1][scientificNameID] = result;
+        };
+        for (var _i = 0, _a = results[scientificNameID]; _i < _a.length; _i++) {
+            var result = _a[_i];
+            _loop_1(result);
+        }
+    }
+    var groupedNameMatches = {};
+    for (var source in prefixes) {
+        groupedNameMatches[source] = prefixes[source]
+            .sort(function (a, b) { return Object.keys(b[1]).length - Object.keys(a[1]).length; })
+            .reduce(function (map, _a) {
+            var prefix = _a[0], taxa = _a[1];
+            map[prefix.join('|')] = taxa;
+            return map;
+        }, {});
+    }
+    return groupedNameMatches;
+}
+exports.groupNameMatches = groupNameMatches;
+function amendResource(resource, source, matches) {
+    for (var id in matches) {
+        var match = matches[id];
+        if (source === '1') {
+            resource.taxa[id].colTaxonID = match.id;
+            if (match.currentId) {
+                resource.taxa[id].colAcceptedTaxonID = match.currentId;
+            }
+        }
+        else if (source === '11') {
+            resource.taxa[id].gbifTaxonID = match.id;
+            if (match.currentId) {
+                resource.taxa[id].gbifAcceptedTaxonID = match.currentId;
+            }
+        }
+    }
+}
+exports.amendResource = amendResource;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larsgw/formica",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "SDK and tools for data from the Library of Identification Resources",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",

package/src/bin/process-resources-index.ts

@@ -21,6 +21,14 @@ function sortObject (object: Record<string, any>): Record<string, any> {
 }
 /* eslint-enable @typescript-eslint/no-explicit-any */
 
+function addTaxon (gbifIndex: Record<string, TaxonId[]>, gbifId: string, taxon: string[]) {
+    if (!(gbifId in gbifIndex)) {
+        gbifIndex[gbifId] = []
+    }
+    gbifIndex[gbifId].push(taxon[0])
+    gbifIndex[gbifId].sort(numericSort)
+}
+
 async function main (args: string[]): Promise<void> {
     const REPO_ROOT = path.resolve(args[0])
 
@@ -50,11 +58,10 @@ async function main (args: string[]): Promise<void> {
             for (const taxon of dwc) {
                 const gbifId = taxon[25]
                 if (gbifId) {
-                    if (!(gbifId in gbifIndex)) {
-                        gbifIndex[gbifId] = []
+                    addTaxon(gbifIndex, gbifId, taxon)
+                    if (taxon[27] !== taxon[25]) {
+                        addTaxon(gbifIndex, taxon[27], taxon)
                     }
-                    gbifIndex[gbifId].push(taxon[0])
-                    gbifIndex[gbifId].sort(numericSort)
                 }
                 amendedResource.taxonCount += 1
             }

package/src/bin/process-resources.ts

@@ -8,17 +8,6 @@ import * as util from 'util'
 import { csv } from '../index'
 import { prompt, promptForAnswers, numericSort, runCommand } from './util'
 
-interface AmendedTaxon extends Taxon {
-    colTaxonID?: string,
-    gbifTaxonID?: string
-}
-
-interface AmendedResource extends Resource {
-    taxa: Record<TaxonId, AmendedTaxon>
-}
-
-type Classifications = Record<string, Array<[AmendedTaxon, string]>>
-
 const DWC_FIELDS: string[] = [
     'scientificNameID',
     'scientificName',
@@ -49,7 +38,9 @@ const DWC_FIELDS: string[] = [
     'higherClassification',
 
     'colTaxonID',
-    'gbifTaxonID'
+    'gbifTaxonID',
+    'colAcceptedTaxonID',
+    'gbifAcceptedTaxonID'
 ]
 
 const DISPLAY_FIELDS: string[] = [
@@ -74,17 +65,9 @@ const GBIF_RANKS: Rank[] = [
     'variety'
 ]
 
-const VALID_COMMON_PREFIXES = [
-    'Plantae|Tracheophyta',
-    'Fungi',
-    'Fungi|Ascomycota',
-    'Fungi|Basidiomycota',
-    'Fungi|Zygomycota'
-]
-
 function runGnverifier (names: string): Promise<string> {
     return new Promise((resolve, reject) => {
-        const proc = spawn('gnverifier', ['-s', '1,11', '-M'])
+        const proc = spawn('gnverifier', ['-s', '1,11', '-f', 'compact', '-M'])
         let stdout = ''
         proc.stdout.on('data', data => { stdout += data })
         proc.stderr.pipe(process.stdout)
@@ -153,6 +136,18 @@ class ResourceProcessor {
         }
     }
 
+    async runMappingsUpdate (): Promise<void> {
+        const input = await fs.readdir(this.DIR_TXT)
+
+        const ids = input
+            .map(file => path.basename(file, '.txt'))
+            .sort((a, b) => parseInt(a.slice(1)) - parseInt(b.slice(1)))
+
+        for (const id of ids) {
+            await this.processWork(id, true)
+        }
+    }
+
     async processWork (id: WorkId, update?: boolean): Promise<void> {
         const resources = await this.processResources(id, update)
 
@@ -174,11 +169,7 @@ class ResourceProcessor {
 
         const amendedResources = []
         for (const resource of resources) {
-            const [results, classifications] = await this.processResourceDwc(resource)
-
-            for (const source in classifications) {
-                await this.checkPrefix(resource, classifications, source)
-            }
+            const results = await this.processResourceDwc(resource)
 
             const skip = await this.shouldBeSkipped(resource.id)
 
@@ -256,100 +247,153 @@ class ResourceProcessor {
         }
     }
 
-    async processResourceDwc (resource: Resource): Promise<[AmendedResource, Classifications]> {
+    async processResourceDwc (resource: Resource): Promise<AmendedResource> {
         console.log(`${resource.workId}: matching ${resource.id}`)
-        const taxa: Record<string, Record<TaxonId, AmendedTaxon>> = {}
-        const names = []
+
+        const filteredResults: Record<TaxonId, TaxonMatch[]> = {}
+        const taxonNames: Record<string, TaxonId[]> = {}
+        const names = new Set()
         for (const id in resource.taxa) {
             const name = resource.taxa[id].scientificName
 
-            if (!taxa[name]) { taxa[name] = {} }
-            taxa[name][id] = { ...resource.taxa[id] }
+            if (!taxonNames[name]) { taxonNames[name] = [] }
+            taxonNames[name].push(id)
 
-            names.push(name)
+            names.add(name)
+            filteredResults[id] = []
         }
 
-        const result = await runGnverifier(names.join('\n'))
-        const classifications: Classifications = { '1': [], '11': [] }
-
-        const [header, ...matches] = csv.parseCsv(result)
-        for (const match of matches) {
-            const name = match[header.indexOf('ScientificName')]
-            const source = match[header.indexOf('DataSourceId')]
-            const id = match[header.indexOf('TaxonId')]
-            const classification = match[header.indexOf('ClassificationPath')]
-
-            for (const loirId in taxa[name]) {
-                const taxon = taxa[name][loirId]
-                if (source === '1' && !taxon.colTaxonID) {
-                    taxon.colTaxonID = id
-                    classifications[source].push([taxon, classification])
+        const result = await runGnverifier(Array.from(names).join('\n'))
+        for (const results of result.trim().split('\n')) {
+            const { name, results: matches } = JSON.parse(results)
+
+            if (!matches) {
+                continue
+            }
+
+            for (const match of matches) {
+                const source = match.dataSourceId
+                const currentRank = match.classificationRanks.split('|').pop()
+
+                if (match.scoreDetails.cardinalityScore === 0) {
+                    // Rank mismatch
+                    continue
+                } else if (source === 11 && currentRank === 'species' && match.classificationPath.endsWith(' spec')) {
+                    // GBIF species like "Nomada spec"
+                    continue
                 }
-                if (source === '11' && GBIF_RANKS.includes(taxon.taxonRank) && !taxon.gbifTaxonID) {
-                    taxon.gbifTaxonID = id
-                    classifications[source].push([taxon, classification])
+
+                for (const loirId of taxonNames[name]) {
+                    const taxon = resource.taxa[loirId]
+
+                    if (source === 11 && !GBIF_RANKS.includes(taxon.taxonRank)) {
+                        // Exclude GBIF matches for ranks that are not in GBIF
+                        continue
+                    } else if (source === 11 && !match.isSynonym && currentRank !== taxon.taxonRank) {
+                        // Exclude matches with rank mismatches (only possible
+                        // for non-synonyms).
+                        continue
+                    }
+
+                    if (!filteredResults[loirId]) {
+                        filteredResults[loirId] = []
+                    }
+
+                    filteredResults[loirId].push({
+                        source,
+                        id: match.recordId,
+                        currentId: match.currentRecordId,
+                        classificationPath: match.classificationPath.split('|')
+                    })
                 }
             }
         }
 
-        const results = {
-            ...resource,
-            taxa: Object.fromEntries(Object.values(resource.taxa).map(taxon => [
-                taxon.scientificNameID,
-                taxa[taxon.scientificName][taxon.scientificNameID]
-            ]))
+        const { taxonNames: { amendResource, groupNameMatches } } = await import('../index')
+        const groupedNameMatches = groupNameMatches(filteredResults)
+
+        const amendedResource: AmendedResource = { ...resource, taxa: { ...resource.taxa } }
+        for (const source in groupedNameMatches) {
+            const matches = await this.selectPrefixes(resource, groupedNameMatches, source)
+            amendResource(amendedResource, source, matches)
         }
 
-        return [results, classifications]
+        return amendedResource
     }
 
-    async checkPrefix (resource: AmendedResource, classifications: Classifications, source: string): Promise<void> {
-        const lists = classifications[source]
-        if (!lists.length) { return }
-        const prefix = lists[0][1].split('|')
-
-        for (const [taxon, list] of lists.slice(1)) {
-            const parts = list.split('|')
-            for (let i = 0; i < parts.length; i++) {
-                if (parts[i] !== prefix[i] && i < 3) {
-                    let choice
-
-                    if (source === '1' || taxon.taxonomicStatus !== 'accepted') {
-                        choice = 'd'
-                    } else if (VALID_COMMON_PREFIXES.includes(prefix.slice(0, i).join('|'))) {
-                        choice = 'k'
-                    } else {
-                        console.log(`${resource.workId}: source ${source} results in short prefix "${prefix.slice(0, i).join('|')}" (${i} taxa)`)
-                        console.log(`  taxon: ${taxon.scientificNameID} "${taxon.scientificName}"`)
-                        console.log(`  class: ${parts.join('|')}`)
-                        console.log(`  prefx: ${prefix.join('|')}`)
-
-                        choice = await promptForAnswers(`  Keep or delete (k/d)? `, ['k', 'K', 'd', 'D'])
-                    }
+    async selectPrefixes (resource: Resource, groupedNameMatches: GroupedNameMatches, source: string): Promise<Record<TaxonId, TaxonMatch>> {
+        const prefixes = Object.keys(groupedNameMatches[source])
+        if (prefixes.length === 0) {
+            return {}
+        } else if (prefixes.length === 1) {
+            return groupedNameMatches[source][prefixes[0]]
+        }
 
-                    switch (choice) {
-                        case 'k':
-                        case 'K': {
-                            console.log(`  keeping...`)
-                            break
-                        }
+        // Count total mapped taxa
+        const mappedTaxa: Record<TaxonId, boolean> = {}
+        for (const prefix of prefixes) {
+            for (const taxon in groupedNameMatches[source][prefix]) {
+                mappedTaxa[taxon] = true
+            }
+        }
+        const missedTaxonCount = Object.keys(mappedTaxa).length - Object.keys(groupedNameMatches[source][prefixes[0]]).length
 
-                        case 'd':
-                        case 'D': {
-                            console.log(`  deleting...`)
-                            if (source === '1') {
-                                delete taxon.colTaxonID
-                            } else if (source === '11') {
-                                delete taxon.gbifTaxonID
-                            }
-                            break
-                        }
-                    }
+        if (missedTaxonCount === 0) {
+            // Multiple prefixes but the first one maps all taxa (not counting that are unmapped in all prefixes)
+            return groupedNameMatches[source][prefixes[0]]
+        }
+
+        console.error(`${resource.workId}: source ${source} results in multiple prefixes`)
+
+        let choice
+        if (missedTaxonCount <= 5) {
+            console.error(`  Most common prefix misses ${missedTaxonCount} taxa: automatically selecting most common prefix...`)
+            choice = '1'
+        } else if (source === '1') {
+            console.error(`  Catalogue of Life: automatically selecting most common prefix...`)
+            choice = '1'
+        } else {
+            for (let i = 0; i < prefixes.length; i++) {
+                const prefix = prefixes[i]
+                const taxa = groupedNameMatches[source][prefix]
+                const taxonIds = Object.keys(taxa)
+
+                console.error(`  [${i + 1}] ${prefix} (${taxonIds.length} taxa)`)
+                for (let j = 0; j < Math.min(9, taxonIds.length); j++) {
+                    const taxonId = taxonIds[j]
+                    const taxon = resource.taxa[taxonId]
+                    const match = taxa[taxonId]
+                    console.error(`      taxon: ${taxonId} "${taxon.scientificName}" - ${match.classificationPath.join('|')}`)
+                }
+                if (taxonIds.length > 9) {
+                    console.error(`      ...`)
+                }
+            }
+
+            do {
+                choice = await prompt(`  Select prefixes (1-${prefixes.length})? `)
+            } while (!/^(|\d+(,\d+)*)$/.test(choice))
+        }
+
+        console.error(`  Applying selection...`)
+
+        if (choice === '') {
+            return {}
+        }
 
-                    break
+        const matches: Record<TaxonId, TaxonMatch> = {}
+        for (const i of choice.split(',')) {
+            const prefix = prefixes[parseInt(i) - 1]
+            const taxa = groupedNameMatches[source][prefix]
+            for (const id in taxa) {
+                if (id in matches) {
+                    continue
                 }
+                matches[id] = taxa[id]
             }
         }
+
+        return matches
     }
 
     checkResults (resource: AmendedResource): boolean {
@@ -388,6 +432,9 @@ function main (): void {
             update: {
                 type: 'boolean',
                 short: 'u'
+            },
+            'update-mappings': {
+                type: 'boolean'
             }
         },
         allowPositionals: true
@@ -398,7 +445,15 @@ function main (): void {
         process.stdout.write('\n')
     })
 
-    const task = args.values.update ? processor.runUpdate() : processor.run()
+    let task
+    if (args.values.update) {
+        task = processor.runUpdate()
+    } else if (args.values['update-mappings']) {
+        task = processor.runMappingsUpdate()
+    } else {
+        task = processor.run()
+    }
+
     task.catch(error => {
         console.error(error)
         process.exit(1)

package/src/index.ts

@@ -1,3 +1,4 @@
 export * as catalog from './catalog/index'
 export * as resources from './resources/index'
+export * as taxonNames from './taxon-names/index'
 export * as csv from './csv'

package/src/module.d.ts

@@ -1,8 +1,8 @@
 type Schema = Record<string, FieldSpecification>
 type FieldSpecification = {
-  required: boolean,
-  multiple: boolean | FieldSpecificationCallbackMultiple
-  format?: string[] | RegExp | FieldSpecificationCallbackFormat
+    required: boolean,
+    multiple: boolean | FieldSpecificationCallbackMultiple
+    format?: string[] | RegExp | FieldSpecificationCallbackFormat
 }
 
 type Value = string[] | string
@@ -11,12 +11,12 @@ type FieldSpecificationCallbackMultiple = (entry: Record<string, Value>) => bool
 type FieldSpecificationCallbackFormat = (value: SingleValue) => boolean;
 
 interface FieldError {
-  field: string,
-  error: string
+    field: string,
+    error: string
 }
 
 interface WorkError extends FieldError {
-  entity: WorkId
+    entity: WorkId
 }
 
 type Rank = string
@@ -28,59 +28,59 @@ type ResourceId = string
 type WorkId = string
 
 interface WorkingTaxon {
-  scientificNameID?: TaxonId,
-  scientificName?: string,
-  scientificNameAuthorship?: string,
-  genericName?: string,
-  infragenericEpithet?: string,
-  specificEpithet?: string,
-  intraspecificEpithet?: string,
-
-  taxonRank?: Rank,
-  taxonRemarks?: string,
-  collectionCode?: ResourceId,
-
-  taxonomicStatus?: TaxonStatus,
-  acceptedNameUsageID?: TaxonId,
-  acceptedNameUsage?: string,
-
-  parentNameUsageID?: TaxonId,
-  parentNameUsage?: string,
-  kingdom?: string,
-  phylum?: string,
-  class?: string,
-  order?: string,
-  family?: string,
-  subfamily?: string,
-  genus?: string,
-  subgenus?: string,
-  higherClassification?: string,
-
-  // Non-standard
-  scientificNameOnly?: string,
-  incorrect?: WorkingTaxon
+    scientificNameID?: TaxonId,
+    scientificName?: string,
+    scientificNameAuthorship?: string,
+    genericName?: string,
+    infragenericEpithet?: string,
+    specificEpithet?: string,
+    intraspecificEpithet?: string,
+
+    taxonRank?: Rank,
+    taxonRemarks?: string,
+    collectionCode?: ResourceId,
+
+    taxonomicStatus?: TaxonStatus,
+    acceptedNameUsageID?: TaxonId,
+    acceptedNameUsage?: string,
+
+    parentNameUsageID?: TaxonId,
+    parentNameUsage?: string,
+    kingdom?: string,
+    phylum?: string,
+    class?: string,
+    order?: string,
+    family?: string,
+    subfamily?: string,
+    genus?: string,
+    subgenus?: string,
+    higherClassification?: string,
+
+    // Non-standard
+    scientificNameOnly?: string,
+    incorrect?: WorkingTaxon
 }
 
 interface Taxon extends WorkingTaxon {
-  scientificNameID: TaxonId,
-  scientificName: string,
-  taxonRank: Rank,
-  collectionCode: ResourceId,
-  taxonomicStatus: string
+    scientificNameID: TaxonId,
+    scientificName: string,
+    taxonRank: Rank,
+    collectionCode: ResourceId,
+    taxonomicStatus: string
 }
 
 interface ResourceMetadata {
-  levels: Rank[],
-  scope: string[],
-  catalog?: object
+    levels: Rank[],
+    scope: string[],
+    catalog?: object
 }
 
 interface Resource {
-  id: string,
-  file: string,
-  workId: string,
-  metadata: ResourceMetadata,
-  taxa: Record<TaxonId, Taxon>
+    id: string,
+    file: string,
+    workId: string,
+    metadata: ResourceMetadata,
+    taxa: Record<TaxonId, Taxon>
 }
 
 interface ResourceHistory {
@@ -103,3 +103,23 @@ declare enum ResourceDiffType {
     Modified = '~',
     Unchanged = '='
 }
+
+interface AmendedTaxon extends Taxon {
+    colTaxonID?: string,
+    colAcceptedTaxonID?: string,
+    gbifTaxonID?: string,
+    gbifAcceptedTaxonID?: string
+}
+
+interface AmendedResource extends Resource {
+    taxa: Record<TaxonId, AmendedTaxon>
+}
+
+interface TaxonMatch {
+    source: number,
+    id: string,
+    currentId?: string,
+    classificationPath: string[]
+}
+
+type GroupedNameMatches = Record<string, Record<string, Record<TaxonId, TaxonMatch>>>

package/src/taxon-names/index.ts

@@ -0,0 +1,79 @@
+const MINIMUM_PREFIX_LENGTH = 3
+const VALID_COMMON_PREFIXES = new Set([
+    'Plantae|Tracheophyta',
+    'Fungi',
+    'Fungi|Ascomycota',
+    'Fungi|Basidiomycota',
+    'Fungi|Zygomycota'
+])
+
+function getCommonPrefix (a: string[], b: string[]): string[] {
+    for (let i = 0; i < Math.max(a.length, b.length); i++) {
+        if (a[i] !== b[i]) {
+            return a.slice(0, i)
+        }
+    }
+    return a.slice()
+}
+
+function isValidPrefix (a: string[], b: string[]): boolean {
+    const prefix = getCommonPrefix(a, b)
+    return VALID_COMMON_PREFIXES.has(prefix.join('|')) || prefix.length >= MINIMUM_PREFIX_LENGTH
+}
+
+export function groupNameMatches (results: Record<TaxonId, TaxonMatch[]>): GroupedNameMatches {
+    const prefixes: Record<string, [string[], Record<TaxonId, TaxonMatch>][]> = {}
+
+    for (const scientificNameID in results) {
+        for (const result of results[scientificNameID]) {
+            if (!prefixes[result.source]) {
+                prefixes[result.source] = []
+            }
+
+            let prefix = prefixes[result.source].find(prefix => isValidPrefix(prefix[0], result.classificationPath))
+
+            if (!prefix) {
+                prefix = [result.classificationPath, {}]
+                prefixes[result.source].push(prefix)
+            } else {
+                prefix[0] = getCommonPrefix(prefix[0], result.classificationPath)
+            }
+
+            if (scientificNameID in prefix[1]) {
+                continue
+            }
+
+            prefix[1][scientificNameID] = result
+        }
+    }
+
+    const groupedNameMatches: GroupedNameMatches = {}
+    for (const source in prefixes) {
+        groupedNameMatches[source] = prefixes[source]
+            .sort((a, b) => Object.keys(b[1]).length - Object.keys(a[1]).length)
+            .reduce((map: Record<string, Record<TaxonId, TaxonMatch>>, [prefix, taxa]) => {
+                map[prefix.join('|')] = taxa
+                return map
+            }, {})
+    }
+
+    return groupedNameMatches
+}
+
+export function amendResource (resource: AmendedResource, source: string, matches: Record<TaxonId, TaxonMatch>) {
+    for (const id in matches) {
+        const match = matches[id]
+
+        if (source === '1') {
+            resource.taxa[id].colTaxonID = match.id
+            if (match.currentId) {
+                resource.taxa[id].colAcceptedTaxonID = match.currentId
+            }
+        } else if (source === '11') {
+            resource.taxa[id].gbifTaxonID = match.id
+            if (match.currentId) {
+                resource.taxa[id].gbifAcceptedTaxonID = match.currentId
+            }
+        }
+    }
+}