@larsgw/formica 0.8.4 → 0.8.6

package/lib/bin/process-resources.js

@@ -1,16 +1,5 @@
 #!/usr/bin/env node
 "use strict";
-var __assign = (this && this.__assign) || function () {
-    __assign = Object.assign || function(t) {
-        for (var s, i = 1, n = arguments.length; i < n; i++) {
-            s = arguments[i];
-            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
-                t[p] = s[p];
-        }
-        return t;
-    };
-    return __assign.apply(this, arguments);
-};
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     var desc = Object.getOwnPropertyDescriptor(m, k);
@@ -53,48 +42,21 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
-    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ResourceProcessorSource = void 0;
-var fs_1 = require("fs");
-var path = __importStar(require("path"));
-var child_process_1 = require("child_process");
-var util = __importStar(require("util"));
-var index_1 = require("../index");
-var util_1 = require("./util");
+const fs_1 = require("fs");
+const path = __importStar(require("path"));
+const child_process_1 = require("child_process");
+const util = __importStar(require("util"));
+const index_1 = require("../index");
+const util_1 = require("./util");
 var ResourceProcessorSource;
 (function (ResourceProcessorSource) {
     ResourceProcessorSource["All"] = "all";
     ResourceProcessorSource["Unprocessed"] = "unprocessed";
     ResourceProcessorSource["Modified"] = "modified";
 })(ResourceProcessorSource || (exports.ResourceProcessorSource = ResourceProcessorSource = {}));
-var DWC_FIELDS = [
+const DWC_FIELDS = [
     'scientificNameID',
     'scientificName',
     'scientificNameAuthorship',
@@ -125,7 +87,7 @@ var DWC_FIELDS = [
     'colAcceptedTaxonID',
     'gbifAcceptedTaxonID'
 ];
-var DISPLAY_FIELDS = [
+const DISPLAY_FIELDS = [
     'scientificNameID',
     'taxonRank',
     'scientificName',
@@ -134,7 +96,7 @@ var DISPLAY_FIELDS = [
     'colTaxonID',
     'gbifTaxonID'
 ];
-var GBIF_RANKS = [
+const GBIF_RANKS = [
     'kingdom',
     'phyllum',
     'class',
@@ -146,12 +108,12 @@ var GBIF_RANKS = [
     'variety'
 ];
 function runGnverifier(names) {
-    return new Promise(function (resolve, reject) {
-        var proc = (0, child_process_1.spawn)('gnverifier', ['-s', '1,11', '-f', 'compact', '-M']);
-        var stdout = '';
-        proc.stdout.on('data', function (data) { stdout += data; });
+    return new Promise((resolve, reject) => {
+        const proc = (0, child_process_1.spawn)('gnverifier', ['-s', '1,11', '-f', 'compact', '-M']);
+        let stdout = '';
+        proc.stdout.on('data', data => { stdout += data; });
         proc.stderr.pipe(process.stdout);
-        proc.on('close', function (code) {
+        proc.on('close', code => {
             if (code === 0) {
                 resolve(stdout);
             }
@@ -164,489 +126,319 @@ function runGnverifier(names) {
     });
 }
 function listFiles(directory) {
-    return __awaiter(this, void 0, void 0, function () {
-        var input;
-        return __generator(this, function (_a) {
-            switch (_a.label) {
-                case 0: return [4 /*yield*/, fs_1.promises.readdir(directory)];
-                case 1:
-                    input = _a.sent();
-                    return [2 /*return*/, input.map(function (file) { return path.basename(file, '.txt'); }).sort(util_1.numericSort)];
-            }
-        });
+    return __awaiter(this, void 0, void 0, function* () {
+        const input = yield fs_1.promises.readdir(directory);
+        return input.map(file => path.basename(file, '.txt')).sort(util_1.numericSort);
     });
 }
 function listUnprocessedFiles(directory, outputDirectory) {
-    return __awaiter(this, void 0, void 0, function () {
-        var input, output, _a;
-        return __generator(this, function (_b) {
-            switch (_b.label) {
-                case 0: return [4 /*yield*/, listFiles(directory)];
-                case 1:
-                    input = _b.sent();
-                    _a = Set.bind;
-                    return [4 /*yield*/, fs_1.promises.readdir(outputDirectory)];
-                case 2:
-                    output = new (_a.apply(Set, [void 0, _b.sent()]))();
-                    return [2 /*return*/, input.filter(function (file) { return !output.has(file + '-1.csv'); })];
-            }
-        });
+    return __awaiter(this, void 0, void 0, function* () {
+        const input = yield listFiles(directory);
+        const output = new Set(yield fs_1.promises.readdir(outputDirectory));
+        return input.filter(file => !output.has(file + '-1.csv'));
     });
 }
 function listChangedFiles(directory) {
-    return __awaiter(this, void 0, void 0, function () {
-        var output;
-        return __generator(this, function (_a) {
-            switch (_a.label) {
-                case 0: return [4 /*yield*/, (0, util_1.runCommand)('git', ['diff', '--name-only', 'HEAD', '--', directory], {
-                        cwd: directory
-                    })];
-                case 1:
-                    output = _a.sent();
-                    return [2 /*return*/, output.trimEnd().split('\n').map(function (file) { return path.basename(file, '.txt'); }).sort(util_1.numericSort)];
-            }
+    return __awaiter(this, void 0, void 0, function* () {
+        const output = yield (0, util_1.runCommand)('git', ['diff', '--name-only', 'HEAD', '--', directory], {
+            cwd: directory
         });
+        return output.trimEnd().split('\n').map(file => path.basename(file, '.txt')).sort(util_1.numericSort);
     });
 }
 function getOldFile(file) {
-    return __awaiter(this, void 0, void 0, function () {
-        var options, gitRoot;
-        return __generator(this, function (_a) {
-            switch (_a.label) {
-                case 0:
-                    options = {
-                        cwd: path.dirname(file)
-                    };
-                    return [4 /*yield*/, (0, util_1.runCommand)('git', ['rev-parse', '--show-toplevel'], options)];
-                case 1:
-                    gitRoot = (_a.sent()).trim();
-                    return [4 /*yield*/, (0, util_1.runCommand)('git', ['show', 'HEAD:' + path.relative(gitRoot, file)], options)];
-                case 2: return [2 /*return*/, _a.sent()];
-            }
-        });
+    return __awaiter(this, void 0, void 0, function* () {
+        const options = {
+            cwd: path.dirname(file)
+        };
+        const gitRoot = (yield (0, util_1.runCommand)('git', ['rev-parse', '--show-toplevel'], options)).trim();
+        return yield (0, util_1.runCommand)('git', ['show', 'HEAD:' + path.relative(gitRoot, file)], options);
     });
 }
-var ResourceProcessor = /** @class */ (function () {
-    function ResourceProcessor(collectionPath) {
+class ResourceProcessor {
+    constructor(collectionPath) {
         this.DIR_ROOT = path.resolve(collectionPath);
         this.DIR_TXT = path.join(this.DIR_ROOT, 'txt');
         this.DIR_DWC = path.join(this.DIR_ROOT, 'dwc');
         this.FILE_PROBLEMS = path.join(this.DIR_ROOT, 'problems.csv');
     }
-    ResourceProcessor.prototype.run = function (source, config) {
-        return __awaiter(this, void 0, void 0, function () {
-            var ids, _i, ids_1, id;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0: return [4 /*yield*/, this.listWorks(source)];
-                    case 1:
-                        ids = _a.sent();
-                        _i = 0, ids_1 = ids;
-                        _a.label = 2;
-                    case 2:
-                        if (!(_i < ids_1.length)) return [3 /*break*/, 5];
-                        id = ids_1[_i];
-                        return [4 /*yield*/, this.processWork(id, config)];
-                    case 3:
-                        _a.sent();
-                        _a.label = 4;
-                    case 4:
-                        _i++;
-                        return [3 /*break*/, 2];
-                    case 5: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    ResourceProcessor.prototype.listWorks = function (source) {
-        return __awaiter(this, void 0, void 0, function () {
-            return __generator(this, function (_a) {
-                switch (source) {
-                    case ResourceProcessorSource.All:
-                        return [2 /*return*/, listFiles(this.DIR_TXT)];
-                    case ResourceProcessorSource.Unprocessed:
-                        return [2 /*return*/, listUnprocessedFiles(this.DIR_TXT, this.DIR_DWC)];
-                    case ResourceProcessorSource.Modified:
-                        return [2 /*return*/, listChangedFiles(this.DIR_TXT)];
-                    default:
-                        return [2 /*return*/, []];
-                }
-                return [2 /*return*/];
-            });
+    run(source, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const ids = yield this.listWorks(source);
+            for (const id of ids) {
+                yield this.processWork(id, config);
+            }
         });
-    };
-    ResourceProcessor.prototype.processWork = function (id, config) {
-        return __awaiter(this, void 0, void 0, function () {
-            var resources;
-            var _this = this;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0: return [4 /*yield*/, this.processResources(id, config)];
-                    case 1:
-                        resources = _a.sent();
-                        return [4 /*yield*/, Promise.all(resources.map(function (resource) {
-                                var header = DWC_FIELDS;
-                                var table = [header];
-                                var _loop_1 = function (id_1) {
-                                    var taxon = resource.taxa[id_1];
-                                    table.push(header.map(function (column) { return taxon[column] || ''; }));
-                                };
-                                for (var id_1 in resource.taxa) {
-                                    _loop_1(id_1);
-                                }
-                                return fs_1.promises.writeFile(path.join(_this.DIR_DWC, "".concat(resource.file, ".csv")), index_1.csv.formatCsv(table, ',').trim());
-                            }))];
-                    case 2:
-                        _a.sent();
-                        return [2 /*return*/];
-                }
-            });
+    }
+    listWorks(source) {
+        return __awaiter(this, void 0, void 0, function* () {
+            switch (source) {
+                case ResourceProcessorSource.All:
+                    return listFiles(this.DIR_TXT);
+                case ResourceProcessorSource.Unprocessed:
+                    return listUnprocessedFiles(this.DIR_TXT, this.DIR_DWC);
+                case ResourceProcessorSource.Modified:
+                    return listChangedFiles(this.DIR_TXT);
+                default:
+                    return [];
+            }
         });
-    };
-    ResourceProcessor.prototype.processResources = function (id, config) {
-        return __awaiter(this, void 0, void 0, function () {
-            var resources, amendedResources, _i, resources_1, resource, results, skip, correct, choice, _a, reason;
-            return __generator(this, function (_b) {
-                switch (_b.label) {
-                    case 0: return [4 /*yield*/, this.processResourceText(id, config)];
-                    case 1:
-                        resources = _b.sent();
-                        amendedResources = [];
-                        _i = 0, resources_1 = resources;
-                        _b.label = 2;
-                    case 2:
-                        if (!(_i < resources_1.length)) return [3 /*break*/, 11];
-                        resource = resources_1[_i];
-                        return [4 /*yield*/, this.processResourceDwc(resource, config)];
-                    case 3:
-                        results = _b.sent();
-                        return [4 /*yield*/, this.shouldBeSkipped(resource.id)];
-                    case 4:
-                        skip = _b.sent();
-                        if (!!skip) return [3 /*break*/, 9];
-                        correct = this.checkResults(results);
-                        if (!!correct) return [3 /*break*/, 9];
-                        return [4 /*yield*/, (0, util_1.promptForAnswers)("".concat(resource.workId, ": problems found in ").concat(resource.id, ". Skip or retry (s/r)? "), ['s', 'S', 'r', 'R'])];
-                    case 5:
-                        choice = _b.sent();
-                        _a = choice;
-                        switch (_a) {
-                            case 's': return [3 /*break*/, 6];
-                            case 'S': return [3 /*break*/, 6];
-                            case 'r': return [3 /*break*/, 8];
-                            case 'R': return [3 /*break*/, 8];
-                        }
-                        return [3 /*break*/, 9];
-                    case 6: return [4 /*yield*/, (0, util_1.prompt)('Reason for skipping? ')];
-                    case 7:
-                        reason = _b.sent();
-                        fs_1.promises.appendFile(this.FILE_PROBLEMS, index_1.csv.formatCsv([[
-                                resource.workId,
-                                resource.id,
-                                reason
-                            ]]));
-                        console.log("".concat(resource.workId, ": skipping ").concat(resource.id));
-                        return [3 /*break*/, 9];
-                    case 8:
-                        {
-                            console.log("".concat(resource.workId, ": retrying ").concat(resource.id));
-                            return [2 /*return*/, this.processResources(id, config)];
-                        }
-                        _b.label = 9;
-                    case 9:
-                        amendedResources.push(results);
-                        _b.label = 10;
-                    case 10:
-                        _i++;
-                        return [3 /*break*/, 2];
-                    case 11: return [2 /*return*/, amendedResources];
+    }
+    processWork(id, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const resources = yield this.processResources(id, config);
+            yield Promise.all(resources.map(resource => {
+                const header = DWC_FIELDS;
+                const table = [header];
+                for (const id in resource.taxa) {
+                    const taxon = resource.taxa[id];
+                    table.push(header.map(column => taxon[column] || ''));
                 }
-            });
+                return fs_1.promises.writeFile(path.join(this.DIR_DWC, `${resource.file}.csv`), index_1.csv.formatCsv(table, ',').trim());
+            }));
         });
-    };
-    ResourceProcessor.prototype.processResourceText = function (id, config) {
-        return __awaiter(this, void 0, void 0, function () {
-            var filePath, file, old, dwc, _i, _a, file_1, filePath_1, _b, _c, _d, _e, resources, error_1, prefix, file;
-            var _f;
-            return __generator(this, function (_g) {
-                switch (_g.label) {
-                    case 0:
-                        _g.trys.push([0, 10, , 12]);
-                        console.log("".concat(id, ": generating Darwin Core"));
-                        filePath = path.join(this.DIR_TXT, id + '.txt');
-                        return [4 /*yield*/, fs_1.promises.readFile(filePath, 'utf-8')];
-                    case 1:
-                        file = _g.sent();
-                        old = undefined;
-                        if (!config.update) return [3 /*break*/, 8];
-                        dwc = [];
-                        _i = 0;
-                        return [4 /*yield*/, fs_1.promises.readdir(this.DIR_DWC)];
-                    case 2:
-                        _a = _g.sent();
-                        _g.label = 3;
-                    case 3:
-                        if (!(_i < _a.length)) return [3 /*break*/, 6];
-                        file_1 = _a[_i];
-                        if (!file_1.startsWith(id + '-')) return [3 /*break*/, 5];
-                        filePath_1 = path.join(this.DIR_DWC, file_1);
-                        _c = (_b = dwc).push;
-                        _e = (_d = index_1.csv).parseCsv;
-                        return [4 /*yield*/, getOldFile(filePath_1)];
-                    case 4:
-                        _c.apply(_b, [_e.apply(_d, [_g.sent()])]);
-                        _g.label = 5;
-                    case 5:
-                        _i++;
-                        return [3 /*break*/, 3];
-                    case 6:
-                        _f = {};
-                        return [4 /*yield*/, getOldFile(filePath)];
-                    case 7:
-                        old = (_f.txt = _g.sent(), _f.dwc = dwc, _f);
-                        _g.label = 8;
-                    case 8: return [4 /*yield*/, Promise.resolve().then(function () { return __importStar(require('../index')); })];
-                    case 9:
-                        resources = (_g.sent()).resources;
-                        return [2 /*return*/, resources.parseTextFile(file, id, old)];
-                    case 10:
-                        error_1 = _g.sent();
-                        console.log(error_1);
-                        return [4 /*yield*/, (0, util_1.prompt)("".concat(id, ": generating Darwin Core failed, retry? "))
-                            // Clear cache to re-import
-                        ];
-                    case 11:
-                        _g.sent();
-                        prefix = path.dirname(require.resolve('../index'));
-                        for (file in require.cache) {
-                            if (file.startsWith(prefix)) {
-                                delete require.cache[file];
+    }
+    processResources(id, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const resources = yield this.processResourceText(id, config);
+            const amendedResources = [];
+            for (const resource of resources) {
+                const results = yield this.processResourceDwc(resource, config);
+                const skip = yield this.shouldBeSkipped(resource.id);
+                if (!skip) {
+                    const correct = this.checkResults(results);
+                    if (!correct) {
+                        const choice = yield (0, util_1.promptForAnswers)(`${resource.workId}: problems found in ${resource.id}. Skip or retry (s/r)? `, ['s', 'S', 'r', 'R']);
+                        switch (choice) {
+                            case 's':
+                            case 'S': {
+                                const reason = yield (0, util_1.prompt)('Reason for skipping? ');
+                                fs_1.promises.appendFile(this.FILE_PROBLEMS, index_1.csv.formatCsv([[
+                                        resource.workId,
+                                        resource.id,
+                                        reason
+                                    ]]));
+                                console.log(`${resource.workId}: skipping ${resource.id}`);
+                                break;
+                            }
+                            case 'r':
+                            case 'R': {
+                                console.log(`${resource.workId}: retrying ${resource.id}`);
+                                return this.processResources(id, config);
                             }
                         }
-                        return [2 /*return*/, this.processResourceText(id, config)];
-                    case 12: return [2 /*return*/];
+                    }
                 }
-            });
+                amendedResources.push(results);
+            }
+            return amendedResources;
         });
-    };
-    ResourceProcessor.prototype.processResourceDwc = function (resource, config) {
-        return __awaiter(this, void 0, void 0, function () {
-            var file, _a, header_1, rows, _b, _c, _i, rows_1, row, oldTaxon, taxon, filteredResults, taxonNames, names, id, name_1, result, _loop_2, _d, _e, results, _f, amendResource, groupNameMatches, groupedNameMatches, amendedResource, _g, _h, _j, _k, source, matches;
-            return __generator(this, function (_l) {
-                switch (_l.label) {
-                    case 0:
-                        console.log("".concat(resource.workId, ": matching ").concat(resource.id));
-                        if (!!config.updateMappings) return [3 /*break*/, 3];
-                        file = path.join(this.DIR_DWC, resource.file + '.csv');
-                        if (!(0, fs_1.existsSync)(file)) return [3 /*break*/, 2];
-                        _c = (_b = index_1.csv).parseCsv;
-                        return [4 /*yield*/, fs_1.promises.readFile(file, 'utf-8')];
-                    case 1:
-                        _a = _c.apply(_b, [_l.sent()]), header_1 = _a[0], rows = _a.slice(1);
-                        for (_i = 0, rows_1 = rows; _i < rows_1.length; _i++) {
-                            row = rows_1[_i];
-                            oldTaxon = row.reduce(function (taxon, value, index) {
-                                taxon[header_1[index]] = value;
-                                return taxon;
-                            }, {});
-                            taxon = resource.taxa[oldTaxon.scientificNameID];
-                            if (taxon) {
-                                taxon.colTaxonID = oldTaxon.colTaxonID;
-                                taxon.colAcceptedTaxonID = oldTaxon.colAcceptedTaxonID;
-                                taxon.gbifTaxonID = oldTaxon.gbifTaxonID;
-                                taxon.gbifAcceptedTaxonID = oldTaxon.gbifAcceptedTaxonID;
-                            }
-                        }
-                        _l.label = 2;
-                    case 2: return [2 /*return*/, resource];
-                    case 3:
-                        filteredResults = {};
-                        taxonNames = {};
-                        names = new Set();
-                        for (id in resource.taxa) {
-                            name_1 = resource.taxa[id].scientificName;
-                            if (!taxonNames[name_1]) {
-                                taxonNames[name_1] = [];
-                            }
-                            taxonNames[name_1].push(id);
-                            names.add(name_1);
-                            filteredResults[id] = [];
-                        }
-                        return [4 /*yield*/, runGnverifier(Array.from(names).join('\n'))];
-                    case 4:
-                        result = _l.sent();
-                        _loop_2 = function (results) {
-                            var _m = JSON.parse(results), name_2 = _m.name, matches = _m.results;
-                            if (!matches) {
-                                return "continue";
-                            }
-                            // Fix author scoring for some species, see https://github.com/gnames/gnverifier/issues/129
-                            matches.sort(function (a, b) {
-                                if (a.sortScore !== b.sortScore) {
-                                    return b.sortScore - a.sortScore;
-                                }
-                                return name_2 === a.matchedName ? -1 : name_2 === b.matchedName ? 1 : 0;
-                            });
-                            for (var _o = 0, _p = matches; _o < _p.length; _o++) {
-                                var match = _p[_o];
-                                var source = match.dataSourceId;
-                                var currentRank = match.classificationRanks.split('|').pop();
-                                if (match.scoreDetails.cardinalityScore === 0) {
-                                    // Rank mismatch
-                                    continue;
-                                }
-                                else if (source === 11 && currentRank === 'species' && match.classificationPath.endsWith(' spec')) {
-                                    // GBIF species like "Nomada spec"
-                                    continue;
-                                }
-                                for (var _q = 0, _r = taxonNames[name_2]; _q < _r.length; _q++) {
-                                    var loirId = _r[_q];
-                                    var taxon = resource.taxa[loirId];
-                                    if (source === 11 && !GBIF_RANKS.includes(taxon.taxonRank)) {
-                                        // Exclude GBIF matches for ranks that are not in GBIF
-                                        continue;
-                                    }
-                                    else if (source === 11 && !match.isSynonym && currentRank !== taxon.taxonRank) {
-                                        // Exclude matches with rank mismatches (only possible
-                                        // for non-synonyms).
-                                        continue;
-                                    }
-                                    if (!filteredResults[loirId]) {
-                                        filteredResults[loirId] = [];
-                                    }
-                                    filteredResults[loirId].push({
-                                        source: source,
-                                        id: match.recordId,
-                                        currentId: match.currentRecordId,
-                                        classificationPath: match.classificationPath.split('|')
-                                    });
-                                }
-                            }
-                        };
-                        for (_d = 0, _e = result.trim().split('\n'); _d < _e.length; _d++) {
-                            results = _e[_d];
-                            _loop_2(results);
+    }
+    processResourceText(id, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                console.log(`${id}: generating Darwin Core`);
+                const filePath = path.join(this.DIR_TXT, id + '.txt');
+                const file = yield fs_1.promises.readFile(filePath, 'utf-8');
+                let old = undefined;
+                if (config.update) {
+                    const dwc = [];
+                    for (const file of yield fs_1.promises.readdir(this.DIR_DWC)) {
+                        if (file.startsWith(id + '-')) {
+                            const filePath = path.join(this.DIR_DWC, file);
+                            dwc.push(index_1.csv.parseCsv(yield getOldFile(filePath)));
                         }
-                        return [4 /*yield*/, Promise.resolve().then(function () { return __importStar(require('../index')); })];
-                    case 5:
-                        _f = (_l.sent()).taxonNames, amendResource = _f.amendResource, groupNameMatches = _f.groupNameMatches;
-                        groupedNameMatches = groupNameMatches(filteredResults);
-                        amendedResource = __assign(__assign({}, resource), { taxa: __assign({}, resource.taxa) });
-                        _g = groupedNameMatches;
-                        _h = [];
-                        for (_j in _g)
-                            _h.push(_j);
-                        _k = 0;
-                        _l.label = 6;
-                    case 6:
-                        if (!(_k < _h.length)) return [3 /*break*/, 9];
-                        _j = _h[_k];
-                        if (!(_j in _g)) return [3 /*break*/, 8];
-                        source = _j;
-                        return [4 /*yield*/, this.selectPrefixes(resource, groupedNameMatches, source)];
-                    case 7:
-                        matches = _l.sent();
-                        amendResource(amendedResource, source, matches);
-                        _l.label = 8;
-                    case 8:
-                        _k++;
-                        return [3 /*break*/, 6];
-                    case 9: return [2 /*return*/, amendedResource];
+                    }
+                    old = { txt: yield getOldFile(filePath), dwc };
                 }
-            });
+                const { resources } = yield Promise.resolve().then(() => __importStar(require('../index')));
+                return resources.parseTextFile(file, id, old);
+            }
+            catch (error) {
+                console.log(error.message);
+                yield (0, util_1.prompt)(`${id}: generating Darwin Core failed, retry? `);
+                // Clear cache to re-import
+                const prefix = path.dirname(require.resolve('../index'));
+                for (const file in require.cache) {
+                    if (file.startsWith(prefix)) {
+                        delete require.cache[file];
+                    }
+                }
+                return this.processResourceText(id, config);
+            }
         });
-    };
-    ResourceProcessor.prototype.selectPrefixes = function (resource, groupedNameMatches, source) {
-        return __awaiter(this, void 0, void 0, function () {
-            var prefixes, mappedTaxa, _i, prefixes_1, prefix, taxon, missedTaxonCount, choice, i, prefix, taxa, taxonIds, j, taxonId, taxon, match, matches, _a, _b, i, prefix, taxa, id;
-            return __generator(this, function (_c) {
-                switch (_c.label) {
-                    case 0:
-                        prefixes = Object.keys(groupedNameMatches[source]);
-                        if (prefixes.length === 0) {
-                            return [2 /*return*/, {}];
-                        }
-                        else if (prefixes.length === 1) {
-                            return [2 /*return*/, groupedNameMatches[source][prefixes[0]]];
-                        }
-                        mappedTaxa = {};
-                        for (_i = 0, prefixes_1 = prefixes; _i < prefixes_1.length; _i++) {
-                            prefix = prefixes_1[_i];
-                            for (taxon in groupedNameMatches[source][prefix]) {
-                                mappedTaxa[taxon] = true;
-                            }
-                        }
-                        missedTaxonCount = Object.keys(mappedTaxa).length - Object.keys(groupedNameMatches[source][prefixes[0]]).length;
-                        if (missedTaxonCount === 0) {
-                            // Multiple prefixes but the first one maps all taxa (not counting that are unmapped in all prefixes)
-                            return [2 /*return*/, groupedNameMatches[source][prefixes[0]]];
+    }
+    processResourceDwc(resource, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            console.log(`${resource.workId}: matching ${resource.id}`);
+            if (!config.updateMappings) {
+                const file = path.join(this.DIR_DWC, resource.file + '.csv');
+                if ((0, fs_1.existsSync)(file)) {
+                    const [header, ...rows] = index_1.csv.parseCsv(yield fs_1.promises.readFile(file, 'utf-8'));
+                    for (const row of rows) {
+                        const oldTaxon = row.reduce((taxon, value, index) => {
+                            taxon[header[index]] = value;
+                            return taxon;
+                        }, {});
+                        const taxon = resource.taxa[oldTaxon.scientificNameID];
+                        if (taxon) {
+                            taxon.colTaxonID = oldTaxon.colTaxonID;
+                            taxon.colAcceptedTaxonID = oldTaxon.colAcceptedTaxonID;
+                            taxon.gbifTaxonID = oldTaxon.gbifTaxonID;
+                            taxon.gbifAcceptedTaxonID = oldTaxon.gbifAcceptedTaxonID;
                         }
-                        console.error("".concat(resource.workId, ": source ").concat(source, " results in multiple prefixes"));
-                        if (!(missedTaxonCount <= 5)) return [3 /*break*/, 1];
-                        console.error("  Most common prefix misses ".concat(missedTaxonCount, " taxa: automatically selecting most common prefix..."));
-                        choice = '1';
-                        return [3 /*break*/, 6];
-                    case 1:
-                        if (!(source === '1')) return [3 /*break*/, 2];
-                        console.error("  Catalogue of Life: automatically selecting most common prefix...");
-                        choice = '1';
-                        return [3 /*break*/, 6];
-                    case 2:
-                        for (i = 0; i < prefixes.length; i++) {
-                            prefix = prefixes[i];
-                            taxa = groupedNameMatches[source][prefix];
-                            taxonIds = Object.keys(taxa);
-                            console.error("  [".concat(i + 1, "] ").concat(prefix, " (").concat(taxonIds.length, " taxa)"));
-                            for (j = 0; j < Math.min(9, taxonIds.length); j++) {
-                                taxonId = taxonIds[j];
-                                taxon = resource.taxa[taxonId];
-                                match = taxa[taxonId];
-                                console.error("      taxon: ".concat(taxonId, " \"").concat(taxon.scientificName, "\" - ").concat(match.classificationPath.join('|')));
-                            }
-                            if (taxonIds.length > 9) {
-                                console.error("      ...");
-                            }
+                    }
+                }
+                return resource;
+            }
+            const filteredResults = {};
+            const taxonNames = {};
+            const names = new Set();
+            for (const id in resource.taxa) {
+                const name = resource.taxa[id].scientificName;
+                if (!taxonNames[name]) {
+                    taxonNames[name] = [];
+                }
+                taxonNames[name].push(id);
+                names.add(name);
+                filteredResults[id] = [];
+            }
+            const result = yield runGnverifier(Array.from(names).join('\n'));
+            for (const results of result.trim().split('\n')) {
+                const { name, results: matches } = JSON.parse(results);
+                if (!matches) {
+                    continue;
+                }
+                // Fix author scoring for some species, see https://github.com/gnames/gnverifier/issues/129
+                matches.sort((a, b) => {
+                    if (a.sortScore !== b.sortScore) {
+                        return b.sortScore - a.sortScore;
+                    }
+                    return name === a.matchedName ? -1 : name === b.matchedName ? 1 : 0;
+                });
+                for (const match of matches) {
+                    const source = match.dataSourceId;
+                    const currentRank = match.classificationRanks.split('|').pop();
+                    if (match.scoreDetails.cardinalityScore === 0) {
+                        // Rank mismatch
+                        continue;
+                    }
+                    else if (source === 11 && currentRank === 'species' && match.classificationPath.endsWith(' spec')) {
+                        // GBIF species like "Nomada spec"
+                        continue;
+                    }
+                    for (const loirId of taxonNames[name]) {
+                        const taxon = resource.taxa[loirId];
+                        if (source === 11 && !GBIF_RANKS.includes(taxon.taxonRank)) {
+                            // Exclude GBIF matches for ranks that are not in GBIF
+                            continue;
                         }
-                        _c.label = 3;
-                    case 3: return [4 /*yield*/, (0, util_1.prompt)("  Select prefixes (1-".concat(prefixes.length, ")? "))];
-                    case 4:
-                        choice = _c.sent();
-                        _c.label = 5;
-                    case 5:
-                        if (!/^(|\d+(,\d+)*)$/.test(choice)) return [3 /*break*/, 3];
-                        _c.label = 6;
-                    case 6:
-                        console.error("  Applying selection...");
-                        if (choice === '') {
-                            return [2 /*return*/, {}];
+                        else if (source === 11 && !match.isSynonym && currentRank !== taxon.taxonRank) {
+                            // Exclude matches with rank mismatches (only possible
+                            // for non-synonyms).
+                            continue;
                         }
-                        matches = {};
-                        for (_a = 0, _b = choice.split(','); _a < _b.length; _a++) {
-                            i = _b[_a];
-                            prefix = prefixes[parseInt(i) - 1];
-                            taxa = groupedNameMatches[source][prefix];
-                            for (id in taxa) {
-                                if (id in matches) {
-                                    continue;
-                                }
-                                matches[id] = taxa[id];
-                            }
+                        if (!filteredResults[loirId]) {
+                            filteredResults[loirId] = [];
                         }
-                        return [2 /*return*/, matches];
+                        filteredResults[loirId].push({
+                            source,
+                            id: match.recordId,
+                            currentId: match.currentRecordId,
+                            classificationPath: match.classificationPath.split('|')
+                        });
+                    }
                 }
-            });
+            }
+            const { taxonNames: { amendResource, groupNameMatches } } = yield Promise.resolve().then(() => __importStar(require('../index')));
+            const groupedNameMatches = groupNameMatches(filteredResults);
+            const amendedResource = Object.assign(Object.assign({}, resource), { taxa: Object.assign({}, resource.taxa) });
+            for (const source in groupedNameMatches) {
+                const matches = yield this.selectPrefixes(resource, groupedNameMatches, source);
+                amendResource(amendedResource, source, matches);
+            }
+            return amendedResource;
         });
-    };
-    ResourceProcessor.prototype.checkResults = function (resource) {
-        var correct = true;
-        var missing = [];
-        for (var id in resource.taxa) {
-            var taxon = resource.taxa[id];
+    }
+    selectPrefixes(resource, groupedNameMatches, source) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const prefixes = Object.keys(groupedNameMatches[source]);
+            if (prefixes.length === 0) {
+                return {};
+            }
+            else if (prefixes.length === 1) {
+                return groupedNameMatches[source][prefixes[0]];
+            }
+            // Count total mapped taxa
+            const mappedTaxa = {};
+            for (const prefix of prefixes) {
+                for (const taxon in groupedNameMatches[source][prefix]) {
+                    mappedTaxa[taxon] = true;
+                }
+            }
+            const missedTaxonCount = Object.keys(mappedTaxa).length - Object.keys(groupedNameMatches[source][prefixes[0]]).length;
+            if (missedTaxonCount === 0) {
+                // Multiple prefixes but the first one maps all taxa (not counting that are unmapped in all prefixes)
+                return groupedNameMatches[source][prefixes[0]];
+            }
+            console.error(`${resource.workId}: source ${source} results in multiple prefixes`);
+            let choice;
+            if (missedTaxonCount <= 5) {
+                console.error(`  Most common prefix misses ${missedTaxonCount} taxa: automatically selecting most common prefix...`);
+                choice = '1';
+            }
+            else if (source === '1') {
+                console.error(`  Catalogue of Life: automatically selecting most common prefix...`);
+                choice = '1';
+            }
+            else {
+                for (let i = 0; i < prefixes.length; i++) {
+                    const prefix = prefixes[i];
+                    const taxa = groupedNameMatches[source][prefix];
+                    const taxonIds = Object.keys(taxa);
+                    console.error(`  [${i + 1}] ${prefix} (${taxonIds.length} taxa)`);
+                    for (let j = 0; j < Math.min(9, taxonIds.length); j++) {
+                        const taxonId = taxonIds[j];
+                        const taxon = resource.taxa[taxonId];
+                        const match = taxa[taxonId];
+                        console.error(`      taxon: ${taxonId} "${taxon.scientificName}" - ${match.classificationPath.join('|')}`);
+                    }
+                    if (taxonIds.length > 9) {
+                        console.error(`      ...`);
+                    }
+                }
+                do {
+                    choice = yield (0, util_1.prompt)(`  Select prefixes (1-${prefixes.length})? `);
+                } while (!/^(|\d+(,\d+)*)$/.test(choice));
+            }
+            console.error(`  Applying selection...`);
+            if (choice === '') {
+                return {};
+            }
+            const matches = {};
+            for (const i of choice.split(',')) {
+                const prefix = prefixes[parseInt(i) - 1];
+                const taxa = groupedNameMatches[source][prefix];
+                for (const id in taxa) {
+                    if (id in matches) {
+                        continue;
+                    }
+                    matches[id] = taxa[id];
+                }
+            }
+            return matches;
+        });
+    }
+    checkResults(resource) {
+        let correct = true;
+        const missing = [];
+        for (const id in resource.taxa) {
+            const taxon = resource.taxa[id];
             if (taxon.taxonomicStatus !== 'accepted') {
                 continue;
             }
-            var missingCol = false; // !taxon.colTaxonID
-            var missingGbif = GBIF_RANKS.includes(taxon.taxonRank) && !taxon.gbifTaxonID;
+            const missingCol = false; // !taxon.colTaxonID
+            const missingGbif = GBIF_RANKS.includes(taxon.taxonRank) && !taxon.gbifTaxonID;
             if (missingCol || missingGbif) {
                 correct = false;
                 missing.push(taxon);
@@ -656,29 +448,16 @@ var ResourceProcessor = /** @class */ (function () {
             console.table(missing, DISPLAY_FIELDS);
         }
         return correct;
-    };
-    ResourceProcessor.prototype.shouldBeSkipped = function (id) {
-        return __awaiter(this, void 0, void 0, function () {
-            var problems, _a, _b;
-            return __generator(this, function (_c) {
-                switch (_c.label) {
-                    case 0:
-                        _b = (_a = index_1.csv).parseCsv;
-                        return [4 /*yield*/, fs_1.promises.readFile(this.FILE_PROBLEMS, 'utf8')];
-                    case 1:
-                        problems = _b.apply(_a, [_c.sent()]);
-                        return [2 /*return*/, problems.some(function (_a) {
-                                var _work = _a[0], resource = _a[1], _problem = _a[2];
-                                return resource === id;
-                            })];
-                }
-            });
+    }
+    shouldBeSkipped(id) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const problems = index_1.csv.parseCsv(yield fs_1.promises.readFile(this.FILE_PROBLEMS, 'utf8'));
+            return problems.some(([_work, resource, _problem]) => resource === id);
         });
-    };
-    return ResourceProcessor;
-}());
+    }
+}
 function main() {
-    var args = util.parseArgs({
+    const args = util.parseArgs({
         options: {
             source: {
                 type: 'string',
@@ -692,16 +471,16 @@ function main() {
         },
         allowPositionals: true
     });
-    var processor = new ResourceProcessor(args.positionals[0]);
-    process.on('exit', function () {
+    const processor = new ResourceProcessor(args.positionals[0]);
+    process.on('exit', () => {
         process.stdout.write('\n');
     });
-    var source = args.values.source;
-    var config = {
+    const source = args.values.source;
+    const config = {
         update: source !== 'unprocessed',
         updateMappings: !args.values['keep-mappings']
     };
-    processor.run(source, config).catch(function (error) {
+    processor.run(source, config).catch(error => {
         console.error(error);
         process.exit(1);
     });