@larsgw/formica 0.1.0

package/lib/bin/process-resources.js

@@ -0,0 +1,594 @@
+"use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+var fs_1 = require("fs");
+var path = require("path");
+var child_process_1 = require("child_process");
+var util = require("util");
+var index_1 = require("../index");
+var util_1 = require("./util");
+var DWC_FIELDS = [
+    'scientificNameID',
+    'scientificName',
+    'scientificNameAuthorship',
+    'genericName',
+    'intragenericEpithet',
+    'specificEpithet',
+    'intraspecificEpithet',
+    'taxonRank',
+    'taxonRemarks',
+    'collectionCode',
+    'taxonomicStatus',
+    'acceptedNameUsageID',
+    'acceptedNameUsage',
+    'parentNameUsageID',
+    'parentNameUsage',
+    'kingdom',
+    'phylum',
+    'class',
+    'order',
+    'family',
+    'subfamily',
+    'genus',
+    'subgenus',
+    'higherClassification',
+    'colTaxonID',
+    'gbifTaxonID'
+];
+var DISPLAY_FIELDS = [
+    'scientificNameID',
+    'taxonRank',
+    'scientificName',
+    'taxonomicStatus',
+    'taxonRemarks',
+    'colTaxonID',
+    'gbifTaxonID'
+];
+var GBIF_RANKS = [
+    'kingdom',
+    'phyllum',
+    'class',
+    'order',
+    'family',
+    'genus',
+    'species',
+    'subspecies',
+    'variety'
+];
+var VALID_COMMON_PREFIXES = [
+    'Plantae|Tracheophyta',
+    'Fungi',
+    'Fungi|Ascomycota',
+    'Fungi|Basidiomycota',
+    'Fungi|Zygomycota'
+];
+function runGnverifier(names) {
+    return new Promise(function (resolve, reject) {
+        var proc = (0, child_process_1.spawn)('gnverifier', ['-s', '1,11', '-M']);
+        var stdout = '';
+        proc.stdout.on('data', function (data) { stdout += data; });
+        proc.stderr.pipe(process.stdout);
+        proc.on('close', function (code) {
+            if (code === 0) {
+                resolve(stdout);
+            }
+            else {
+                reject();
+            }
+        });
+        proc.stdin.write(names);
+        proc.stdin.end();
+    });
+}
+function listChangedFiles(directory) {
+    return __awaiter(this, void 0, void 0, function () {
+        var output;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0: return [4 /*yield*/, (0, util_1.runCommand)('git', ['diff', '--name-only', 'HEAD', '--', directory], {
+                        cwd: directory
+                    })];
+                case 1:
+                    output = _a.sent();
+                    return [2 /*return*/, output.trimEnd().split('\n').sort(util_1.numericSort)];
+            }
+        });
+    });
+}
+function getOldFile(file) {
+    return __awaiter(this, void 0, void 0, function () {
+        var options, gitRoot;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    options = {
+                        cwd: path.dirname(file)
+                    };
+                    return [4 /*yield*/, (0, util_1.runCommand)('git', ['rev-parse', '--show-toplevel'], options)];
+                case 1:
+                    gitRoot = (_a.sent()).trim();
+                    return [4 /*yield*/, (0, util_1.runCommand)('git', ['show', 'HEAD:' + path.relative(gitRoot, file)], options)];
+                case 2: return [2 /*return*/, _a.sent()];
+            }
+        });
+    });
+}
+var ResourceProcessor = /** @class */ (function () {
+    function ResourceProcessor(collectionPath) {
+        this.DIR_ROOT = path.resolve(collectionPath);
+        this.DIR_TXT = path.join(this.DIR_ROOT, 'txt');
+        this.DIR_DWC = path.join(this.DIR_ROOT, 'dwc');
+        this.FILE_PROBLEMS = path.join(this.DIR_ROOT, 'problems.csv');
+    }
+    ResourceProcessor.prototype.run = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var input, output, ids, _loop_1, this_1, _i, ids_1, id;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, fs_1.promises.readdir(this.DIR_TXT)];
+                    case 1:
+                        input = _a.sent();
+                        return [4 /*yield*/, fs_1.promises.readdir(this.DIR_DWC)];
+                    case 2:
+                        output = _a.sent();
+                        ids = input
+                            .map(function (file) { return path.basename(file, '.txt'); })
+                            .sort(function (a, b) { return parseInt(a.slice(1)) - parseInt(b.slice(1)); });
+                        _loop_1 = function (id) {
+                            return __generator(this, function (_b) {
+                                switch (_b.label) {
+                                    case 0:
+                                        // Skip existing files
+                                        if (output.some(function (file) { return file.startsWith(id + '-'); })) {
+                                            return [2 /*return*/, "continue"];
+                                        }
+                                        return [4 /*yield*/, this_1.processWork(id)];
+                                    case 1:
+                                        _b.sent();
+                                        return [2 /*return*/];
+                                }
+                            });
+                        };
+                        this_1 = this;
+                        _i = 0, ids_1 = ids;
+                        _a.label = 3;
+                    case 3:
+                        if (!(_i < ids_1.length)) return [3 /*break*/, 6];
+                        id = ids_1[_i];
+                        return [5 /*yield**/, _loop_1(id)];
+                    case 4:
+                        _a.sent();
+                        _a.label = 5;
+                    case 5:
+                        _i++;
+                        return [3 /*break*/, 3];
+                    case 6: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    ResourceProcessor.prototype.runUpdate = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var _i, _a, file, id;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        _i = 0;
+                        return [4 /*yield*/, listChangedFiles(this.DIR_TXT)];
+                    case 1:
+                        _a = _b.sent();
+                        _b.label = 2;
+                    case 2:
+                        if (!(_i < _a.length)) return [3 /*break*/, 5];
+                        file = _a[_i];
+                        id = path.basename(file, '.txt');
+                        return [4 /*yield*/, this.processWork(id, true)];
+                    case 3:
+                        _b.sent();
+                        _b.label = 4;
+                    case 4:
+                        _i++;
+                        return [3 /*break*/, 2];
+                    case 5: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    ResourceProcessor.prototype.processWork = function (id, update) {
+        return __awaiter(this, void 0, void 0, function () {
+            var resources;
+            var _this = this;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.processResources(id, update)];
+                    case 1:
+                        resources = _a.sent();
+                        return [4 /*yield*/, Promise.all(resources.map(function (resource) {
+                                var header = DWC_FIELDS;
+                                var table = [header];
+                                var _loop_2 = function (id_1) {
+                                    var taxon = resource.taxa[id_1];
+                                    table.push(header.map(function (column) { return taxon[column] || ''; }));
+                                };
+                                for (var id_1 in resource.taxa) {
+                                    _loop_2(id_1);
+                                }
+                                return fs_1.promises.writeFile(path.join(_this.DIR_DWC, "".concat(resource.file, ".csv")), index_1.csv.formatCsv(table, ',').trim());
+                            }))];
+                    case 2:
+                        _a.sent();
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    ResourceProcessor.prototype.processResources = function (id, update) {
+        return __awaiter(this, void 0, void 0, function () {
+            var resources, amendedResources, _i, resources_1, resource, _a, results, classifications, _b, _c, _d, _e, source, skip, correct, choice, _f, reason;
+            return __generator(this, function (_g) {
+                switch (_g.label) {
+                    case 0: return [4 /*yield*/, this.processResourceText(id, update)];
+                    case 1:
+                        resources = _g.sent();
+                        amendedResources = [];
+                        _i = 0, resources_1 = resources;
+                        _g.label = 2;
+                    case 2:
+                        if (!(_i < resources_1.length)) return [3 /*break*/, 15];
+                        resource = resources_1[_i];
+                        return [4 /*yield*/, this.processResourceDwc(resource)];
+                    case 3:
+                        _a = _g.sent(), results = _a[0], classifications = _a[1];
+                        _b = classifications;
+                        _c = [];
+                        for (_d in _b)
+                            _c.push(_d);
+                        _e = 0;
+                        _g.label = 4;
+                    case 4:
+                        if (!(_e < _c.length)) return [3 /*break*/, 7];
+                        _d = _c[_e];
+                        if (!(_d in _b)) return [3 /*break*/, 6];
+                        source = _d;
+                        return [4 /*yield*/, this.checkPrefix(resource, classifications, source)];
+                    case 5:
+                        _g.sent();
+                        _g.label = 6;
+                    case 6:
+                        _e++;
+                        return [3 /*break*/, 4];
+                    case 7: return [4 /*yield*/, this.shouldBeSkipped(resource.id)];
+                    case 8:
+                        skip = _g.sent();
+                        if (!!skip) return [3 /*break*/, 13];
+                        correct = this.checkResults(results);
+                        if (!!correct) return [3 /*break*/, 13];
+                        return [4 /*yield*/, (0, util_1.promptForAnswers)("".concat(resource.workId, ": problems found in ").concat(resource.id, ". Skip or retry (s/r)? "), ['s', 'S', 'r', 'R'])];
+                    case 9:
+                        choice = _g.sent();
+                        _f = choice;
+                        switch (_f) {
+                            case 's': return [3 /*break*/, 10];
+                            case 'S': return [3 /*break*/, 10];
+                            case 'r': return [3 /*break*/, 12];
+                            case 'R': return [3 /*break*/, 12];
+                        }
+                        return [3 /*break*/, 13];
+                    case 10: return [4 /*yield*/, (0, util_1.prompt)('Reason for skipping? ')];
+                    case 11:
+                        reason = _g.sent();
+                        fs_1.promises.appendFile(this.FILE_PROBLEMS, index_1.csv.formatCsv([[
+                                resource.workId,
+                                resource.id,
+                                reason
+                            ]]));
+                        console.log("".concat(resource.workId, ": skipping ").concat(resource.id));
+                        return [3 /*break*/, 13];
+                    case 12:
+                        {
+                            console.log("".concat(resource.workId, ": retrying ").concat(resource.id));
+                            return [2 /*return*/, this.processResources(id, update)];
+                        }
+                        _g.label = 13;
+                    case 13:
+                        amendedResources.push(results);
+                        _g.label = 14;
+                    case 14:
+                        _i++;
+                        return [3 /*break*/, 2];
+                    case 15: return [2 /*return*/, amendedResources];
+                }
+            });
+        });
+    };
+    ResourceProcessor.prototype.processResourceText = function (id, update) {
+        return __awaiter(this, void 0, void 0, function () {
+            var filePath, file, old, dwc, _i, _a, file_1, filePath_1, _b, _c, _d, _e, resources, error_1, prefix, file;
+            var _f;
+            return __generator(this, function (_g) {
+                switch (_g.label) {
+                    case 0:
+                        _g.trys.push([0, 10, , 12]);
+                        console.log("".concat(id, ": generating Darwin Core"));
+                        filePath = path.join(this.DIR_TXT, id + '.txt');
+                        return [4 /*yield*/, fs_1.promises.readFile(filePath, 'utf-8')];
+                    case 1:
+                        file = _g.sent();
+                        old = undefined;
+                        if (!update) return [3 /*break*/, 8];
+                        dwc = [];
+                        _i = 0;
+                        return [4 /*yield*/, fs_1.promises.readdir(this.DIR_DWC)];
+                    case 2:
+                        _a = _g.sent();
+                        _g.label = 3;
+                    case 3:
+                        if (!(_i < _a.length)) return [3 /*break*/, 6];
+                        file_1 = _a[_i];
+                        if (!file_1.startsWith(id + '-')) return [3 /*break*/, 5];
+                        filePath_1 = path.join(this.DIR_DWC, file_1);
+                        _c = (_b = dwc).push;
+                        _e = (_d = index_1.csv).parseCsv;
+                        return [4 /*yield*/, getOldFile(filePath_1)];
+                    case 4:
+                        _c.apply(_b, [_e.apply(_d, [_g.sent()])]);
+                        _g.label = 5;
+                    case 5:
+                        _i++;
+                        return [3 /*break*/, 3];
+                    case 6:
+                        _f = {};
+                        return [4 /*yield*/, getOldFile(filePath)];
+                    case 7:
+                        old = (_f.txt = _g.sent(), _f.dwc = dwc, _f);
+                        _g.label = 8;
+                    case 8: return [4 /*yield*/, Promise.resolve().then(function () { return require('../index'); })];
+                    case 9:
+                        resources = (_g.sent()).resources;
+                        return [2 /*return*/, resources.parseTextFile(file, id, old)];
+                    case 10:
+                        error_1 = _g.sent();
+                        console.log(error_1);
+                        return [4 /*yield*/, (0, util_1.prompt)("".concat(id, ": generating Darwin Core failed, retry? "))
+                            // Clear cache to re-import
+                        ];
+                    case 11:
+                        _g.sent();
+                        prefix = path.dirname(require.resolve('../index'));
+                        for (file in require.cache) {
+                            if (file.startsWith(prefix)) {
+                                delete require.cache[file];
+                            }
+                        }
+                        return [2 /*return*/, this.processResourceText(id, update)];
+                    case 12: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    ResourceProcessor.prototype.processResourceDwc = function (resource) {
+        return __awaiter(this, void 0, void 0, function () {
+            var taxa, names, id, name_1, result, classifications, _a, header, matches, _i, matches_1, match, name_2, source, id, classification, loirId, taxon, results;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        console.log("".concat(resource.workId, ": matching ").concat(resource.id));
+                        taxa = {};
+                        names = [];
+                        for (id in resource.taxa) {
+                            name_1 = resource.taxa[id].scientificName;
+                            if (!taxa[name_1]) {
+                                taxa[name_1] = {};
+                            }
+                            taxa[name_1][id] = __assign({}, resource.taxa[id]);
+                            names.push(name_1);
+                        }
+                        return [4 /*yield*/, runGnverifier(names.join('\n'))];
+                    case 1:
+                        result = _b.sent();
+                        classifications = { '1': [], '11': [] };
+                        _a = index_1.csv.parseCsv(result), header = _a[0], matches = _a.slice(1);
+                        for (_i = 0, matches_1 = matches; _i < matches_1.length; _i++) {
+                            match = matches_1[_i];
+                            name_2 = match[header.indexOf('ScientificName')];
+                            source = match[header.indexOf('DataSourceId')];
+                            id = match[header.indexOf('TaxonId')];
+                            classification = match[header.indexOf('ClassificationPath')];
+                            for (loirId in taxa[name_2]) {
+                                taxon = taxa[name_2][loirId];
+                                if (source === '1' && !taxon.colTaxonID) {
+                                    taxon.colTaxonID = id;
+                                    classifications[source].push([taxon, classification]);
+                                }
+                                if (source === '11' && GBIF_RANKS.includes(taxon.taxonRank) && !taxon.gbifTaxonID) {
+                                    taxon.gbifTaxonID = id;
+                                    classifications[source].push([taxon, classification]);
+                                }
+                            }
+                        }
+                        results = __assign(__assign({}, resource), { taxa: Object.fromEntries(Object.values(resource.taxa).map(function (taxon) { return [
+                                taxon.scientificNameID,
+                                taxa[taxon.scientificName][taxon.scientificNameID]
+                            ]; })) });
+                        return [2 /*return*/, [results, classifications]];
+                }
+            });
+        });
+    };
+    ResourceProcessor.prototype.checkPrefix = function (resource, classifications, source) {
+        return __awaiter(this, void 0, void 0, function () {
+            var lists, prefix, _i, _a, _b, taxon, list, parts, i, choice;
+            return __generator(this, function (_c) {
+                switch (_c.label) {
+                    case 0:
+                        lists = classifications[source];
+                        if (!lists.length) {
+                            return [2 /*return*/];
+                        }
+                        prefix = lists[0][1].split('|');
+                        _i = 0, _a = lists.slice(1);
+                        _c.label = 1;
+                    case 1:
+                        if (!(_i < _a.length)) return [3 /*break*/, 9];
+                        _b = _a[_i], taxon = _b[0], list = _b[1];
+                        parts = list.split('|');
+                        i = 0;
+                        _c.label = 2;
+                    case 2:
+                        if (!(i < parts.length)) return [3 /*break*/, 8];
+                        if (!(parts[i] !== prefix[i] && i < 3)) return [3 /*break*/, 7];
+                        choice = void 0;
+                        if (!(source === '1' || taxon.taxonomicStatus !== 'accepted')) return [3 /*break*/, 3];
+                        choice = 'd';
+                        return [3 /*break*/, 6];
+                    case 3:
+                        if (!VALID_COMMON_PREFIXES.includes(prefix.slice(0, i).join('|'))) return [3 /*break*/, 4];
+                        choice = 'k';
+                        return [3 /*break*/, 6];
+                    case 4:
+                        console.log("".concat(resource.workId, ": source ").concat(source, " results in short prefix \"").concat(prefix.slice(0, i).join('|'), "\" (").concat(i, " taxa)"));
+                        console.log("  taxon: ".concat(taxon.scientificNameID, " \"").concat(taxon.scientificName, "\""));
+                        console.log("  class: ".concat(parts.join('|')));
+                        console.log("  prefx: ".concat(prefix.join('|')));
+                        return [4 /*yield*/, (0, util_1.promptForAnswers)("  Keep or delete (k/d)? ", ['k', 'K', 'd', 'D'])];
+                    case 5:
+                        choice = _c.sent();
+                        _c.label = 6;
+                    case 6:
+                        switch (choice) {
+                            case 'k':
+                            case 'K': {
+                                console.log("  keeping...");
+                                break;
+                            }
+                            case 'd':
+                            case 'D': {
+                                console.log("  deleting...");
+                                if (source === '1') {
+                                    delete taxon.colTaxonID;
+                                }
+                                else if (source === '11') {
+                                    delete taxon.gbifTaxonID;
+                                }
+                                break;
+                            }
+                        }
+                        return [3 /*break*/, 8];
+                    case 7:
+                        i++;
+                        return [3 /*break*/, 2];
+                    case 8:
+                        _i++;
+                        return [3 /*break*/, 1];
+                    case 9: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    ResourceProcessor.prototype.checkResults = function (resource) {
+        var correct = true;
+        var missing = [];
+        for (var id in resource.taxa) {
+            var taxon = resource.taxa[id];
+            if (taxon.taxonomicStatus !== 'accepted') {
+                continue;
+            }
+            var missingCol = false; // !taxon.colTaxonID
+            var missingGbif = GBIF_RANKS.includes(taxon.taxonRank) && !taxon.gbifTaxonID;
+            if (missingCol || missingGbif) {
+                correct = false;
+                missing.push(taxon);
+            }
+        }
+        if (missing.length) {
+            console.table(missing, DISPLAY_FIELDS);
+        }
+        return correct;
+    };
+    ResourceProcessor.prototype.shouldBeSkipped = function (id) {
+        return __awaiter(this, void 0, void 0, function () {
+            var problems, _a, _b;
+            return __generator(this, function (_c) {
+                switch (_c.label) {
+                    case 0:
+                        _b = (_a = index_1.csv).parseCsv;
+                        return [4 /*yield*/, fs_1.promises.readFile(this.FILE_PROBLEMS, 'utf8')];
+                    case 1:
+                        problems = _b.apply(_a, [_c.sent()]);
+                        return [2 /*return*/, problems.some(function (_a) {
+                                var _work = _a[0], resource = _a[1], _problem = _a[2];
+                                return resource === id;
+                            })];
+                }
+            });
+        });
+    };
+    return ResourceProcessor;
+}());
+function main() {
+    var args = util.parseArgs({
+        options: {
+            update: {
+                type: 'boolean',
+                short: 'u'
+            }
+        },
+        allowPositionals: true
+    });
+    var processor = new ResourceProcessor(args.positionals[0]);
+    process.on('exit', function () {
+        process.stdout.write('\n');
+    });
+    var task = args.values.update ? processor.runUpdate() : processor.run();
+    task.catch(function (error) {
+        console.error(error);
+        process.exit(1);
+    });
+}
+main();

package/lib/bin/util.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Comparison function to sort strings with numerical components.
+ */
+export declare function numericSort(a: string, b: string): number;
+/**
+ * Create a CLI prompt and return any answer.
+ */
+export declare function prompt(question: string): Promise<string>;
+/**
+ * Create a CLI prompt and only accept certain answers.
+ */
+export declare function promptForAnswers(question: string, answers: string[]): Promise<string>;
+/**
+ * Run a command and return stdout.
+ */
+export declare function runCommand(command: string, args: string[], options?: Record<string, unknown>): Promise<string>;