@larsgw/formica 0.4.3 → 0.5.0

package/CHANGELOG.md

@@ -1,3 +1,35 @@
+# [0.5.0](https://github.com/identification-resources/formica/compare/v0.4.3...v0.5.0) (2023-10-24)
+
+
+### Bug Fixes
+
+* **catalog:** make validation more robust ([522db8a](https://github.com/identification-resources/formica/commit/522db8ad428c2d4645569a7f541fd203f5d628e8))
+* **csv:** fix csv formatting ([064e5e5](https://github.com/identification-resources/formica/commit/064e5e57b2a642d22850501ab2a0b95879779b85))
+* **resources:** do not count abbreviations as species ([ac0cc5a](https://github.com/identification-resources/formica/commit/ac0cc5a7cfd1f68df9b8d1d1c543a2f593839daa))
+* **resources:** fix typos in output column names ([0d02823](https://github.com/identification-resources/formica/commit/0d0282378b21e92fbe028a53a9d121f42e791cc5))
+
+
+### Features
+
+* **resources:** encode names with spaces with underscores ([9c7efd4](https://github.com/identification-resources/formica/commit/9c7efd4daf80660be98f2fec8ef9faa5d5b42d67))
+* **resources:** include verbatimIdentification column ([4a2e920](https://github.com/identification-resources/formica/commit/4a2e9201467347e900f5690d2301a3b231ded822))
+* **resources:** move "scope" metadata to "catalog" ([286c5b2](https://github.com/identification-resources/formica/commit/286c5b223ce41336f3ebd63c1797403aad21aef6))
+* **resources:** update processor configuration ([64daa26](https://github.com/identification-resources/formica/commit/64daa26d70c96df24298c7b78f2b93ef0f5b2a48))
+
+
+### BREAKING CHANGES
+
+* **resources:** - "intragenericEpithet" is now "infragenericEpithet"
+- "intraspecificEpithet" is now "infraspecificEpithet"
+* **resources:** Adds additional column, between "higherClassification" and "colTaxonID".
+* **resources:** - Instead of "-u, --update" use "-s modified"
+- Instead of "--update-mapping" use "-s all"
+* **resources:** Index will no longer contain "scope" array. "scope" field in YAML header
+does no longer pass validation. Use the "taxon_scope" and "scope" fields
+in "catalog" instead.
+
+
+
 ## [0.4.3](https://github.com/identification-resources/formica/compare/v0.4.2...v0.4.3) (2023-09-30)
 
 

package/lib/bin/process-resources.d.ts

@@ -1,2 +1,6 @@
 #!/usr/bin/env node
-export {};
+export declare enum ResourceProcessorSource {
+    All = "all",
+    Unprocessed = "unprocessed",
+    Modified = "modified"
+}

package/lib/bin/process-resources.js

@@ -48,20 +48,27 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
     }
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.ResourceProcessorSource = void 0;
 var fs_1 = require("fs");
 var path = require("path");
 var child_process_1 = require("child_process");
 var util = require("util");
 var index_1 = require("../index");
 var util_1 = require("./util");
+var ResourceProcessorSource;
+(function (ResourceProcessorSource) {
+    ResourceProcessorSource["All"] = "all";
+    ResourceProcessorSource["Unprocessed"] = "unprocessed";
+    ResourceProcessorSource["Modified"] = "modified";
+})(ResourceProcessorSource = exports.ResourceProcessorSource || (exports.ResourceProcessorSource = {}));
 var DWC_FIELDS = [
     'scientificNameID',
     'scientificName',
     'scientificNameAuthorship',
     'genericName',
-    'intragenericEpithet',
+    'infragenericEpithet',
     'specificEpithet',
-    'intraspecificEpithet',
+    'infraspecificEpithet',
     'taxonRank',
     'taxonRemarks',
     'collectionCode',
@@ -79,6 +86,7 @@ var DWC_FIELDS = [
     'genus',
     'subgenus',
     'higherClassification',
+    'verbatimIdentification',
     'colTaxonID',
     'gbifTaxonID',
     'colAcceptedTaxonID',
@@ -122,6 +130,36 @@ function runGnverifier(names) {
         proc.stdin.end();
     });
 }
+function listFiles(directory) {
+    return __awaiter(this, void 0, void 0, function () {
+        var input;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0: return [4 /*yield*/, fs_1.promises.readdir(directory)];
+                case 1:
+                    input = _a.sent();
+                    return [2 /*return*/, input.map(function (file) { return path.basename(file, '.txt'); }).sort(util_1.numericSort)];
+            }
+        });
+    });
+}
+function listUnprocessedFiles(directory, outputDirectory) {
+    return __awaiter(this, void 0, void 0, function () {
+        var input, output, _a;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
+                case 0: return [4 /*yield*/, listFiles(directory)];
+                case 1:
+                    input = _b.sent();
+                    _a = Set.bind;
+                    return [4 /*yield*/, fs_1.promises.readdir(outputDirectory)];
+                case 2:
+                    output = new (_a.apply(Set, [void 0, _b.sent()]))();
+                    return [2 /*return*/, input.filter(function (file) { return output.has(file + '-1'); })];
+            }
+        });
+    });
+}
 function listChangedFiles(directory) {
     return __awaiter(this, void 0, void 0, function () {
         var output;
@@ -132,7 +170,7 @@ function listChangedFiles(directory) {
                     })];
                 case 1:
                     output = _a.sent();
-                    return [2 /*return*/, output.trimEnd().split('\n').sort(util_1.numericSort)];
+                    return [2 /*return*/, output.trimEnd().split('\n').map(function (file) { return path.basename(file, '.txt'); }).sort(util_1.numericSort)];
             }
         });
     });
@@ -162,72 +200,23 @@ var ResourceProcessor = /** @class */ (function () {
         this.DIR_DWC = path.join(this.DIR_ROOT, 'dwc');
         this.FILE_PROBLEMS = path.join(this.DIR_ROOT, 'problems.csv');
     }
-    ResourceProcessor.prototype.run = function () {
+    ResourceProcessor.prototype.run = function (source, config) {
         return __awaiter(this, void 0, void 0, function () {
-            var input, output, ids, _loop_1, this_1, _i, ids_1, id;
+            var ids, _i, ids_1, id;
             return __generator(this, function (_a) {
                 switch (_a.label) {
-                    case 0: return [4 /*yield*/, fs_1.promises.readdir(this.DIR_TXT)];
+                    case 0: return [4 /*yield*/, this.listWorks(source)];
                     case 1:
-                        input = _a.sent();
-                        return [4 /*yield*/, fs_1.promises.readdir(this.DIR_DWC)];
-                    case 2:
-                        output = _a.sent();
-                        ids = input
-                            .map(function (file) { return path.basename(file, '.txt'); })
-                            .sort(function (a, b) { return parseInt(a.slice(1)) - parseInt(b.slice(1)); });
-                        _loop_1 = function (id) {
-                            return __generator(this, function (_b) {
-                                switch (_b.label) {
-                                    case 0:
-                                        // Skip existing files
-                                        if (output.some(function (file) { return file.startsWith(id + '-'); })) {
-                                            return [2 /*return*/, "continue"];
-                                        }
-                                        return [4 /*yield*/, this_1.processWork(id)];
-                                    case 1:
-                                        _b.sent();
-                                        return [2 /*return*/];
-                                }
-                            });
-                        };
-                        this_1 = this;
+                        ids = _a.sent();
                         _i = 0, ids_1 = ids;
-                        _a.label = 3;
-                    case 3:
-                        if (!(_i < ids_1.length)) return [3 /*break*/, 6];
-                        id = ids_1[_i];
-                        return [5 /*yield**/, _loop_1(id)];
-                    case 4:
-                        _a.sent();
-                        _a.label = 5;
-                    case 5:
-                        _i++;
-                        return [3 /*break*/, 3];
-                    case 6: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    ResourceProcessor.prototype.runUpdate = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            var _i, _a, file, id;
-            return __generator(this, function (_b) {
-                switch (_b.label) {
-                    case 0:
-                        _i = 0;
-                        return [4 /*yield*/, listChangedFiles(this.DIR_TXT)];
-                    case 1:
-                        _a = _b.sent();
-                        _b.label = 2;
+                        _a.label = 2;
                     case 2:
-                        if (!(_i < _a.length)) return [3 /*break*/, 5];
-                        file = _a[_i];
-                        id = path.basename(file, '.txt');
-                        return [4 /*yield*/, this.processWork(id, true)];
+                        if (!(_i < ids_1.length)) return [3 /*break*/, 5];
+                        id = ids_1[_i];
+                        return [4 /*yield*/, this.processWork(id, config)];
                     case 3:
-                        _b.sent();
-                        _b.label = 4;
+                        _a.sent();
+                        _a.label = 4;
                     case 4:
                         _i++;
                         return [3 /*break*/, 2];
@@ -236,52 +225,41 @@ var ResourceProcessor = /** @class */ (function () {
             });
         });
     };
-    ResourceProcessor.prototype.runMappingsUpdate = function () {
+    ResourceProcessor.prototype.listWorks = function (source) {
         return __awaiter(this, void 0, void 0, function () {
-            var input, ids, _i, ids_2, id;
             return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0: return [4 /*yield*/, fs_1.promises.readdir(this.DIR_TXT)];
-                    case 1:
-                        input = _a.sent();
-                        ids = input
-                            .map(function (file) { return path.basename(file, '.txt'); })
-                            .sort(function (a, b) { return parseInt(a.slice(1)) - parseInt(b.slice(1)); });
-                        _i = 0, ids_2 = ids;
-                        _a.label = 2;
-                    case 2:
-                        if (!(_i < ids_2.length)) return [3 /*break*/, 5];
-                        id = ids_2[_i];
-                        return [4 /*yield*/, this.processWork(id, true)];
-                    case 3:
-                        _a.sent();
-                        _a.label = 4;
-                    case 4:
-                        _i++;
-                        return [3 /*break*/, 2];
-                    case 5: return [2 /*return*/];
+                switch (source) {
+                    case ResourceProcessorSource.All:
+                        return [2 /*return*/, listFiles(this.DIR_TXT)];
+                    case ResourceProcessorSource.Unprocessed:
+                        return [2 /*return*/, listUnprocessedFiles(this.DIR_TXT, this.DIR_DWC)];
+                    case ResourceProcessorSource.Modified:
+                        return [2 /*return*/, listChangedFiles(this.DIR_TXT)];
+                    default:
+                        return [2 /*return*/, []];
                 }
+                return [2 /*return*/];
             });
         });
     };
-    ResourceProcessor.prototype.processWork = function (id, update) {
+    ResourceProcessor.prototype.processWork = function (id, config) {
         return __awaiter(this, void 0, void 0, function () {
             var resources;
             var _this = this;
             return __generator(this, function (_a) {
                 switch (_a.label) {
-                    case 0: return [4 /*yield*/, this.processResources(id, update)];
+                    case 0: return [4 /*yield*/, this.processResources(id, config)];
                     case 1:
                         resources = _a.sent();
                         return [4 /*yield*/, Promise.all(resources.map(function (resource) {
                                 var header = DWC_FIELDS;
                                 var table = [header];
-                                var _loop_2 = function (id_1) {
+                                var _loop_1 = function (id_1) {
                                     var taxon = resource.taxa[id_1];
                                     table.push(header.map(function (column) { return taxon[column] || ''; }));
                                 };
                                 for (var id_1 in resource.taxa) {
-                                    _loop_2(id_1);
+                                    _loop_1(id_1);
                                 }
                                 return fs_1.promises.writeFile(path.join(_this.DIR_DWC, "".concat(resource.file, ".csv")), index_1.csv.formatCsv(table, ',').trim());
                             }))];
@@ -292,12 +270,12 @@ var ResourceProcessor = /** @class */ (function () {
             });
         });
     };
-    ResourceProcessor.prototype.processResources = function (id, update) {
+    ResourceProcessor.prototype.processResources = function (id, config) {
         return __awaiter(this, void 0, void 0, function () {
             var resources, amendedResources, _i, resources_1, resource, results, skip, correct, choice, _a, reason;
             return __generator(this, function (_b) {
                 switch (_b.label) {
-                    case 0: return [4 /*yield*/, this.processResourceText(id, update)];
+                    case 0: return [4 /*yield*/, this.processResourceText(id, config)];
                     case 1:
                         resources = _b.sent();
                         amendedResources = [];
@@ -306,7 +284,7 @@ var ResourceProcessor = /** @class */ (function () {
                     case 2:
                         if (!(_i < resources_1.length)) return [3 /*break*/, 11];
                         resource = resources_1[_i];
-                        return [4 /*yield*/, this.processResourceDwc(resource)];
+                        return [4 /*yield*/, this.processResourceDwc(resource, config)];
                     case 3:
                         results = _b.sent();
                         return [4 /*yield*/, this.shouldBeSkipped(resource.id)];
@@ -339,7 +317,7 @@ var ResourceProcessor = /** @class */ (function () {
                     case 8:
                         {
                             console.log("".concat(resource.workId, ": retrying ").concat(resource.id));
-                            return [2 /*return*/, this.processResources(id, update)];
+                            return [2 /*return*/, this.processResources(id, config)];
                         }
                         _b.label = 9;
                     case 9:
@@ -353,7 +331,7 @@ var ResourceProcessor = /** @class */ (function () {
             });
         });
     };
-    ResourceProcessor.prototype.processResourceText = function (id, update) {
+    ResourceProcessor.prototype.processResourceText = function (id, config) {
         return __awaiter(this, void 0, void 0, function () {
             var filePath, file, old, dwc, _i, _a, file_1, filePath_1, _b, _c, _d, _e, resources, error_1, prefix, file;
             var _f;
@@ -367,7 +345,7 @@ var ResourceProcessor = /** @class */ (function () {
                     case 1:
                         file = _g.sent();
                         old = undefined;
-                        if (!update) return [3 /*break*/, 8];
+                        if (!config.update) return [3 /*break*/, 8];
                         dwc = [];
                         _i = 0;
                         return [4 /*yield*/, fs_1.promises.readdir(this.DIR_DWC)];
@@ -412,19 +390,43 @@ var ResourceProcessor = /** @class */ (function () {
                                 delete require.cache[file];
                             }
                         }
-                        return [2 /*return*/, this.processResourceText(id, update)];
+                        return [2 /*return*/, this.processResourceText(id, config)];
                     case 12: return [2 /*return*/];
                 }
             });
         });
     };
-    ResourceProcessor.prototype.processResourceDwc = function (resource) {
+    ResourceProcessor.prototype.processResourceDwc = function (resource, config) {
         return __awaiter(this, void 0, void 0, function () {
-            var filteredResults, taxonNames, names, id, name_1, result, _i, _a, results, _b, name_2, matches, _c, matches_1, match, source, currentRank, _d, _e, loirId, taxon, _f, amendResource, groupNameMatches, groupedNameMatches, amendedResource, _g, _h, _j, _k, source, matches;
-            return __generator(this, function (_l) {
-                switch (_l.label) {
+            var file, _a, header_1, rows, _b, _c, _i, rows_1, row, oldTaxon, taxon, filteredResults, taxonNames, names, id, name_1, result, _d, _e, results, _f, name_2, matches, _g, matches_1, match, source, currentRank, _h, _j, loirId, taxon, _k, amendResource, groupNameMatches, groupedNameMatches, amendedResource, _l, _m, _o, _p, source, matches;
+            return __generator(this, function (_q) {
+                switch (_q.label) {
                     case 0:
                         console.log("".concat(resource.workId, ": matching ").concat(resource.id));
+                        if (!!config.updateMappings) return [3 /*break*/, 3];
+                        file = path.join(this.DIR_DWC, resource.file + '.csv');
+                        if (!(0, fs_1.existsSync)(file)) return [3 /*break*/, 2];
+                        _c = (_b = index_1.csv).parseCsv;
+                        return [4 /*yield*/, fs_1.promises.readFile(file, 'utf-8')];
+                    case 1:
+                        _a = _c.apply(_b, [_q.sent()]), header_1 = _a[0], rows = _a.slice(1);
+                        for (_i = 0, rows_1 = rows; _i < rows_1.length; _i++) {
+                            row = rows_1[_i];
+                            oldTaxon = row.reduce(function (taxon, value, index) {
+                                taxon[header_1[index]] = value;
+                                return taxon;
+                            }, {});
+                            taxon = resource.taxa[oldTaxon.scientificNameID];
+                            if (taxon) {
+                                taxon.colTaxonID = oldTaxon.colTaxonID;
+                                taxon.colAcceptedTaxonID = oldTaxon.colAcceptedTaxonID;
+                                taxon.gbifTaxonID = oldTaxon.gbifTaxonID;
+                                taxon.gbifAcceptedTaxonID = oldTaxon.gbifAcceptedTaxonID;
+                            }
+                        }
+                        _q.label = 2;
+                    case 2: return [2 /*return*/, resource];
+                    case 3:
                         filteredResults = {};
                         taxonNames = {};
                         names = new Set();
@@ -438,16 +440,16 @@ var ResourceProcessor = /** @class */ (function () {
                             filteredResults[id] = [];
                         }
                         return [4 /*yield*/, runGnverifier(Array.from(names).join('\n'))];
-                    case 1:
-                        result = _l.sent();
-                        for (_i = 0, _a = result.trim().split('\n'); _i < _a.length; _i++) {
-                            results = _a[_i];
-                            _b = JSON.parse(results), name_2 = _b.name, matches = _b.results;
+                    case 4:
+                        result = _q.sent();
+                        for (_d = 0, _e = result.trim().split('\n'); _d < _e.length; _d++) {
+                            results = _e[_d];
+                            _f = JSON.parse(results), name_2 = _f.name, matches = _f.results;
                             if (!matches) {
                                 continue;
                             }
-                            for (_c = 0, matches_1 = matches; _c < matches_1.length; _c++) {
-                                match = matches_1[_c];
+                            for (_g = 0, matches_1 = matches; _g < matches_1.length; _g++) {
+                                match = matches_1[_g];
                                 source = match.dataSourceId;
                                 currentRank = match.classificationRanks.split('|').pop();
                                 if (match.scoreDetails.cardinalityScore === 0) {
@@ -458,8 +460,8 @@ var ResourceProcessor = /** @class */ (function () {
                                     // GBIF species like "Nomada spec"
                                     continue;
                                 }
-                                for (_d = 0, _e = taxonNames[name_2]; _d < _e.length; _d++) {
-                                    loirId = _e[_d];
+                                for (_h = 0, _j = taxonNames[name_2]; _h < _j.length; _h++) {
+                                    loirId = _j[_h];
                                     taxon = resource.taxa[loirId];
                                     if (source === 11 && !GBIF_RANKS.includes(taxon.taxonRank)) {
                                         // Exclude GBIF matches for ranks that are not in GBIF
@@ -483,30 +485,30 @@ var ResourceProcessor = /** @class */ (function () {
                             }
                         }
                         return [4 /*yield*/, Promise.resolve().then(function () { return require('../index'); })];
-                    case 2:
-                        _f = (_l.sent()).taxonNames, amendResource = _f.amendResource, groupNameMatches = _f.groupNameMatches;
+                    case 5:
+                        _k = (_q.sent()).taxonNames, amendResource = _k.amendResource, groupNameMatches = _k.groupNameMatches;
                         groupedNameMatches = groupNameMatches(filteredResults);
                         amendedResource = __assign(__assign({}, resource), { taxa: __assign({}, resource.taxa) });
-                        _g = groupedNameMatches;
-                        _h = [];
-                        for (_j in _g)
-                            _h.push(_j);
-                        _k = 0;
-                        _l.label = 3;
-                    case 3:
-                        if (!(_k < _h.length)) return [3 /*break*/, 6];
-                        _j = _h[_k];
-                        if (!(_j in _g)) return [3 /*break*/, 5];
-                        source = _j;
+                        _l = groupedNameMatches;
+                        _m = [];
+                        for (_o in _l)
+                            _m.push(_o);
+                        _p = 0;
+                        _q.label = 6;
+                    case 6:
+                        if (!(_p < _m.length)) return [3 /*break*/, 9];
+                        _o = _m[_p];
+                        if (!(_o in _l)) return [3 /*break*/, 8];
+                        source = _o;
                         return [4 /*yield*/, this.selectPrefixes(resource, groupedNameMatches, source)];
-                    case 4:
-                        matches = _l.sent();
+                    case 7:
+                        matches = _q.sent();
                         amendResource(amendedResource, source, matches);
-                        _l.label = 5;
-                    case 5:
-                        _k++;
-                        return [3 /*break*/, 3];
-                    case 6: return [2 /*return*/, amendedResource];
+                        _q.label = 8;
+                    case 8:
+                        _p++;
+                        return [3 /*break*/, 6];
+                    case 9: return [2 /*return*/, amendedResource];
                 }
             });
         });
@@ -635,12 +637,14 @@ var ResourceProcessor = /** @class */ (function () {
 function main() {
     var args = util.parseArgs({
         options: {
-            update: {
-                type: 'boolean',
-                short: 'u'
+            source: {
+                type: 'string',
+                short: 's',
+                default: 'unprocessed'
             },
-            'update-mappings': {
-                type: 'boolean'
+            'keep-mappings': {
+                type: 'boolean',
+                short: 'k'
             }
         },
         allowPositionals: true
@@ -649,17 +653,12 @@ function main() {
     process.on('exit', function () {
         process.stdout.write('\n');
     });
-    var task;
-    if (args.values.update) {
-        task = processor.runUpdate();
-    }
-    else if (args.values['update-mappings']) {
-        task = processor.runMappingsUpdate();
-    }
-    else {
-        task = processor.run();
-    }
-    task.catch(function (error) {
+    var source = args.values.source;
+    var config = {
+        update: source !== 'unprocessed',
+        updateMappings: !args.values['keep-mappings']
+    };
+    processor.run(source, config).catch(function (error) {
         console.error(error);
         process.exit(1);
     });

package/lib/catalog/value.js

@@ -22,10 +22,10 @@ exports.FORMATS = {
 };
 exports.CHECK = {
     MULTILANG: function (entry) {
-        return entry.language.length > 1;
+        return Array.isArray(entry.language) && entry.language.length > 1;
     },
     ISBN: function (entry) {
-        if (entry.ISBN.length < 2) {
+        if (!Array.isArray(entry.ISBN) || entry.ISBN.length < 2) {
             return false;
         }
         var a = entry.ISBN[0].length === 10;

package/lib/csv.js

@@ -25,7 +25,7 @@ function formatCsv(table, delimiter) {
         .map(function (row) {
         return row.map(function (value) {
             if (/["\n]/.test(value) || value.includes(delimiter)) {
-                return "\"".concat(value.replace(/"/, '"""'), "\"");
+                return "\"".concat(value.replace(/"/g, '""'), "\"");
             }
             else {
                 return value;

package/lib/resources/parse-text.js

@@ -13,6 +13,7 @@ var __assign = (this && this.__assign) || function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.parseFileHeader = exports.parseFile = void 0;
 var yaml = require("js-yaml");
+var work_1 = require("../catalog/tables/work");
 var diff_resource_1 = require("./diff-resource");
 var RANKS = [
     'class',
@@ -122,9 +123,9 @@ var NAME_PATTERN = new RegExp('^' +
  *   $1 genus+subgenus (+ trailing space): (?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?
  *     $1.1 genus: ([A-Z]\S+)
  *     $1.2 subgenus: (?:\(([A-Z]\S+?)\) )?
- *   $2 species: ((?:x )?[a-z]\S+)
+ *   $2 species: ((?:x )?[a-z][^\s.]+)
  */
-var BINAME_PATTERN = /^(?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?((?:x )?[a-z]\S+)(?= |$)/;
+var BINAME_PATTERN = /^(?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?((?:x )?[a-z][^\s.]+)(?= |$)/;
 function compareRanks(a, b) {
     return RANKS.indexOf(a) - RANKS.indexOf(b);
 }
@@ -174,55 +175,62 @@ function parseName(name, rank, parent) {
     if (/^\[(_|\d+)\] /.test(name)) {
         name = name.replace(/^\[(_|\d+)\] /, '');
     }
+    // Set verbatim identification after subsequent syntax is removed.
+    item.verbatimIdentification = name;
     // Parent context is used for parsing and formatting binomial names.
-    var parentContext = __assign({}, parent);
-    if (parent.incorrect) {
-        parentContext.incorrect = __assign({}, parent.incorrect);
-    }
-    // The parent context should be amended in the two cases where binomial names
-    // are truly accepted: synonyms and species (and below) without parents (resp.
-    // genera and genera and species) to provide parts of the name.
+    // For formatting, it needs to match external databases (i.e. be correct).
+    // For parsing, it needs to match the current file. If relevant parents
+    // (i.e. genus, species) had mistakes that were corrected, the uncorrected
+    // genus and species names need to be used.
+    var parentContext = {
+        genus: parent.genus,
+        subgenus: parent.subgenus,
+        specificEpithet: parent.specificEpithet,
+        incorrect: {
+            genus: parent.incorrect && parent.incorrect.genus,
+            specificEpithet: parent.incorrect && parent.incorrect.specificEpithet
+        }
+    };
+    // Both contexts should be amended in the two cases where binomial names
+    // are fully used: (1) synonyms and (2) multinomial taxa without parents to
+    // provide parts of the name (e.g. bare species without a genus parent, or
+    // even subspecies without a species or genus parent).
     if (isSynonym || !parentContext.genus || (compareRanks('species', rank) < 0 && !parentContext.specificEpithet)) {
         var _a = name.match(BINAME_PATTERN) || [], genus = _a[1], subgenus = _a[2], species = _a[3];
         if (genus) {
-            parentContext.genus = capitalize(genus);
-            if (parentContext.incorrect)
-                parentContext.incorrect.genus = capitalize(genus);
+            parentContext.genus = parentContext.incorrect.genus = capitalize(genus);
         }
         if (subgenus) {
             parentContext.subgenus = capitalize(subgenus);
-            if (parentContext.incorrect)
-                parentContext.incorrect.subgenus = capitalize(subgenus);
         }
         else if (genus) {
-            // If a genus is given but no subgenus, remove it from the parent context
+            // If a genus is given but no subgenus, remove any existing subgenus
+            // from the parent context.
             delete parentContext.subgenus;
-            if (parentContext.incorrect)
-                delete parentContext.incorrect.subgenus;
         }
         if (species) {
-            parentContext.specificEpithet = species;
-            if (parentContext.incorrect)
-                parentContext.incorrect.specificEpithet = species;
+            parentContext.specificEpithet = parentContext.incorrect.specificEpithet = species;
         }
     }
     // In taxa of group, species or lower, the name should just contain the
-    // (inter)specific epithet and the author information & remarks when processing
+    // (infra)specific epithet and the author information & remarks when processing
     // further.
     if (compareRanks('group', rank) <= 0) {
-        var parseContext = parentContext.incorrect || parentContext;
-        if (!parseContext.genus) {
-            parseContext.genus = name.split(' ', 1)[0];
-        }
-        var genusPrefix = new RegExp("^".concat(parentContext.genus, " (\\(.*?\\) )?"), 'i');
-        if (name[0] === parentContext.genus[0]) {
-            name = name.replace(genusPrefix, '');
-        }
+        // Remove genus
+        var genus = parentContext.incorrect.genus || parentContext.genus || '';
+        if (name[0] === genus[0] && name.toLowerCase().startsWith(genus.toLowerCase() + ' ')) {
+            name = name.slice(genus.length + 1);
+        }
+        // Remove subgenus
+        name = name.replace(/^\(.*?\) /, '');
+        // Infraspecific taxa
         if (compareRanks('species', rank) < 0) {
-            var speciesPrefix = parseContext.specificEpithet + ' ';
-            if (name.startsWith(speciesPrefix)) {
-                name = name.slice(speciesPrefix.length);
+            // Remove specific epithet
+            var species = parentContext.incorrect.specificEpithet || parentContext.specificEpithet || '';
+            if (name.startsWith(species + ' ')) {
+                name = name.slice(species.length + 1);
             }
+            // Remove rank abbreviations
             name = name.replace(/^(st|r|ab|f|var|ssp|subsp)\. /, '');
         }
     }
@@ -236,6 +244,11 @@ function parseName(name, rank, parent) {
     if (!nameParts) {
         throw new Error("Taxon \"".concat(name, "\" could not be parsed"));
     }
+    // To encode old names with spaces (e.g. "Orsillus pini canariensis Lindberg, 1953")
+    // underscores are used, which are replaced here.
+    if (nameParts[1].includes('_')) {
+        nameParts[1] = nameParts[1].replace(/_/g, ' ');
+    }
     var _ = nameParts[0], taxon = nameParts[1], _b = nameParts[2], citation = _b === void 0 ? '' : _b, notes = nameParts[3];
     item.scientificNameAuthorship = capitalizeAuthors(citation);
     item.taxonRemarks = notes;
@@ -284,21 +297,21 @@ function parseName(name, rank, parent) {
         item.genericName = parentContext.genus;
         item.infragenericEpithet = parentContext.subgenus;
         item.specificEpithet = parentContext.specificEpithet;
-        item.intraspecificEpithet = taxon.toLowerCase();
+        item.infraspecificEpithet = taxon.toLowerCase();
         // If possible, names below species should have abbreviations for ranks,
         // like "subsp."
         var nameParts_1 = [
             item.genericName,
             item.specificEpithet,
-            item.intraspecificEpithet
+            item.infraspecificEpithet
         ];
         if (item.taxonRank in RANK_LABELS) {
             nameParts_1.splice(2, 0, RANK_LABELS[item.taxonRank]);
         }
         item.scientificName = nameParts_1.join(' ');
-        if (item.intraspecificEpithet !== taxon) {
+        if (item.infraspecificEpithet !== taxon) {
             console.log(item, taxon);
-            throw new Error("Intraspecific epithet should be lowercase: \"".concat(taxon, "\""));
+            throw new Error("Infraspecific epithet should be lowercase: \"".concat(taxon, "\""));
         }
     }
     // Re-add authorship information
@@ -335,15 +348,8 @@ function parseHeader(header) {
     else {
         levels = config.levels;
     }
-    var scope;
-    if (!('scope' in config)) {
-        scope = [];
-    }
-    else if (!Array.isArray(config.scope)) {
-        throw new SyntaxError('"scope" should be an array');
-    }
-    else {
-        scope = config.scope;
+    if ('scope' in config) {
+        throw new SyntaxError('"scope" data should go in "catalog"');
     }
     // No taxon ranks
     if (levels.length === 0) {
@@ -354,9 +360,36 @@ function parseHeader(header) {
     if (invalidTaxonRanks.length) {
         throw new SyntaxError("\"levels\" contains invalid values: ".concat(invalidTaxonRanks.join(', ')));
     }
-    var metadata = { levels: levels, scope: scope };
+    var metadata = { levels: levels };
     if ('catalog' in config && typeof config.catalog === 'object' && config.catalog !== null) {
-        metadata.catalog = config.catalog;
+        var catalog = {};
+        for (var key in config.catalog) {
+            var value = config.catalog[key];
+            if (typeof value === 'number') {
+                catalog[key] = value.toString();
+            }
+            else if (typeof value === 'string') {
+                catalog[key] = value;
+            }
+            else {
+                throw new SyntaxError("\"catalog\" should contain only strings (\"".concat(key, "\")"));
+            }
+        }
+        var work = new work_1.Work(catalog);
+        var errors = work.validate().filter(function (_a) {
+            var error = _a.error;
+            return error !== 'Value(s) required but missing';
+        });
+        if (errors.length > 0) {
+            throw new SyntaxError("\"catalog\" contains errors: ".concat(errors.map(function (_a) {
+                var field = _a.field, error = _a.error;
+                return "[".concat(field, "] ").concat(error);
+            }).join('; ')));
+        }
+        metadata.catalog = {};
+        for (var key in work.fields) {
+            metadata.catalog[key] = work.get(key);
+        }
     }
     return metadata;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larsgw/formica",
-  "version": "0.4.3",
+  "version": "0.5.0",
   "description": "SDK and tools for data from the Library of Identification Resources",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",

package/src/bin/process-resources.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-import { promises as fs } from 'fs'
+import { promises as fs, existsSync as doesFileExist } from 'fs'
 import * as path from 'path'
 import { spawn } from 'child_process'
 import * as util from 'util'
@@ -8,14 +8,20 @@ import * as util from 'util'
 import { csv } from '../index'
 import { prompt, promptForAnswers, numericSort, runCommand } from './util'
 
-const DWC_FIELDS: string[] = [
+export enum ResourceProcessorSource {
+    All = 'all',
+    Unprocessed = 'unprocessed',
+    Modified = 'modified'
+}
+
+const DWC_FIELDS: (keyof AmendedTaxon)[] = [
     'scientificNameID',
     'scientificName',
     'scientificNameAuthorship',
     'genericName',
-    'intragenericEpithet',
+    'infragenericEpithet',
     'specificEpithet',
-    'intraspecificEpithet',
+    'infraspecificEpithet',
 
     'taxonRank',
     'taxonRemarks',
@@ -36,6 +42,7 @@ const DWC_FIELDS: string[] = [
     'genus',
     'subgenus',
     'higherClassification',
+    'verbatimIdentification',
 
     'colTaxonID',
     'gbifTaxonID',
@@ -43,7 +50,7 @@ const DWC_FIELDS: string[] = [
     'gbifAcceptedTaxonID'
 ]
 
-const DISPLAY_FIELDS: string[] = [
+const DISPLAY_FIELDS: (keyof AmendedTaxon)[] = [
     'scientificNameID',
     'taxonRank',
     'scientificName',
@@ -83,11 +90,22 @@ function runGnverifier (names: string): Promise<string> {
     })
 }
 
+async function listFiles (directory: string): Promise<string[]> {
+    const input = await fs.readdir(directory)
+    return input.map(file => path.basename(file, '.txt')).sort(numericSort)
+}
+
+async function listUnprocessedFiles (directory: string, outputDirectory: string): Promise<string[]> {
+    const input = await listFiles(directory)
+    const output = new Set(await fs.readdir(outputDirectory))
+    return input.filter(file => output.has(file + '-1'))
+}
+
 async function listChangedFiles (directory: string): Promise<string[]> {
     const output = await runCommand('git', ['diff', '--name-only', 'HEAD', '--', directory], {
         cwd: directory
     })
-    return output.trimEnd().split('\n').sort(numericSort)
+    return output.trimEnd().split('\n').map(file => path.basename(file, '.txt')).sort(numericSort)
 }
 
 async function getOldFile (file: string): Promise<string> {
@@ -111,49 +129,32 @@ class ResourceProcessor {
         this.FILE_PROBLEMS = path.join(this.DIR_ROOT, 'problems.csv')
     }
 
-    async run (): Promise<void> {
-        const input = await fs.readdir(this.DIR_TXT)
-        const output = await fs.readdir(this.DIR_DWC)
-
-        const ids = input
-            .map(file => path.basename(file, '.txt'))
-            .sort((a, b) => parseInt(a.slice(1)) - parseInt(b.slice(1)))
-
+    async run (source: ResourceProcessorSource, config: ResourceProcessorConfig): Promise<void> {
+        const ids = await this.listWorks(source)
         for (const id of ids) {
-            // Skip existing files
-            if (output.some(file => file.startsWith(id + '-'))) {
-                continue
-            }
-
-            await this.processWork(id)
-        }
-    }
-
-    async runUpdate (): Promise<void> {
-        for (const file of await listChangedFiles(this.DIR_TXT)) {
-            const id = path.basename(file, '.txt')
-            await this.processWork(id, true)
+            await this.processWork(id, config)
         }
     }
 
-    async runMappingsUpdate (): Promise<void> {
-        const input = await fs.readdir(this.DIR_TXT)
-
-        const ids = input
-            .map(file => path.basename(file, '.txt'))
-            .sort((a, b) => parseInt(a.slice(1)) - parseInt(b.slice(1)))
-
-        for (const id of ids) {
-            await this.processWork(id, true)
+    async listWorks (source: ResourceProcessorSource): Promise<string[]> {
+        switch (source) {
+            case ResourceProcessorSource.All:
+                return listFiles(this.DIR_TXT)
+            case ResourceProcessorSource.Unprocessed:
+                return listUnprocessedFiles(this.DIR_TXT, this.DIR_DWC)
+            case ResourceProcessorSource.Modified:
+                return listChangedFiles(this.DIR_TXT)
+            default:
+                return []
         }
     }
 
-    async processWork (id: WorkId, update?: boolean): Promise<void> {
-        const resources = await this.processResources(id, update)
+    async processWork (id: WorkId, config: ResourceProcessorConfig): Promise<void> {
+        const resources = await this.processResources(id, config)
 
         await Promise.all(resources.map(resource => {
             const header = DWC_FIELDS
-            const table = [header]
+            const table: string[][] = [header]
 
             for (const id in resource.taxa) {
                 const taxon = resource.taxa[id] as unknown as Record<string, string | undefined>
@@ -164,12 +165,12 @@ class ResourceProcessor {
         }))
     }
 
-    async processResources (id: WorkId, update?: boolean): Promise<AmendedResource[]> {
-        const resources = await this.processResourceText(id, update)
+    async processResources (id: WorkId, config: ResourceProcessorConfig): Promise<AmendedResource[]> {
+        const resources = await this.processResourceText(id, config)
 
         const amendedResources = []
         for (const resource of resources) {
-            const results = await this.processResourceDwc(resource)
+            const results = await this.processResourceDwc(resource, config)
 
             const skip = await this.shouldBeSkipped(resource.id)
 
@@ -198,7 +199,7 @@ class ResourceProcessor {
                         case 'r':
                         case 'R': {
                             console.log(`${resource.workId}: retrying ${resource.id}`)
-                            return this.processResources(id, update)
+                            return this.processResources(id, config)
                         }
                     }
                 }
@@ -210,14 +211,14 @@ class ResourceProcessor {
         return amendedResources
     }
 
-    async processResourceText (id: WorkId, update?: boolean): Promise<Resource[]> {
+    async processResourceText (id: WorkId, config: ResourceProcessorConfig): Promise<Resource[]> {
         try {
             console.log(`${id}: generating Darwin Core`)
             const filePath = path.join(this.DIR_TXT, id + '.txt')
             const file = await fs.readFile(filePath, 'utf-8')
 
             let old = undefined
-            if (update) {
+            if (config.update) {
                 const dwc = []
                 for (const file of await fs.readdir(this.DIR_DWC)) {
                     if (file.startsWith(id + '-')) {
@@ -243,13 +244,34 @@ class ResourceProcessor {
                 }
             }
 
-            return this.processResourceText(id, update)
+            return this.processResourceText(id, config)
         }
     }
 
-    async processResourceDwc (resource: Resource): Promise<AmendedResource> {
+    async processResourceDwc (resource: Resource, config: ResourceProcessorConfig): Promise<AmendedResource> {
         console.log(`${resource.workId}: matching ${resource.id}`)
 
+        if (!config.updateMappings) {
+            const file = path.join(this.DIR_DWC, resource.file + '.csv')
+            if (doesFileExist(file)) {
+                const [header, ...rows] = csv.parseCsv(await fs.readFile(file, 'utf-8'))
+                for (const row of rows) {
+                    const oldTaxon = row.reduce((taxon, value, index) => {
+                        taxon[header[index]] = value
+                        return taxon
+                    }, {} as Record<string, string>)
+                    const taxon = resource.taxa[oldTaxon.scientificNameID] as AmendedTaxon
+                    if (taxon) {
+                        taxon.colTaxonID = oldTaxon.colTaxonID
+                        taxon.colAcceptedTaxonID = oldTaxon.colAcceptedTaxonID
+                        taxon.gbifTaxonID = oldTaxon.gbifTaxonID
+                        taxon.gbifAcceptedTaxonID = oldTaxon.gbifAcceptedTaxonID
+                    }
+                }
+            }
+            return resource as AmendedResource
+        }
+
         const filteredResults: Record<TaxonId, TaxonMatch[]> = {}
         const taxonNames: Record<string, TaxonId[]> = {}
         const names = new Set()
@@ -429,12 +451,14 @@ class ResourceProcessor {
 function main (): void {
     const args = util.parseArgs({
         options: {
-            update: {
-                type: 'boolean',
-                short: 'u'
+            source: {
+                type: 'string',
+                short: 's',
+                default: 'unprocessed'
             },
-            'update-mappings': {
-                type: 'boolean'
+            'keep-mappings': {
+                type: 'boolean',
+                short: 'k'
             }
         },
         allowPositionals: true
@@ -445,16 +469,13 @@ function main (): void {
         process.stdout.write('\n')
     })
 
-    let task
-    if (args.values.update) {
-        task = processor.runUpdate()
-    } else if (args.values['update-mappings']) {
-        task = processor.runMappingsUpdate()
-    } else {
-        task = processor.run()
+    const source = args.values.source as ResourceProcessorSource
+    const config: ResourceProcessorConfig = {
+        update: source !== 'unprocessed',
+        updateMappings: !args.values['keep-mappings']
     }
 
-    task.catch(error => {
+    processor.run(source, config).catch(error => {
         console.error(error)
         process.exit(1)
     })

package/src/catalog/value.ts

@@ -24,11 +24,11 @@ export const FORMATS = {
 
 export const CHECK = {
     MULTILANG (entry: Record<string, Value>) {
-        return entry.language.length > 1
+        return Array.isArray(entry.language) && entry.language.length > 1
     },
 
     ISBN (entry: Record<string, Value>) {
-        if (entry.ISBN.length < 2) { return false }
+        if (!Array.isArray(entry.ISBN) || entry.ISBN.length < 2) { return false }
         const a = entry.ISBN[0].length === 10
         const b = entry.ISBN[0].length === 13
         const c = entry.ISBN[1].length === 10

package/src/csv.ts

@@ -23,7 +23,7 @@ export function formatCsv (table: string[][], delimiter = ',') {
         .map((row: string[]) => {
             return row.map(value => {
                 if (/["\n]/.test(value) || value.includes(delimiter)) {
-                    return `"${value.replace(/"/, '"""')}"`
+                    return `"${value.replace(/"/g, '""')}"`
                 } else {
                     return value
                 }

package/src/module.d.ts

@@ -34,7 +34,7 @@ interface WorkingTaxon {
     genericName?: string,
     infragenericEpithet?: string,
     specificEpithet?: string,
-    intraspecificEpithet?: string,
+    infraspecificEpithet?: string,
 
     taxonRank?: Rank,
     taxonRemarks?: string,
@@ -55,6 +55,7 @@ interface WorkingTaxon {
     genus?: string,
     subgenus?: string,
     higherClassification?: string,
+    verbatimIdentification?: string,
 
     // Non-standard
     scientificNameOnly?: string,
@@ -66,13 +67,13 @@ interface Taxon extends WorkingTaxon {
     scientificName: string,
     taxonRank: Rank,
     collectionCode: ResourceId,
-    taxonomicStatus: string
+    taxonomicStatus: string,
+    verbatimIdentification: string
 }
 
 interface ResourceMetadata {
     levels: Rank[],
-    scope: string[],
-    catalog?: object
+    catalog?: Record<string, Value>
 }
 
 interface Resource {
@@ -123,3 +124,14 @@ interface TaxonMatch {
 }
 
 type GroupedNameMatches = Record<string, Record<string, Record<TaxonId, TaxonMatch>>>
+
+declare enum ResourceProcessorSource {
+    All = 'all',
+    Unprocessed = 'unprocessed',
+    Modified = 'modified'
+}
+
+interface ResourceProcessorConfig {
+    update: boolean,
+    updateMappings: boolean
+}

package/src/resources/parse-text.ts

@@ -1,4 +1,5 @@
 import * as yaml from 'js-yaml'
+import { Work } from '../catalog/tables/work'
 import { createDiff, ResourceDiffType } from './diff-resource'
 
 const RANKS: Rank[] = [
@@ -120,9 +121,9 @@ const NAME_PATTERN = new RegExp(
  *   $1 genus+subgenus (+ trailing space): (?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?
  *     $1.1 genus: ([A-Z]\S+)
  *     $1.2 subgenus: (?:\(([A-Z]\S+?)\) )?
- *   $2 species: ((?:x )?[a-z]\S+)
+ *   $2 species: ((?:x )?[a-z][^\s.]+)
  */
-const BINAME_PATTERN = /^(?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?((?:x )?[a-z]\S+)(?= |$)/
+const BINAME_PATTERN = /^(?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?((?:x )?[a-z][^\s.]+)(?= |$)/
 
 function compareRanks (a: Rank, b: Rank): number {
     return RANKS.indexOf(a) - RANKS.indexOf(b)
@@ -178,49 +179,67 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
         name = name.replace(/^\[(_|\d+)\] /, '')
     }
 
+    // Set verbatim identification after subsequent syntax is removed.
+    item.verbatimIdentification = name
+
     // Parent context is used for parsing and formatting binomial names.
-    const parentContext = { ...parent }
-    if (parent.incorrect) { parentContext.incorrect = { ...parent.incorrect } }
+    // For formatting, it needs to match external databases (i.e. be correct).
+    // For parsing, it needs to match the current file. If relevant parents
+    // (i.e. genus, species) had mistakes that were corrected, the uncorrected
+    // genus and species names need to be used.
+    const parentContext = {
+        genus: parent.genus,
+        subgenus: parent.subgenus,
+        specificEpithet: parent.specificEpithet,
+        incorrect: {
+            genus: parent.incorrect && parent.incorrect.genus,
+            specificEpithet: parent.incorrect && parent.incorrect.specificEpithet
+        }
+    }
 
-    // The parent context should be amended in the two cases where binomial names
-    // are truly accepted: synonyms and species (and below) without parents (resp.
-    // genera and genera and species) to provide parts of the name.
+    // Both contexts should be amended in the two cases where binomial names
+    // are fully used: (1) synonyms and (2) multinomial taxa without parents to
+    // provide parts of the name (e.g. bare species without a genus parent, or
+    // even subspecies without a species or genus parent).
     if (isSynonym || !parentContext.genus || (compareRanks('species', rank) < 0 && !parentContext.specificEpithet)) {
         const [, genus, subgenus, species] = name.match(BINAME_PATTERN) || []
         if (genus) {
-            parentContext.genus = capitalize(genus)
-            if (parentContext.incorrect) parentContext.incorrect.genus = capitalize(genus)
+            parentContext.genus = parentContext.incorrect.genus = capitalize(genus)
         }
         if (subgenus) {
             parentContext.subgenus = capitalize(subgenus)
-            if (parentContext.incorrect) parentContext.incorrect.subgenus = capitalize(subgenus)
         } else if (genus) {
-            // If a genus is given but no subgenus, remove it from the parent context
+            // If a genus is given but no subgenus, remove any existing subgenus
+            // from the parent context.
             delete parentContext.subgenus
-            if (parentContext.incorrect) delete parentContext.incorrect.subgenus
         }
         if (species) {
-            parentContext.specificEpithet = species
-            if (parentContext.incorrect) parentContext.incorrect.specificEpithet = species
+            parentContext.specificEpithet = parentContext.incorrect.specificEpithet = species
         }
     }
 
     // In taxa of group, species or lower, the name should just contain the
-    // (inter)specific epithet and the author information & remarks when processing
+    // (infra)specific epithet and the author information & remarks when processing
     // further.
     if (compareRanks('group', rank) <= 0) {
-        const parseContext = parentContext.incorrect || parentContext
-        if (!parseContext.genus) { parseContext.genus = name.split(' ', 1)[0] }
-        const genusPrefix = new RegExp(`^${parentContext.genus} (\\(.*?\\) )?`, 'i')
-        if (name[0] === (parentContext.genus as string)[0]) {
-            name = name.replace(genusPrefix, '')
+        // Remove genus
+        const genus = parentContext.incorrect.genus || parentContext.genus || ''
+        if (name[0] === genus[0] && name.toLowerCase().startsWith(genus.toLowerCase() + ' ')) {
+            name = name.slice(genus.length + 1)
         }
 
+        // Remove subgenus
+        name = name.replace(/^\(.*?\) /, '')
+
+        // Infraspecific taxa
         if (compareRanks('species', rank) < 0) {
-            const speciesPrefix = parseContext.specificEpithet + ' '
-            if (name.startsWith(speciesPrefix)) {
-                name = name.slice(speciesPrefix.length)
+            // Remove specific epithet
+            const species = parentContext.incorrect.specificEpithet || parentContext.specificEpithet || ''
+            if (name.startsWith(species + ' ')) {
+                name = name.slice(species.length + 1)
             }
+
+            // Remove rank abbreviations
             name = name.replace(/^(st|r|ab|f|var|ssp|subsp)\. /, '')
         }
     }
@@ -236,6 +255,13 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
     if (!nameParts) {
         throw new Error(`Taxon "${name}" could not be parsed`)
     }
+
+    // To encode old names with spaces (e.g. "Orsillus pini canariensis Lindberg, 1953")
+    // underscores are used, which are replaced here.
+    if (nameParts[1].includes('_')) {
+        nameParts[1] = nameParts[1].replace(/_/g, ' ')
+    }
+
     const [_, taxon, citation = '', notes] = nameParts
     item.scientificNameAuthorship = capitalizeAuthors(citation)
     item.taxonRemarks = notes
@@ -282,23 +308,23 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
         item.genericName = parentContext.genus
         item.infragenericEpithet = parentContext.subgenus
         item.specificEpithet = parentContext.specificEpithet
-        item.intraspecificEpithet = taxon.toLowerCase()
+        item.infraspecificEpithet = taxon.toLowerCase()
 
         // If possible, names below species should have abbreviations for ranks,
         // like "subsp."
         const nameParts = [
             item.genericName,
             item.specificEpithet,
-            item.intraspecificEpithet
+            item.infraspecificEpithet
         ]
         if (item.taxonRank in RANK_LABELS) {
             nameParts.splice(2, 0, RANK_LABELS[item.taxonRank])
         }
         item.scientificName = nameParts.join(' ')
 
-        if (item.intraspecificEpithet !== taxon) {
+        if (item.infraspecificEpithet !== taxon) {
             console.log(item, taxon)
-            throw new Error(`Intraspecific epithet should be lowercase: "${taxon}"`)
+            throw new Error(`Infraspecific epithet should be lowercase: "${taxon}"`)
         }
     }
 
@@ -341,13 +367,8 @@ function parseHeader (header: string): ResourceMetadata {
         levels = config.levels
     }
 
-    let scope
-    if (!('scope' in config)) {
-        scope = []
-    } else if (!Array.isArray(config.scope)) {
-        throw new SyntaxError('"scope" should be an array')
-    } else {
-        scope = config.scope
+    if ('scope' in config) {
+        throw new SyntaxError('"scope" data should go in "catalog"')
     }
 
     // No taxon ranks
@@ -361,10 +382,29 @@ function parseHeader (header: string): ResourceMetadata {
         throw new SyntaxError(`"levels" contains invalid values: ${invalidTaxonRanks.join(', ')}`)
     }
 
-    const metadata: ResourceMetadata = { levels, scope }
+    const metadata: ResourceMetadata = { levels }
 
     if ('catalog' in config && typeof config.catalog === 'object' && config.catalog !== null) {
-        metadata.catalog = config.catalog
+        const catalog: Record<string, string> = {}
+        for (const key in config.catalog) {
+            const value = config.catalog[key as keyof object]
+            if (typeof value === 'number') {
+                catalog[key] = (value as number).toString()
+            } else if (typeof value === 'string') {
+                catalog[key] = value
+            } else {
+                throw new SyntaxError(`"catalog" should contain only strings ("${key}")`)
+            }
+        }
+        const work = new Work(catalog)
+        const errors = work.validate().filter(({ error }) => error !== 'Value(s) required but missing')
+        if (errors.length > 0) {
+            throw new SyntaxError(`"catalog" contains errors: ${errors.map(({ field, error }) => `[${field}] ${error}`).join('; ')}`)
+        }
+        metadata.catalog = {}
+        for (const key in work.fields) {
+            metadata.catalog[key] = work.get(key) as Value
+        }
     }
 
     return metadata

package/test/resources.js

@@ -100,4 +100,28 @@ Nematus fåhraei Thomson
 `, 'T1')
         assert.strictEqual(resource.taxa['T1:1:1'].scientificName, 'Nematus fåhraei Thomson')
     })
+
+    await t.test('parses synonyms in different genera', (t) => {
+        const [resource] = resources.parseTextFile(`---
+levels: [species]
+---
+
+Katamenes arbustorum subsp. burlinii
+  > arbustorum subsp. burlinii
+  = Eumenes arbustorum var. burlinii
+`, 'T1')
+        assert.strictEqual(resource.taxa['T1:1:2'].scientificName, 'Eumenes arbustorum var. burlinii')
+    })
+
+    await t.test('parses species without generic names', (t) => {
+        const [resource] = resources.parseTextFile(`---
+levels: [genus, subgenus, species]
+---
+
+Microdynerus Thomson, 1874
+  Alastorynerus Blüthgen, 1938
+    microdynerus (Dalla Torre, 1889)
+`, 'T1')
+        assert.strictEqual(resource.taxa['T1:1:3'].scientificName, 'Microdynerus microdynerus (Dalla Torre, 1889)')
+    })
 })