@larsgw/formica 0.6.6 → 0.6.8

package/CHANGELOG.md

@@ -1,3 +1,30 @@
+## [0.6.8](https://github.com/identification-resources/formica/compare/v0.6.7...v0.6.8) (2025-03-13)
+
+
+### Features
+
+* **resources:** add support for intergeneric hybrids without parents ([3cb0c0c](https://github.com/identification-resources/formica/commit/3cb0c0cd60fc5a4ba88add623fbadc3b90d8530c))
+* **resources:** create index of CoL identifiers ([3792903](https://github.com/identification-resources/formica/commit/37929035374bc0c67ed2403e775e06a904bdbd4d))
+* **resources:** support intergeneric hybrids ([0a6696c](https://github.com/identification-resources/formica/commit/0a6696c5fd0603a0366c80c2298b6544636ed0c6))
+
+
+
+## [0.6.7](https://github.com/identification-resources/formica/compare/v0.6.6...v0.6.7) (2025-02-26)
+
+
+### Bug Fixes
+
+* **resources:** keep uncorrected name in verbatimIdentification ([dc4d5d1](https://github.com/identification-resources/formica/commit/dc4d5d17e7a412a94538f6162dfd839d808cbc30))
+* **resources:** update heuristics for recognizing author citations ([bca6a7f](https://github.com/identification-resources/formica/commit/bca6a7f5a6c2dc3dc421d885f6270f5464fc5ea5))
+* **resources:** use corrected generic name in output ([b472384](https://github.com/identification-resources/formica/commit/b4723842e87515f076230c603246082bc3bbec9d))
+
+
+### Features
+
+* **resources:** add support for completeness flags ([55b5427](https://github.com/identification-resources/formica/commit/55b5427125b4f1072191cadbc461ba31d37e6f2e))
+
+
+
 ## [0.6.6](https://github.com/identification-resources/formica/compare/v0.6.5...v0.6.6) (2025-02-16)
 
 

package/lib/bin/process-resources-index.js

@@ -52,26 +52,27 @@ var fs_1 = require("fs");
 var path = require("path");
 var index_1 = require("../index");
 var util_1 = require("./util");
-/* eslint-disable @typescript-eslint/no-explicit-any */
-function sortObject(object) {
+function alphabeticSort(a, b) {
+    return a > b ? 1 : a < b ? -1 : 0;
+}
+function sortObject(object, sorter) {
     var sorted = {};
-    for (var _i = 0, _a = Object.keys(object).sort(util_1.numericSort); _i < _a.length; _i++) {
+    for (var _i = 0, _a = Object.keys(object).sort(sorter !== null && sorter !== void 0 ? sorter : util_1.numericSort); _i < _a.length; _i++) {
         var key = _a[_i];
         sorted[key] = object[key];
     }
     return sorted;
 }
-/* eslint-enable @typescript-eslint/no-explicit-any */
-function addTaxon(gbifIndex, gbifId, taxon) {
-    if (!(gbifId in gbifIndex)) {
-        gbifIndex[gbifId] = [];
+function addTaxon(index, id, taxon) {
+    if (!(id in index)) {
+        index[id] = [];
     }
-    gbifIndex[gbifId].push(taxon[0]);
-    gbifIndex[gbifId].sort(util_1.numericSort);
+    index[id].push(taxon[0]);
+    index[id].sort(util_1.numericSort);
 }
 function main(args) {
     return __awaiter(this, void 0, void 0, function () {
-        var REPO_ROOT, files, gbifIndex, resourceIndex;
+        var REPO_ROOT, files, gbifIndex, colIndex, resourceIndex;
         return __generator(this, function (_a) {
             switch (_a.label) {
                 case 0:
@@ -80,6 +81,7 @@ function main(args) {
                 case 1:
                     files = _a.sent();
                     gbifIndex = {};
+                    colIndex = {};
                     resourceIndex = {};
                     return [4 /*yield*/, Promise.all(files.map(function (fileName) {
                             return __awaiter(this, void 0, void 0, function () {
@@ -96,7 +98,7 @@ function main(args) {
                                             file = _a.sent();
                                             return [2 /*return*/, Promise.all(index_1.resources.parseTextFileHeader(file).map(function (resource, index) {
                                                     return __awaiter(this, void 0, void 0, function () {
-                                                        var amendedResource, dwcFile, _a, header, dwc, _b, _c, gbifColumn, gbifAcceptedColumn, _i, dwc_1, taxon, gbifId;
+                                                        var amendedResource, dwcFile, _a, header, dwc, _b, _c, gbifColumn, gbifAcceptedColumn, colColumn, colAcceptedColumn, _i, dwc_1, taxon, gbifId, colId;
                                                         return __generator(this, function (_d) {
                                                             switch (_d.label) {
                                                                 case 0:
@@ -111,6 +113,8 @@ function main(args) {
                                                                     _a = _c.apply(_b, [_d.sent()]), header = _a[0], dwc = _a.slice(1);
                                                                     gbifColumn = header.indexOf('gbifTaxonID');
                                                                     gbifAcceptedColumn = header.indexOf('gbifAcceptedTaxonID');
+                                                                    colColumn = header.indexOf('colTaxonID');
+                                                                    colAcceptedColumn = header.indexOf('colAcceptedTaxonID');
                                                                     for (_i = 0, dwc_1 = dwc; _i < dwc_1.length; _i++) {
                                                                         taxon = dwc_1[_i];
                                                                         gbifId = taxon[gbifColumn];
@@ -120,6 +124,13 @@ function main(args) {
                                                                                 addTaxon(gbifIndex, taxon[gbifAcceptedColumn], taxon);
                                                                             }
                                                                         }
+                                                                        colId = taxon[colColumn];
+                                                                        if (colId) {
+                                                                            addTaxon(colIndex, colId, taxon);
+                                                                            if (taxon[colAcceptedColumn] !== taxon[colColumn]) {
+                                                                                addTaxon(colIndex, taxon[colAcceptedColumn], taxon);
+                                                                            }
+                                                                        }
                                                                         amendedResource.taxonCount += 1;
                                                                     }
                                                                     resourceIndex[amendedResource.id] = amendedResource;
@@ -136,6 +147,7 @@ function main(args) {
                     _a.sent();
                     return [4 /*yield*/, Promise.all([
                             fs_1.promises.writeFile(path.join(REPO_ROOT, 'gbif.index.json'), JSON.stringify(sortObject(gbifIndex), null, 2)),
+                            fs_1.promises.writeFile(path.join(REPO_ROOT, 'col.index.json'), JSON.stringify(sortObject(colIndex, alphabeticSort), null, 2)),
                             fs_1.promises.writeFile(path.join(REPO_ROOT, 'index.json'), JSON.stringify(sortObject(resourceIndex), null, 2))
                         ])];
                 case 3:

package/lib/bin/process-resources-problems.js

@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+"use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+var fs_1 = require("fs");
+var path = require("path");
+var util = require("util");
+var index_1 = require("../index");
+var GBIF_RANKS = [
+    'kingdom',
+    'phyllum',
+    'class',
+    'order',
+    'family',
+    'genus',
+    'species',
+    'subspecies',
+    'variety'
+];
+var DISPLAY_FIELDS = [
+    'scientificNameID',
+    'taxonRank',
+    'scientificName',
+    'taxonomicStatus',
+    'taxonRemarks',
+    'colTaxonID',
+    'gbifTaxonID'
+];
+var ResourceProblemsProcessor = /** @class */ (function () {
+    function ResourceProblemsProcessor(collectionPath) {
+        this.DIR_ROOT = path.resolve(collectionPath);
+        this.DIR_DWC = path.join(this.DIR_ROOT, 'dwc');
+        this.FILE_INDEX = path.join(this.DIR_ROOT, 'index.json');
+        this.FILE_PROBLEMS = path.join(this.DIR_ROOT, 'problems.csv');
+    }
+    ResourceProblemsProcessor.prototype.run = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var problems, _a, _b, resources, _c, _d, _i, problems_1, problem, _work, resourceId, _problemText, resource, missing;
+            return __generator(this, function (_e) {
+                switch (_e.label) {
+                    case 0:
+                        _b = (_a = index_1.csv).parseCsv;
+                        return [4 /*yield*/, fs_1.promises.readFile(this.FILE_PROBLEMS, 'utf8')];
+                    case 1:
+                        problems = _b.apply(_a, [_e.sent()]).slice(1);
+                        _d = (_c = JSON).parse;
+                        return [4 /*yield*/, fs_1.promises.readFile(this.FILE_INDEX, 'utf8')];
+                    case 2:
+                        resources = _d.apply(_c, [_e.sent()]);
+                        _i = 0, problems_1 = problems;
+                        _e.label = 3;
+                    case 3:
+                        if (!(_i < problems_1.length)) return [3 /*break*/, 6];
+                        problem = problems_1[_i];
+                        _work = problem[0], resourceId = problem[1], _problemText = problem[2];
+                        return [4 /*yield*/, this.readAmendedResource(resourceId, resources)];
+                    case 4:
+                        resource = _e.sent();
+                        missing = this.getMissingMappings(resource);
+                        if (missing.length === 0) {
+                            this.removeProblem(resourceId);
+                        }
+                        _e.label = 5;
+                    case 5:
+                        _i++;
+                        return [3 /*break*/, 3];
+                    case 6: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    ResourceProblemsProcessor.prototype.readAmendedResource = function (resourceId, resources) {
+        return __awaiter(this, void 0, void 0, function () {
+            var resource, resourceFileId, resourceFile, _a, header, taxonRows, _b, _c, taxa, _i, taxonRows_1, row, taxon;
+            return __generator(this, function (_d) {
+                switch (_d.label) {
+                    case 0:
+                        resource = resources[resourceId];
+                        resourceFileId = resourceId.split(':').join('-');
+                        resourceFile = path.join(this.DIR_DWC, "".concat(resourceFileId, ".csv"));
+                        _c = (_b = index_1.csv).parseCsv;
+                        return [4 /*yield*/, fs_1.promises.readFile(resourceFile, 'utf8')];
+                    case 1:
+                        _a = _c.apply(_b, [_d.sent()]), header = _a[0], taxonRows = _a.slice(1);
+                        taxa = {};
+                        for (_i = 0, taxonRows_1 = taxonRows; _i < taxonRows_1.length; _i++) {
+                            row = taxonRows_1[_i];
+                            taxon = row.reduce(function (taxon, value, index) {
+                                taxon[header[index]] = value;
+                                return taxon;
+                            }, {});
+                            taxa[taxon.scientificNameID] = taxon;
+                        }
+                        return [2 /*return*/, __assign(__assign({}, resource), { taxa: taxa })];
+                }
+            });
+        });
+    };
+    ResourceProblemsProcessor.prototype.getMissingMappings = function (resource) {
+        var missing = [];
+        for (var id in resource.taxa) {
+            var taxon = resource.taxa[id];
+            if (taxon.taxonomicStatus !== 'accepted') {
+                continue;
+            }
+            var missingCol = false; // !taxon.colTaxonID
+            var missingGbif = GBIF_RANKS.includes(taxon.taxonRank) && !taxon.gbifTaxonID;
+            if (missingCol || missingGbif) {
+                missing.push(taxon);
+            }
+        }
+        return missing;
+    };
+    ResourceProblemsProcessor.prototype.removeProblem = function (resourceId) {
+        resourceId;
+        //
+    };
+    return ResourceProblemsProcessor;
+}());
+function main() {
+    var args = util.parseArgs({ allowPositionals: true });
+    var processor = new ResourceProblemsProcessor(args.positionals[0]);
+    process.on('exit', function () {
+        process.stdout.write('\n');
+    });
+    processor.run().catch(function (error) {
+        console.error(error);
+        process.exit(1);
+    });
+}
+main();

package/lib/resources/parse-text.js

@@ -64,6 +64,11 @@ var DWC_RANKS = [
     'genus',
     'subgenus'
 ];
+var FLAGS = [
+    'MISSING_TAXA',
+    'MISSING_PARENT_TAXA',
+    'MISSING_SYNONYMS'
+];
 var TAXONOMIC_STATUS = {
     '>': 'incorrect',
     '+': 'heterotypic synonym',
@@ -93,6 +98,7 @@ var RANK_LABELS_REVERSE = {
     'ssp': 'subspecies',
     'subsp': 'subspecies'
 };
+var HYBRID_SIGN = '\u00D7';
 /**
  *   1. Any number of
  *      - capitalized words
@@ -103,7 +109,7 @@ var RANK_LABELS_REVERSE = {
  *   2. Followed by a capitalized word
  *   3. Optionally, followed by "et al."
  */
-var LOWERCASE_NAME_PARTICLES = ['y', 'der', 'den', 'de', 'van'].join('|');
+var LOWERCASE_NAME_PARTICLES = ['y', 'der', 'den', 'de', 'van', 'von'].join('|');
 var SIMPLE_AUTHOR_PATTERN = '(?:(?:\\p{Lu}\\S*|&|in|ex|' + LOWERCASE_NAME_PARTICLES + ')\\s*)*\\p{Lu}\\S+(?:\\s+et\\s+al\\.)?';
 var NAME_PATTERN = new RegExp('^' +
     // $1 main name part
@@ -111,7 +117,7 @@ var NAME_PATTERN = new RegExp('^' +
     // $2 optional author citation
     '(?: ' +
     // but not auct(t)., etc.
-    '(?!auctt?\\.|(?:syn|comb|sp|spec)\\. n(?:ov)?\\.|s(?:ens[.u]|\\.))' +
+    '(?!auctt?\\.|(?:syn|comb|sp|spec|nom)\\. n(?:ov)?\\.|s(?:ens[.u]|\\.)|in part|partim)' +
     '(' +
     // $2.1 anything in parentheses, followed by optional revising author(s)
     '\\(.+?\\)(?:\\s+' + SIMPLE_AUTHOR_PATTERN + ')?' +
@@ -128,22 +134,30 @@ var NAME_PATTERN = new RegExp('^' +
 /**
  * Structure
  *   $1 genus+subgenus (+ trailing space): (?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?
- *     $1.1 genus: ([A-Z]\S+)
+ *     $1.1 genus: ((?:x )?[A-Z]\S+)
  *     $1.2 subgenus: (?:\(([A-Z]\S+?)\) )?
- *   $2 species: ((?:x )?[a-z][^\s.]+)
+ *   $2 species: (x [a-z]+|[a-z][^\s.]+(?: x [a-z]+)?|[A-Z][a-z]+_[a-z]+ x [A-Z][a-z]+_[a-z]+)
+ *     $2a: x [a-z]+
+ *     $2b hybrid: [a-z][^\s.]+(?: x [a-z]+)?
+ *     $2c intergeneric hybrid: [A-Z][a-z]+_[a-z]+ x [A-Z][a-z]+_[a-z]+
  */
-var BINAME_PATTERN = /^(?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?((?:x )?[a-z][^\s.]+)(?= |$)/;
+var BINAME_PATTERN = /^(?:((?:x )?[A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?(x [a-z]+|[a-z][^\s.]+(?: x [a-z]+)?|[A-Z][a-z]+_[a-z]+ x [A-Z][a-z]+_[a-z]+)(?= |$)/;
 function compareRanks(a, b) {
     return RANKS.indexOf(a) - RANKS.indexOf(b);
 }
 function capitalize(name) {
     return name[0].toUpperCase() + name.slice(1).toLowerCase();
 }
+function capitalizeGenericName(name) {
+    if (name[0] === HYBRID_SIGN) {
+        return HYBRID_SIGN + capitalize(name.slice(1));
+    }
+    return capitalize(name);
+}
 function isUpperCase(name) {
     return name === name.toUpperCase();
 }
 function getSynonymRank(name, rank) {
-    var BINAME_PATTERN = /^([A-Z]\S+ (\([A-Z]\S+\) )?)?(x )?[a-z0-9-]+(?= |$)/;
     var rest = name.replace(BINAME_PATTERN, '');
     var rankPrefix = rest.match(/^(?: |^)(st|r|ab|f|var|ssp|subsp)\. /);
     if (rankPrefix) {
@@ -183,7 +197,7 @@ function parseName(name, rank, parent) {
         name = name.replace(/^\[(_|\d+)\] /, '');
     }
     // Set verbatim identification after subsequent syntax is removed.
-    item.verbatimIdentification = name;
+    item.verbatimIdentification = name.replace(/(?<=^| )x(?=$| )/g, HYBRID_SIGN).replace(/_/g, ' ');
     // Parent context is used for parsing and formatting binomial names.
     // For formatting, it needs to match external databases (i.e. be correct).
     // For parsing, it needs to match the current file. If relevant parents
@@ -205,7 +219,8 @@ function parseName(name, rank, parent) {
     if (isSynonym || !parentContext.genus || (compareRanks('species', rank) < 0 && !parentContext.specificEpithet)) {
         var _a = name.match(BINAME_PATTERN) || [], genus = _a[1], subgenus = _a[2], species = _a[3];
         if (genus) {
-            parentContext.genus = parentContext.incorrect.genus = capitalize(genus);
+            parentContext.incorrect.genus = genus;
+            parentContext.genus = capitalizeGenericName(genus.replace(/(^| )x /, HYBRID_SIGN));
         }
         if (subgenus) {
             parentContext.subgenus = capitalize(subgenus);
@@ -215,8 +230,9 @@ function parseName(name, rank, parent) {
             // from the parent context.
             delete parentContext.subgenus;
         }
-        if (species) {
-            parentContext.specificEpithet = parentContext.incorrect.specificEpithet = species;
+        if (species && compareRanks('species', rank) < 0) {
+            parentContext.incorrect.specificEpithet = species;
+            parentContext.specificEpithet = species.replace(/(^| )x /, HYBRID_SIGN);
         }
     }
     // In taxa of group, species or lower, the name should just contain the
@@ -242,8 +258,11 @@ function parseName(name, rank, parent) {
         }
     }
     // Hybrids
+    if (rank === 'genus' && name.startsWith('x ')) {
+        name = HYBRID_SIGN + name.slice(2);
+    }
     if (rank === 'species' && /(^| )x /.test(name)) {
-        name = name.replace(/(^| )x /, '\u00D7');
+        name = name.replace(/(^| )x /, HYBRID_SIGN);
     }
     // Divide the name into the main scientific name (only the epithet for taxa
     // lower than genus), the authorship information, and optionally remarks
@@ -252,7 +271,9 @@ function parseName(name, rank, parent) {
         throw new Error("Taxon \"".concat(name, "\" could not be parsed"));
     }
     // To encode old names with spaces (e.g. "Orsillus pini canariensis Lindberg, 1953")
-    // underscores are used, which are replaced here.
+    // underscores are used, which are replaced here. This is also used for undescribed
+    // species (e.g. "Leiobunum species A") and intergeneric hybrids (e.g. "×Festulpia
+    // Festuca rubra × Vulpia bromoides")
     if (nameParts[1].includes('_')) {
         nameParts[1] = nameParts[1].replace(/_/g, ' ');
     }
@@ -264,9 +285,15 @@ function parseName(name, rank, parent) {
         throw new Error("Taxon name contains unexpected characters: \"".concat(taxon, "\""));
     }
     // Validate names and recompose binomial and trinomial names
-    if (compareRanks('group', rank) > 0) {
+    if (rank === 'genus') {
+        item.scientificName = capitalizeGenericName(taxon);
+        if (taxon[0].toUpperCase() !== taxon[0] || (taxon[0] === HYBRID_SIGN && taxon[1].toUpperCase() !== taxon[1])) {
+            throw new Error("Generic epithet should be capitalized: \"".concat(taxon, "\""));
+        }
+    }
+    else if (compareRanks('group', rank) > 0) {
         item.scientificName = capitalize(taxon);
-        if (item.scientificName[0] !== taxon[0]) {
+        if (taxon[0].toUpperCase() !== taxon[0]) {
             throw new Error("Taxon name (".concat(rank, ") should be capitalized: \"").concat(taxon, "\""));
         }
     }
@@ -293,12 +320,12 @@ function parseName(name, rank, parent) {
     else if (rank === 'species') {
         item.genericName = parentContext.genus;
         item.infragenericEpithet = parentContext.subgenus;
-        item.specificEpithet = taxon.toLowerCase();
-        item.scientificName = "".concat(item.genericName, " ").concat(item.specificEpithet);
-        if (item.specificEpithet !== taxon) {
+        if (taxon.toLowerCase() !== taxon && !/^[A-Z][a-z]+ [a-z]+\xD7[A-Z][a-z]+ [a-z]+$/.test(taxon)) {
             console.log(item, taxon);
             throw new Error("Specific epithet should be lowercase: \"".concat(taxon, "\""));
         }
+        item.specificEpithet = taxon;
+        item.scientificName = "".concat(item.genericName, " ").concat(item.specificEpithet);
     }
     else if (compareRanks('species', rank) < 0) {
         item.genericName = parentContext.genus;
@@ -326,17 +353,6 @@ function parseName(name, rank, parent) {
     if (item.scientificNameAuthorship) {
         item.scientificName += " ".concat(item.scientificNameAuthorship);
     }
-    // Amend "parent" with corrections
-    if (item.taxonomicStatus === 'incorrect') {
-        var itemAsObject = item;
-        var parentAsObject = parent;
-        parent.incorrect = __assign({}, parent);
-        for (var key in item) {
-            if (key !== 'taxonomicStatus') {
-                parentAsObject[key] = itemAsObject[key];
-            }
-        }
-    }
     return item;
 }
 function parseHeader(header) {
@@ -401,6 +417,16 @@ function parseHeader(header) {
             metadata.catalog[key] = work.get(key);
         }
     }
+    if ('flags' in config) {
+        if (!Array.isArray(config.flags)) {
+            throw new SyntaxError('"flags" should be an array if present');
+        }
+        var invalidFlags = config.flags.filter(function (flag) { return !FLAGS.includes(flag); });
+        if (invalidFlags.length) {
+            throw new SyntaxError("\"flags\" contains invalid values: ".concat(invalidFlags.join(', ')));
+        }
+        metadata.flags = config.flags;
+    }
     return metadata;
 }
 function validateResource(config, content) {
@@ -480,27 +506,11 @@ function parseResourceContent(content, resource, oldIds) {
         var rank = resource.metadata.levels[groupIndent / 2];
         var item = parseName(name_1, rank, parent_1);
         var isSynonym = item.taxonomicStatus !== 'accepted';
-        if (item.taxonomicStatus === 'incorrect') {
-            return "continue";
-        }
-        if (type === diff_resource_1.ResourceDiffType.Added) {
-            newIdOffset++;
-            item.scientificNameID = idBase + newIdOffset.toString();
-        }
-        else {
-            id++;
-            item.scientificNameID = idBase + (oldIds[id - 1] || id).toString();
-        }
-        previousId = item.scientificNameID;
-        item.parentNameUsageID = isSynonym ? undefined : parent_1.scientificNameID;
-        item.parentNameUsage = isSynonym ? undefined : parent_1.scientificName;
-        item.acceptedNameUsageID = isSynonym ? parent_1.scientificNameID : undefined;
-        item.acceptedNameUsage = isSynonym ? parent_1.scientificName : undefined;
-        item.collectionCode = idBase.slice(0, -1);
+        // Add higher classification info
+        var itemAsObject = item;
+        var parentAsObject = parent_1;
         for (var _b = 0, DWC_RANKS_1 = DWC_RANKS; _b < DWC_RANKS_1.length; _b++) {
             var rank_1 = DWC_RANKS_1[_b];
-            var itemAsObject = item;
-            var parentAsObject = parent_1;
             itemAsObject[rank_1] = undefined;
             if (parentAsObject[rank_1]) {
                 itemAsObject[rank_1] = parentAsObject[rank_1];
@@ -524,6 +534,31 @@ function parseResourceContent(content, resource, oldIds) {
         else if (parentId) {
             item.higherClassification = parent_1.scientificNameOnly;
         }
+        // Amend "parent" with corrections
+        if (item.taxonomicStatus === 'incorrect') {
+            parent_1.incorrect = __assign({}, parent_1);
+            for (var key in item) {
+                if (key !== 'taxonomicStatus' && key !== 'verbatimIdentification') {
+                    parentAsObject[key] = itemAsObject[key];
+                }
+            }
+            return "continue";
+        }
+        // Set identifiers
+        if (type === diff_resource_1.ResourceDiffType.Added) {
+            newIdOffset++;
+            item.scientificNameID = idBase + newIdOffset.toString();
+        }
+        else {
+            id++;
+            item.scientificNameID = idBase + (oldIds[id - 1] || id).toString();
+        }
+        previousId = item.scientificNameID;
+        item.parentNameUsageID = isSynonym ? undefined : parent_1.scientificNameID;
+        item.parentNameUsage = isSynonym ? undefined : parent_1.scientificName;
+        item.acceptedNameUsageID = isSynonym ? parent_1.scientificNameID : undefined;
+        item.acceptedNameUsage = isSynonym ? parent_1.scientificName : undefined;
+        item.collectionCode = idBase.slice(0, -1);
         data[item.scientificNameID] = item;
     };
     for (var _i = 0, content_1 = content; _i < content_1.length; _i++) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larsgw/formica",
-  "version": "0.6.6",
+  "version": "0.6.8",
   "description": "SDK and tools for data from the Library of Identification Resources",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",

package/src/bin/process-resources-index.ts

@@ -11,22 +11,25 @@ interface AmendedResourceMetadata extends ResourceMetadata {
     taxonCount: number
 }
 
-/* eslint-disable @typescript-eslint/no-explicit-any */
-function sortObject (object: Record<string, any>): Record<string, any> {
-    const sorted: Record<string, any> = {}
-    for (const key of Object.keys(object).sort(numericSort)) {
+type SortObjectCallback = (a: string, b: string) => number
+function alphabeticSort (a: string, b: string): number {
+    return a > b ? 1 : a < b ? -1 : 0
+}
+
+function sortObject (object: Record<string, unknown>, sorter?: SortObjectCallback): Record<string, unknown> {
+    const sorted: Record<string, unknown> = {}
+    for (const key of Object.keys(object).sort(sorter ?? numericSort)) {
         sorted[key] = object[key]
     }
     return sorted
 }
-/* eslint-enable @typescript-eslint/no-explicit-any */
 
-function addTaxon (gbifIndex: Record<string, TaxonId[]>, gbifId: string, taxon: string[]) {
-    if (!(gbifId in gbifIndex)) {
-        gbifIndex[gbifId] = []
+function addTaxon (index: Record<string, TaxonId[]>, id: string, taxon: string[]) {
+    if (!(id in index)) {
+        index[id] = []
     }
-    gbifIndex[gbifId].push(taxon[0])
-    gbifIndex[gbifId].sort(numericSort)
+    index[id].push(taxon[0])
+    index[id].sort(numericSort)
 }
 
 async function main (args: string[]): Promise<void> {
@@ -35,6 +38,7 @@ async function main (args: string[]): Promise<void> {
     const files = await fs.readdir(path.join(REPO_ROOT, 'txt'))
 
     const gbifIndex: Record<string, TaxonId[]> = {}
+    const colIndex: Record<string, TaxonId[]> = {}
     const resourceIndex: Record<TaxonId, AmendedResourceMetadata> = {}
 
     await Promise.all(files.map(async function (fileName) {
@@ -57,6 +61,8 @@ async function main (args: string[]): Promise<void> {
             const [header, ...dwc] = csv.parseCsv(await fs.readFile(dwcFile, 'utf-8'))
             const gbifColumn = header.indexOf('gbifTaxonID')
             const gbifAcceptedColumn = header.indexOf('gbifAcceptedTaxonID')
+            const colColumn = header.indexOf('colTaxonID')
+            const colAcceptedColumn = header.indexOf('colAcceptedTaxonID')
             for (const taxon of dwc) {
                 const gbifId = taxon[gbifColumn]
                 if (gbifId) {
@@ -65,6 +71,15 @@ async function main (args: string[]): Promise<void> {
                         addTaxon(gbifIndex, taxon[gbifAcceptedColumn], taxon)
                     }
                 }
+
+                const colId = taxon[colColumn]
+                if (colId) {
+                    addTaxon(colIndex, colId, taxon)
+                    if (taxon[colAcceptedColumn] !== taxon[colColumn]) {
+                        addTaxon(colIndex, taxon[colAcceptedColumn], taxon)
+                    }
+                }
+
                 amendedResource.taxonCount += 1
             }
 
@@ -74,6 +89,7 @@ async function main (args: string[]): Promise<void> {
 
     await Promise.all([
         fs.writeFile(path.join(REPO_ROOT, 'gbif.index.json'), JSON.stringify(sortObject(gbifIndex), null, 2)),
+        fs.writeFile(path.join(REPO_ROOT, 'col.index.json'), JSON.stringify(sortObject(colIndex, alphabeticSort), null, 2)),
         fs.writeFile(path.join(REPO_ROOT, 'index.json'), JSON.stringify(sortObject(resourceIndex), null, 2))
     ])
 }

package/src/bin/process-resources.ts

@@ -175,7 +175,6 @@ class ResourceProcessor {
             const skip = await this.shouldBeSkipped(resource.id)
 
             if (!skip) {
-                // TODO const correct = checkResults(results, classifications)
                 const correct = this.checkResults(results)
                 if (!correct) {
                     const choice = await promptForAnswers(

package/src/module.d.ts

@@ -71,8 +71,11 @@ interface Taxon extends WorkingTaxon {
     verbatimIdentification: string
 }
 
+type ResourceFlag = string;
+
 interface ResourceMetadata {
     levels: Rank[],
+    flags?: ResourceFlag[],
     catalog?: Record<string, Value>
 }
 

package/src/resources/parse-text.ts

@@ -54,6 +54,12 @@ const DWC_RANKS: DwcRank[] = [
     'subgenus'
 ]
 
+const FLAGS: ResourceFlag[] = [
+    'MISSING_TAXA',
+    'MISSING_PARENT_TAXA',
+    'MISSING_SYNONYMS'
+]
+
 const TAXONOMIC_STATUS: Record<string, TaxonStatus> = {
     '>': 'incorrect',
     '+': 'heterotypic synonym',
@@ -87,6 +93,8 @@ const RANK_LABELS_REVERSE: Record<string, Rank> = {
     'subsp': 'subspecies'
 }
 
+const HYBRID_SIGN = '\u00D7'
+
 /**
  *   1. Any number of
  *      - capitalized words
@@ -97,7 +105,7 @@ const RANK_LABELS_REVERSE: Record<string, Rank> = {
  *   2. Followed by a capitalized word
  *   3. Optionally, followed by "et al."
  */
-const LOWERCASE_NAME_PARTICLES = ['y', 'der', 'den', 'de', 'van'].join('|')
+const LOWERCASE_NAME_PARTICLES = ['y', 'der', 'den', 'de', 'van', 'von'].join('|')
 const SIMPLE_AUTHOR_PATTERN = '(?:(?:\\p{Lu}\\S*|&|in|ex|' + LOWERCASE_NAME_PARTICLES + ')\\s*)*\\p{Lu}\\S+(?:\\s+et\\s+al\\.)?'
 
 const NAME_PATTERN = new RegExp(
@@ -107,7 +115,7 @@ const NAME_PATTERN = new RegExp(
         // $2 optional author citation
         '(?: ' +
             // but not auct(t)., etc.
-            '(?!auctt?\\.|(?:syn|comb|sp|spec)\\. n(?:ov)?\\.|s(?:ens[.u]|\\.))' +
+            '(?!auctt?\\.|(?:syn|comb|sp|spec|nom)\\. n(?:ov)?\\.|s(?:ens[.u]|\\.)|in part|partim)' +
         '(' +
             // $2.1 anything in parentheses, followed by optional revising author(s)
             '\\(.+?\\)(?:\\s+' + SIMPLE_AUTHOR_PATTERN + ')?' +
@@ -127,11 +135,14 @@ const NAME_PATTERN = new RegExp(
 /**
  * Structure
  *   $1 genus+subgenus (+ trailing space): (?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?
- *     $1.1 genus: ([A-Z]\S+)
+ *     $1.1 genus: ((?:x )?[A-Z]\S+)
  *     $1.2 subgenus: (?:\(([A-Z]\S+?)\) )?
- *   $2 species: ((?:x )?[a-z][^\s.]+)
+ *   $2 species: (x [a-z]+|[a-z][^\s.]+(?: x [a-z]+)?|[A-Z][a-z]+_[a-z]+ x [A-Z][a-z]+_[a-z]+)
+ *     $2a: x [a-z]+
+ *     $2b hybrid: [a-z][^\s.]+(?: x [a-z]+)?
+ *     $2c intergeneric hybrid: [A-Z][a-z]+_[a-z]+ x [A-Z][a-z]+_[a-z]+
  */
-const BINAME_PATTERN = /^(?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?((?:x )?[a-z][^\s.]+)(?= |$)/
+const BINAME_PATTERN = /^(?:((?:x )?[A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?(x [a-z]+|[a-z][^\s.]+(?: x [a-z]+)?|[A-Z][a-z]+_[a-z]+ x [A-Z][a-z]+_[a-z]+)(?= |$)/
 
 function compareRanks (a: Rank, b: Rank): number {
     return RANKS.indexOf(a) - RANKS.indexOf(b)
@@ -141,12 +152,19 @@ function capitalize (name: string): string {
     return name[0].toUpperCase() + name.slice(1).toLowerCase()
 }
 
+function capitalizeGenericName (name: string): string {
+    if (name[0] === HYBRID_SIGN) {
+        return HYBRID_SIGN + capitalize(name.slice(1))
+    }
+
+    return capitalize(name)
+}
+
 function isUpperCase (name: string): boolean {
     return name === name.toUpperCase()
 }
 
 function getSynonymRank (name: string, rank: Rank): Rank {
-    const BINAME_PATTERN = /^([A-Z]\S+ (\([A-Z]\S+\) )?)?(x )?[a-z0-9-]+(?= |$)/
     const rest = name.replace(BINAME_PATTERN, '')
     const rankPrefix = rest.match(/^(?: |^)(st|r|ab|f|var|ssp|subsp)\. /)
     if (rankPrefix) {
@@ -188,7 +206,7 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
     }
 
     // Set verbatim identification after subsequent syntax is removed.
-    item.verbatimIdentification = name
+    item.verbatimIdentification = name.replace(/(?<=^| )x(?=$| )/g, HYBRID_SIGN).replace(/_/g, ' ')
 
     // Parent context is used for parsing and formatting binomial names.
     // For formatting, it needs to match external databases (i.e. be correct).
@@ -212,7 +230,8 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
     if (isSynonym || !parentContext.genus || (compareRanks('species', rank) < 0 && !parentContext.specificEpithet)) {
         const [, genus, subgenus, species] = name.match(BINAME_PATTERN) || []
         if (genus) {
-            parentContext.genus = parentContext.incorrect.genus = capitalize(genus)
+            parentContext.incorrect.genus = genus
+            parentContext.genus = capitalizeGenericName(genus.replace(/(^| )x /, HYBRID_SIGN))
         }
         if (subgenus) {
             parentContext.subgenus = capitalize(subgenus)
@@ -221,8 +240,9 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
             // from the parent context.
             delete parentContext.subgenus
         }
-        if (species) {
-            parentContext.specificEpithet = parentContext.incorrect.specificEpithet = species
+        if (species && compareRanks('species', rank) < 0) {
+            parentContext.incorrect.specificEpithet = species
+            parentContext.specificEpithet = species.replace(/(^| )x /, HYBRID_SIGN)
         }
     }
 
@@ -253,8 +273,12 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
     }
 
     // Hybrids
+    if (rank === 'genus' && name.startsWith('x ')) {
+        name = HYBRID_SIGN + name.slice(2)
+    }
+
     if (rank === 'species' && /(^| )x /.test(name)) {
-        name = name.replace(/(^| )x /, '\u00D7')
+        name = name.replace(/(^| )x /, HYBRID_SIGN)
     }
 
     // Divide the name into the main scientific name (only the epithet for taxa
@@ -265,7 +289,9 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
     }
 
     // To encode old names with spaces (e.g. "Orsillus pini canariensis Lindberg, 1953")
-    // underscores are used, which are replaced here.
+    // underscores are used, which are replaced here. This is also used for undescribed
+    // species (e.g. "Leiobunum species A") and intergeneric hybrids (e.g. "×Festulpia
+    // Festuca rubra × Vulpia bromoides")
     if (nameParts[1].includes('_')) {
         nameParts[1] = nameParts[1].replace(/_/g, ' ')
     }
@@ -280,9 +306,14 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
     }
 
     // Validate names and recompose binomial and trinomial names
-    if (compareRanks('group', rank) > 0) {
+    if (rank === 'genus') {
+        item.scientificName = capitalizeGenericName(taxon)
+        if (taxon[0].toUpperCase() !== taxon[0] || (taxon[0] === HYBRID_SIGN && taxon[1].toUpperCase() !== taxon[1])) {
+            throw new Error(`Generic epithet should be capitalized: "${taxon}"`)
+        }
+    } else if (compareRanks('group', rank) > 0) {
         item.scientificName = capitalize(taxon)
-        if (item.scientificName[0] !== taxon[0]) {
+        if (taxon[0].toUpperCase() !== taxon[0]) {
             throw new Error(`Taxon name (${rank}) should be capitalized: "${taxon}"`)
         }
     } else if (rank === 'group') {
@@ -306,12 +337,12 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
     } else if (rank === 'species') {
         item.genericName = parentContext.genus
         item.infragenericEpithet = parentContext.subgenus
-        item.specificEpithet = taxon.toLowerCase()
-        item.scientificName = `${item.genericName} ${item.specificEpithet}`
-        if (item.specificEpithet !== taxon) {
+        if (taxon.toLowerCase() !== taxon && !/^[A-Z][a-z]+ [a-z]+\xD7[A-Z][a-z]+ [a-z]+$/.test(taxon)) {
             console.log(item, taxon)
             throw new Error(`Specific epithet should be lowercase: "${taxon}"`)
         }
+        item.specificEpithet = taxon
+        item.scientificName = `${item.genericName} ${item.specificEpithet}`
     } else if (compareRanks('species', rank) < 0) {
         item.genericName = parentContext.genus
         item.infragenericEpithet = parentContext.subgenus
@@ -342,19 +373,6 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
         item.scientificName += ` ${item.scientificNameAuthorship}`
     }
 
-    // Amend "parent" with corrections
-    if (item.taxonomicStatus === 'incorrect') {
-        const itemAsObject = item as { [index: string]: unknown }
-        const parentAsObject = parent as { [index: string]: unknown }
-
-        parent.incorrect = { ...parent }
-        for (const key in item) {
-            if (key !== 'taxonomicStatus') {
-                parentAsObject[key] = itemAsObject[key]
-            }
-        }
-    }
-
     return item
 }
 
@@ -418,6 +436,19 @@ function parseHeader (header: string): ResourceMetadata {
         }
     }
 
+    if ('flags' in config) {
+        if (!Array.isArray(config.flags)) {
+            throw new SyntaxError('"flags" should be an array if present')
+        }
+
+        const invalidFlags = config.flags.filter(flag => !FLAGS.includes(flag))
+        if (invalidFlags.length) {
+            throw new SyntaxError(`"flags" contains invalid values: ${invalidFlags.join(', ')}`)
+        }
+
+        metadata.flags = config.flags
+    }
+
     return metadata
 }
 
@@ -510,29 +541,10 @@ function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds
         const item = parseName(name, rank, parent)
         const isSynonym = item.taxonomicStatus !== 'accepted'
 
-        if (item.taxonomicStatus === 'incorrect') {
-            continue
-        }
-
-        if (type === ResourceDiffType.Added) {
-            newIdOffset++
-            item.scientificNameID = idBase + newIdOffset.toString()
-        } else {
-            id++
-            item.scientificNameID = idBase + (oldIds[id - 1] || id).toString()
-        }
-        previousId = item.scientificNameID
-
-        item.parentNameUsageID = isSynonym ? undefined : parent.scientificNameID
-        item.parentNameUsage = isSynonym ? undefined : parent.scientificName
-        item.acceptedNameUsageID = isSynonym ? parent.scientificNameID : undefined
-        item.acceptedNameUsage = isSynonym ? parent.scientificName : undefined
-        item.collectionCode = idBase.slice(0, -1)
-
+        // Add higher classification info
+        const itemAsObject = item as { [index: string]: unknown }
+        const parentAsObject = parent as { [index: string]: unknown }
         for (const rank of DWC_RANKS) {
-            const itemAsObject = item as { [index: string]: unknown }
-            const parentAsObject = parent as { [index: string]: unknown }
-
             itemAsObject[rank] = undefined
             if (parentAsObject[rank]) {
                 itemAsObject[rank] = parentAsObject[rank]
@@ -557,6 +569,33 @@ function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds
             item.higherClassification = parent.scientificNameOnly
         }
 
+        // Amend "parent" with corrections
+        if (item.taxonomicStatus === 'incorrect') {
+            parent.incorrect = { ...parent }
+            for (const key in item) {
+                if (key !== 'taxonomicStatus' && key !== 'verbatimIdentification') {
+                    parentAsObject[key] = itemAsObject[key]
+                }
+            }
+            continue
+        }
+
+        // Set identifiers
+        if (type === ResourceDiffType.Added) {
+            newIdOffset++
+            item.scientificNameID = idBase + newIdOffset.toString()
+        } else {
+            id++
+            item.scientificNameID = idBase + (oldIds[id - 1] || id).toString()
+        }
+        previousId = item.scientificNameID
+
+        item.parentNameUsageID = isSynonym ? undefined : parent.scientificNameID
+        item.parentNameUsage = isSynonym ? undefined : parent.scientificName
+        item.acceptedNameUsageID = isSynonym ? parent.scientificNameID : undefined
+        item.acceptedNameUsage = isSynonym ? parent.scientificName : undefined
+        item.collectionCode = idBase.slice(0, -1)
+
         data[item.scientificNameID] = item
     }
 

package/test/resources.js

@@ -136,4 +136,43 @@ Tilia x vulgaris
         assert.strictEqual(resource.taxa['T1:1:1'].scientificName, 'Rumex conglomeratus×maritimus')
         assert.strictEqual(resource.taxa['T1:1:2'].scientificName, 'Tilia ×vulgaris')
     })
+
+    await t.test('outputs corrected generic names', (t) => {
+        const [resource] = resources.parseTextFile(`---
+levels: [genus, species]
+---
+
+Bogdania Kerzhner, 1964
+  > Bogdiana Kerzhner, 1964
+  myrmica Kerzhner, 1964
+`, 'T1')
+        assert.strictEqual(resource.taxa['T1:1:1'].scientificName, 'Bogdiana Kerzhner, 1964')
+        assert.strictEqual(resource.taxa['T1:1:2'].scientificName, 'Bogdiana myrmica Kerzhner, 1964')
+    })
+
+    await t.test('parses cross-genus hybrids', (t) => {
+        const [resource] = resources.parseTextFile(`---
+levels: [genus, species]
+---
+
+x Triticosecale
+  Triticosecale indet.
+x Festulpia
+  Festuca_rubra x Vulpia_bromoides
+`, 'T1')
+        assert.strictEqual(resource.taxa['T1:1:1'].scientificName, '×Triticosecale')
+        assert.strictEqual(resource.taxa['T1:1:2'].scientificName, '×Festulpia')
+        assert.strictEqual(resource.taxa['T1:1:3'].scientificName, '×Festulpia Festuca rubra×Vulpia bromoides')
+    })
+
+    await t.test('parses cross-genus hybrids without parent context', (t) => {
+        const [resource] = resources.parseTextFile(`---
+levels: [species]
+---
+
+x Festulpia Festuca_rubra x Vulpia_bromoides
+`, 'T1')
+        assert.strictEqual(resource.taxa['T1:1:1'].scientificName, '×Festulpia Festuca rubra×Vulpia bromoides')
+        assert.strictEqual(resource.taxa['T1:1:1'].verbatimIdentification, '× Festulpia Festuca rubra × Vulpia bromoides')
+    })
 })