@larsgw/formica 0.6.7 → 0.7.0

package/CHANGELOG.md

@@ -1,3 +1,33 @@
+# [0.7.0](https://github.com/identification-resources/formica/compare/v0.6.8...v0.7.0) (2025-03-14)
+
+
+### Bug Fixes
+
+* **resources:** improve diff regarding indet lines ([2615437](https://github.com/identification-resources/formica/commit/26154374f50d680c5f3e187cd6a1a5eadbac6b5d))
+
+
+### Features
+
+* **resources:** change syntax of "indet" line ([d53363e](https://github.com/identification-resources/formica/commit/d53363e65bfa77a2eb247999316621f9ca0bff59))
+
+
+### BREAKING CHANGES
+
+* **resources:** "indet" lines now have to be prefixed with "[indet]"
+
+
+
+## [0.6.8](https://github.com/identification-resources/formica/compare/v0.6.7...v0.6.8) (2025-03-13)
+
+
+### Features
+
+* **resources:** add support for intergeneric hybrids without parents ([3cb0c0c](https://github.com/identification-resources/formica/commit/3cb0c0cd60fc5a4ba88add623fbadc3b90d8530c))
+* **resources:** create index of CoL identifiers ([3792903](https://github.com/identification-resources/formica/commit/37929035374bc0c67ed2403e775e06a904bdbd4d))
+* **resources:** support intergeneric hybrids ([0a6696c](https://github.com/identification-resources/formica/commit/0a6696c5fd0603a0366c80c2298b6544636ed0c6))
+
+
+
 ## [0.6.7](https://github.com/identification-resources/formica/compare/v0.6.6...v0.6.7) (2025-02-26)
 
 

package/lib/bin/process-resources-index.js

@@ -52,26 +52,27 @@ var fs_1 = require("fs");
 var path = require("path");
 var index_1 = require("../index");
 var util_1 = require("./util");
-/* eslint-disable @typescript-eslint/no-explicit-any */
-function sortObject(object) {
+function alphabeticSort(a, b) {
+    return a > b ? 1 : a < b ? -1 : 0;
+}
+function sortObject(object, sorter) {
     var sorted = {};
-    for (var _i = 0, _a = Object.keys(object).sort(util_1.numericSort); _i < _a.length; _i++) {
+    for (var _i = 0, _a = Object.keys(object).sort(sorter !== null && sorter !== void 0 ? sorter : util_1.numericSort); _i < _a.length; _i++) {
         var key = _a[_i];
         sorted[key] = object[key];
     }
     return sorted;
 }
-/* eslint-enable @typescript-eslint/no-explicit-any */
-function addTaxon(gbifIndex, gbifId, taxon) {
-    if (!(gbifId in gbifIndex)) {
-        gbifIndex[gbifId] = [];
+function addTaxon(index, id, taxon) {
+    if (!(id in index)) {
+        index[id] = [];
     }
-    gbifIndex[gbifId].push(taxon[0]);
-    gbifIndex[gbifId].sort(util_1.numericSort);
+    index[id].push(taxon[0]);
+    index[id].sort(util_1.numericSort);
 }
 function main(args) {
     return __awaiter(this, void 0, void 0, function () {
-        var REPO_ROOT, files, gbifIndex, resourceIndex;
+        var REPO_ROOT, files, gbifIndex, colIndex, resourceIndex;
         return __generator(this, function (_a) {
             switch (_a.label) {
                 case 0:
@@ -80,6 +81,7 @@ function main(args) {
                 case 1:
                     files = _a.sent();
                     gbifIndex = {};
+                    colIndex = {};
                     resourceIndex = {};
                     return [4 /*yield*/, Promise.all(files.map(function (fileName) {
                             return __awaiter(this, void 0, void 0, function () {
@@ -96,7 +98,7 @@ function main(args) {
                                             file = _a.sent();
                                             return [2 /*return*/, Promise.all(index_1.resources.parseTextFileHeader(file).map(function (resource, index) {
                                                     return __awaiter(this, void 0, void 0, function () {
-                                                        var amendedResource, dwcFile, _a, header, dwc, _b, _c, gbifColumn, gbifAcceptedColumn, _i, dwc_1, taxon, gbifId;
+                                                        var amendedResource, dwcFile, _a, header, dwc, _b, _c, gbifColumn, gbifAcceptedColumn, colColumn, colAcceptedColumn, _i, dwc_1, taxon, gbifId, colId;
                                                         return __generator(this, function (_d) {
                                                             switch (_d.label) {
                                                                 case 0:
@@ -111,6 +113,8 @@ function main(args) {
                                                                     _a = _c.apply(_b, [_d.sent()]), header = _a[0], dwc = _a.slice(1);
                                                                     gbifColumn = header.indexOf('gbifTaxonID');
                                                                     gbifAcceptedColumn = header.indexOf('gbifAcceptedTaxonID');
+                                                                    colColumn = header.indexOf('colTaxonID');
+                                                                    colAcceptedColumn = header.indexOf('colAcceptedTaxonID');
                                                                     for (_i = 0, dwc_1 = dwc; _i < dwc_1.length; _i++) {
                                                                         taxon = dwc_1[_i];
                                                                         gbifId = taxon[gbifColumn];
@@ -120,6 +124,13 @@ function main(args) {
                                                                                 addTaxon(gbifIndex, taxon[gbifAcceptedColumn], taxon);
                                                                             }
                                                                         }
+                                                                        colId = taxon[colColumn];
+                                                                        if (colId) {
+                                                                            addTaxon(colIndex, colId, taxon);
+                                                                            if (taxon[colAcceptedColumn] !== taxon[colColumn]) {
+                                                                                addTaxon(colIndex, taxon[colAcceptedColumn], taxon);
+                                                                            }
+                                                                        }
                                                                         amendedResource.taxonCount += 1;
                                                                     }
                                                                     resourceIndex[amendedResource.id] = amendedResource;
@@ -136,6 +147,7 @@ function main(args) {
                     _a.sent();
                     return [4 /*yield*/, Promise.all([
                             fs_1.promises.writeFile(path.join(REPO_ROOT, 'gbif.index.json'), JSON.stringify(sortObject(gbifIndex), null, 2)),
+                            fs_1.promises.writeFile(path.join(REPO_ROOT, 'col.index.json'), JSON.stringify(sortObject(colIndex, alphabeticSort), null, 2)),
                             fs_1.promises.writeFile(path.join(REPO_ROOT, 'index.json'), JSON.stringify(sortObject(resourceIndex), null, 2))
                         ])];
                 case 3:

package/lib/resources/diff-resource.d.ts

@@ -4,4 +4,4 @@ export declare enum ResourceDiffType {
     Modified = "~",
     Unchanged = "="
 }
-export declare function createDiff(a: string, b: string, tokenize?: ResourceDiffTokenizer): ResourceDiffPart[];
+export declare function createDiff(a: string, b: string, tokenize?: ResourceDiffTokenizer): ResourceDiff;

package/lib/resources/diff-resource.js

@@ -68,6 +68,7 @@ function gitTokenize(text) {
     return text.match(/\S+|\n|[\r\t\f\v \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+/g);
 }
 function createDiff(a, b, tokenize) {
+    var _a, _b;
     if (tokenize === void 0) { tokenize = gitTokenize; }
     var X = tokenize(a.trimEnd());
     var Y = tokenize(b.trimEnd());
@@ -106,7 +107,7 @@ function createDiff(a, b, tokenize) {
         // Start of line
         if (line === null) {
             deletedNewlines = 0;
-            line = { text: '', type: change.type };
+            line = { type: change.type };
         }
         // End of line (could be same token)
         if (change.text === '\n') {
@@ -132,7 +133,8 @@ function createDiff(a, b, tokenize) {
             // Add placeholders for deleted lines
             while (deletedNewlines--) {
                 lines.push({
-                    text: '',
+                    text: undefined,
+                    original: '',
                     type: ResourceDiffType.Deleted
                 });
             }
@@ -144,7 +146,10 @@ function createDiff(a, b, tokenize) {
             line.type = ResourceDiffType.Modified;
         }
         if (change.type !== ResourceDiffType.Deleted) {
-            line.text += change.text;
+            line.text = ((_a = line.text) !== null && _a !== void 0 ? _a : '') + change.text;
+        }
+        if (change.type !== ResourceDiffType.Added) {
+            line.original = ((_b = line.original) !== null && _b !== void 0 ? _b : '') + change.text;
         }
     }
     return lines;

package/lib/resources/parse-text.js

@@ -74,13 +74,6 @@ var TAXONOMIC_STATUS = {
     '+': 'heterotypic synonym',
     '=': 'synonym'
 };
-var INDET_SUFFIXES = new Set([
-    'sp.',
-    'spec.',
-    'indet.',
-    'sp. indet.',
-    'spec. indet.'
-]);
 var RANK_LABELS = {
     'subspecies': 'subsp.',
     'variety': 'var.',
@@ -98,6 +91,7 @@ var RANK_LABELS_REVERSE = {
     'ssp': 'subspecies',
     'subsp': 'subspecies'
 };
+var HYBRID_SIGN = '\u00D7';
 /**
  *   1. Any number of
  *      - capitalized words
@@ -133,22 +127,30 @@ var NAME_PATTERN = new RegExp('^' +
 /**
  * Structure
  *   $1 genus+subgenus (+ trailing space): (?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?
- *     $1.1 genus: ([A-Z]\S+)
+ *     $1.1 genus: ((?:x )?[A-Z]\S+)
  *     $1.2 subgenus: (?:\(([A-Z]\S+?)\) )?
- *   $2 species: ((?:x )?[a-z][^\s.]+)
+ *   $2 species: (x [a-z]+|[a-z][^\s.]+(?: x [a-z]+)?|[A-Z][a-z]+_[a-z]+ x [A-Z][a-z]+_[a-z]+)
+ *     $2a: x [a-z]+
+ *     $2b hybrid: [a-z][^\s.]+(?: x [a-z]+)?
+ *     $2c intergeneric hybrid: [A-Z][a-z]+_[a-z]+ x [A-Z][a-z]+_[a-z]+
  */
-var BINAME_PATTERN = /^(?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?((?:x )?[a-z][^\s.]+)(?= |$)/;
+var BINAME_PATTERN = /^(?:((?:x )?[A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?(x [a-z]+|[a-z][^\s.]+(?: x [a-z]+)?|[A-Z][a-z]+_[a-z]+ x [A-Z][a-z]+_[a-z]+)(?= |$)/;
 function compareRanks(a, b) {
     return RANKS.indexOf(a) - RANKS.indexOf(b);
 }
 function capitalize(name) {
     return name[0].toUpperCase() + name.slice(1).toLowerCase();
 }
+function capitalizeGenericName(name) {
+    if (name[0] === HYBRID_SIGN) {
+        return HYBRID_SIGN + capitalize(name.slice(1));
+    }
+    return capitalize(name);
+}
 function isUpperCase(name) {
     return name === name.toUpperCase();
 }
 function getSynonymRank(name, rank) {
-    var BINAME_PATTERN = /^([A-Z]\S+ (\([A-Z]\S+\) )?)?(x )?[a-z0-9-]+(?= |$)/;
     var rest = name.replace(BINAME_PATTERN, '');
     var rankPrefix = rest.match(/^(?: |^)(st|r|ab|f|var|ssp|subsp)\. /);
     if (rankPrefix) {
@@ -188,7 +190,7 @@ function parseName(name, rank, parent) {
         name = name.replace(/^\[(_|\d+)\] /, '');
     }
     // Set verbatim identification after subsequent syntax is removed.
-    item.verbatimIdentification = name;
+    item.verbatimIdentification = name.replace(/(?<=^| )x(?=$| )/g, HYBRID_SIGN).replace(/_/g, ' ');
     // Parent context is used for parsing and formatting binomial names.
     // For formatting, it needs to match external databases (i.e. be correct).
     // For parsing, it needs to match the current file. If relevant parents
@@ -210,7 +212,8 @@ function parseName(name, rank, parent) {
     if (isSynonym || !parentContext.genus || (compareRanks('species', rank) < 0 && !parentContext.specificEpithet)) {
         var _a = name.match(BINAME_PATTERN) || [], genus = _a[1], subgenus = _a[2], species = _a[3];
         if (genus) {
-            parentContext.genus = parentContext.incorrect.genus = capitalize(genus);
+            parentContext.incorrect.genus = genus;
+            parentContext.genus = capitalizeGenericName(genus.replace(/(^| )x /, HYBRID_SIGN));
         }
         if (subgenus) {
             parentContext.subgenus = capitalize(subgenus);
@@ -220,8 +223,9 @@ function parseName(name, rank, parent) {
             // from the parent context.
             delete parentContext.subgenus;
         }
-        if (species) {
-            parentContext.specificEpithet = parentContext.incorrect.specificEpithet = species;
+        if (species && compareRanks('species', rank) < 0) {
+            parentContext.incorrect.specificEpithet = species;
+            parentContext.specificEpithet = species.replace(/(^| )x /, HYBRID_SIGN);
         }
     }
     // In taxa of group, species or lower, the name should just contain the
@@ -247,8 +251,11 @@ function parseName(name, rank, parent) {
         }
     }
     // Hybrids
+    if (rank === 'genus' && name.startsWith('x ')) {
+        name = HYBRID_SIGN + name.slice(2);
+    }
     if (rank === 'species' && /(^| )x /.test(name)) {
-        name = name.replace(/(^| )x /, '\u00D7');
+        name = name.replace(/(^| )x /, HYBRID_SIGN);
     }
     // Divide the name into the main scientific name (only the epithet for taxa
     // lower than genus), the authorship information, and optionally remarks
@@ -257,7 +264,9 @@ function parseName(name, rank, parent) {
         throw new Error("Taxon \"".concat(name, "\" could not be parsed"));
     }
     // To encode old names with spaces (e.g. "Orsillus pini canariensis Lindberg, 1953")
-    // underscores are used, which are replaced here.
+    // underscores are used, which are replaced here. This is also used for undescribed
+    // species (e.g. "Leiobunum species A") and intergeneric hybrids (e.g. "×Festulpia
+    // Festuca rubra × Vulpia bromoides")
     if (nameParts[1].includes('_')) {
         nameParts[1] = nameParts[1].replace(/_/g, ' ');
     }
@@ -265,13 +274,19 @@ function parseName(name, rank, parent) {
     item.scientificNameAuthorship = capitalizeAuthors(citation);
     item.taxonRemarks = notes;
     item.taxonRank = rank;
-    if (/[^\p{L}0-9\u{00D7}\- ]/u.test(taxon) && !INDET_SUFFIXES.has(taxon)) {
+    if (/[^\p{L}0-9\u{00D7}\- ]/u.test(taxon)) {
         throw new Error("Taxon name contains unexpected characters: \"".concat(taxon, "\""));
     }
     // Validate names and recompose binomial and trinomial names
-    if (compareRanks('group', rank) > 0) {
+    if (rank === 'genus') {
+        item.scientificName = capitalizeGenericName(taxon);
+        if (taxon[0].toUpperCase() !== taxon[0] || (taxon[0] === HYBRID_SIGN && taxon[1].toUpperCase() !== taxon[1])) {
+            throw new Error("Generic epithet should be capitalized: \"".concat(taxon, "\""));
+        }
+    }
+    else if (compareRanks('group', rank) > 0) {
         item.scientificName = capitalize(taxon);
-        if (item.scientificName[0] !== taxon[0]) {
+        if (taxon[0].toUpperCase() !== taxon[0]) {
             throw new Error("Taxon name (".concat(rank, ") should be capitalized: \"").concat(taxon, "\""));
         }
     }
@@ -298,12 +313,12 @@ function parseName(name, rank, parent) {
     else if (rank === 'species') {
         item.genericName = parentContext.genus;
         item.infragenericEpithet = parentContext.subgenus;
-        item.specificEpithet = taxon.toLowerCase();
-        item.scientificName = "".concat(item.genericName, " ").concat(item.specificEpithet);
-        if (item.specificEpithet !== taxon) {
+        if (taxon.toLowerCase() !== taxon && !/^[A-Z][a-z]+ [a-z]+\xD7[A-Z][a-z]+ [a-z]+$/.test(taxon)) {
             console.log(item, taxon);
             throw new Error("Specific epithet should be lowercase: \"".concat(taxon, "\""));
         }
+        item.specificEpithet = taxon;
+        item.scientificName = "".concat(item.genericName, " ").concat(item.specificEpithet);
     }
     else if (compareRanks('species', rank) < 0) {
         item.genericName = parentContext.genus;
@@ -435,6 +450,16 @@ function parseResource(resource) {
     var content = rest.join('');
     return [config, content];
 }
+function getIndentation(line) {
+    return line.match(/^ */)[0].length;
+}
+function isIndetLine(line, indent) {
+    if (indent === undefined) {
+        indent = getIndentation(line);
+    }
+    line = line.slice(indent);
+    return line.startsWith('[indet]');
+}
 function parseResourceContent(content, resource, oldIds) {
     var idBase = "".concat(resource.id, ":");
     var data = resource.taxa;
@@ -443,20 +468,19 @@ function parseResourceContent(content, resource, oldIds) {
     var groupIndent = 0;
     var previousId = '';
     var newIdOffset = Math.max.apply(Math, oldIds);
-    var _loop_1 = function (line, type) {
-        if (type === diff_resource_1.ResourceDiffType.Deleted) {
-            id++;
-            return "continue";
-        }
-        // Do not process "indet" lines further, as they only serve to indicate
-        // that subtaxa are explicitely omitted
-        if (Array.from(INDET_SUFFIXES).some(function (suffix) { return line.endsWith(' ' + suffix); })) {
-            return "continue";
+    for (var _i = 0, content_1 = content; _i < content_1.length; _i++) {
+        var line = content_1[_i];
+        if (line.type === diff_resource_1.ResourceDiffType.Deleted) {
+            // Increase id counter for removed line unless it was an "indet line"
+            if (!isIndetLine(line.original)) {
+                id++;
+            }
+            continue;
         }
-        var lineIndent = line.match(/^ */)[0].length;
+        var lineIndent = getIndentation(line.text);
         if (lineIndent > groupIndent) {
             // Do not count synonyms as parents (unless this is correcting a typo in the synonym)
-            if (data[previousId] && data[previousId].taxonomicStatus === 'accepted' || /^( {2})+> /.test(line)) {
+            if (data[previousId] && data[previousId].taxonomicStatus === 'accepted' || /^( {2})+> /.test(line.text)) {
                 parents.push(previousId);
             }
             else {
@@ -478,17 +502,26 @@ function parseResourceContent(content, resource, oldIds) {
             parents = parents.slice(0, lineIndent / 2);
             groupIndent = lineIndent;
         }
+        // Do not process "indet" lines further, as they only serve to indicate
+        // that subtaxa are explicitely omitted
+        if (isIndetLine(line.text, lineIndent)) {
+            // If the line was previously not and ndet line, increase the id counter
+            if (line.type === diff_resource_1.ResourceDiffType.Modified && !isIndetLine(line.original)) {
+                id++;
+            }
+            continue;
+        }
         var parentId = parents.reduce(function (grandparent, parent) { return parent || grandparent; }, null);
         var parent_1 = parentId === null ? {} : data[parentId];
-        var name_1 = line.slice(groupIndent);
+        var name_1 = line.text.slice(groupIndent);
         var rank = resource.metadata.levels[groupIndent / 2];
         var item = parseName(name_1, rank, parent_1);
         var isSynonym = item.taxonomicStatus !== 'accepted';
         // Add higher classification info
         var itemAsObject = item;
         var parentAsObject = parent_1;
-        for (var _b = 0, DWC_RANKS_1 = DWC_RANKS; _b < DWC_RANKS_1.length; _b++) {
-            var rank_1 = DWC_RANKS_1[_b];
+        for (var _a = 0, DWC_RANKS_1 = DWC_RANKS; _a < DWC_RANKS_1.length; _a++) {
+            var rank_1 = DWC_RANKS_1[_a];
             itemAsObject[rank_1] = undefined;
             if (parentAsObject[rank_1]) {
                 itemAsObject[rank_1] = parentAsObject[rank_1];
@@ -520,10 +553,14 @@ function parseResourceContent(content, resource, oldIds) {
                     parentAsObject[key] = itemAsObject[key];
                 }
             }
-            return "continue";
+            continue;
         }
         // Set identifiers
-        if (type === diff_resource_1.ResourceDiffType.Added) {
+        if (line.type === diff_resource_1.ResourceDiffType.Added) {
+            newIdOffset++;
+            item.scientificNameID = idBase + newIdOffset.toString();
+        }
+        else if (line.type === diff_resource_1.ResourceDiffType.Modified && isIndetLine(line.original)) {
             newIdOffset++;
             item.scientificNameID = idBase + newIdOffset.toString();
         }
@@ -538,10 +575,6 @@ function parseResourceContent(content, resource, oldIds) {
         item.acceptedNameUsage = isSynonym ? parent_1.scientificName : undefined;
         item.collectionCode = idBase.slice(0, -1);
         data[item.scientificNameID] = item;
-    };
-    for (var _i = 0, content_1 = content; _i < content_1.length; _i++) {
-        var _a = content_1[_i], line = _a.text, type = _a.type;
-        _loop_1(line, type);
     }
     return resource;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larsgw/formica",
-  "version": "0.6.7",
+  "version": "0.7.0",
   "description": "SDK and tools for data from the Library of Identification Resources",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",

package/src/bin/process-resources-index.ts

@@ -11,22 +11,25 @@ interface AmendedResourceMetadata extends ResourceMetadata {
     taxonCount: number
 }
 
-/* eslint-disable @typescript-eslint/no-explicit-any */
-function sortObject (object: Record<string, any>): Record<string, any> {
-    const sorted: Record<string, any> = {}
-    for (const key of Object.keys(object).sort(numericSort)) {
+type SortObjectCallback = (a: string, b: string) => number
+function alphabeticSort (a: string, b: string): number {
+    return a > b ? 1 : a < b ? -1 : 0
+}
+
+function sortObject (object: Record<string, unknown>, sorter?: SortObjectCallback): Record<string, unknown> {
+    const sorted: Record<string, unknown> = {}
+    for (const key of Object.keys(object).sort(sorter ?? numericSort)) {
         sorted[key] = object[key]
     }
     return sorted
 }
-/* eslint-enable @typescript-eslint/no-explicit-any */
 
-function addTaxon (gbifIndex: Record<string, TaxonId[]>, gbifId: string, taxon: string[]) {
-    if (!(gbifId in gbifIndex)) {
-        gbifIndex[gbifId] = []
+function addTaxon (index: Record<string, TaxonId[]>, id: string, taxon: string[]) {
+    if (!(id in index)) {
+        index[id] = []
     }
-    gbifIndex[gbifId].push(taxon[0])
-    gbifIndex[gbifId].sort(numericSort)
+    index[id].push(taxon[0])
+    index[id].sort(numericSort)
 }
 
 async function main (args: string[]): Promise<void> {
@@ -35,6 +38,7 @@ async function main (args: string[]): Promise<void> {
     const files = await fs.readdir(path.join(REPO_ROOT, 'txt'))
 
     const gbifIndex: Record<string, TaxonId[]> = {}
+    const colIndex: Record<string, TaxonId[]> = {}
     const resourceIndex: Record<TaxonId, AmendedResourceMetadata> = {}
 
     await Promise.all(files.map(async function (fileName) {
@@ -57,6 +61,8 @@ async function main (args: string[]): Promise<void> {
             const [header, ...dwc] = csv.parseCsv(await fs.readFile(dwcFile, 'utf-8'))
             const gbifColumn = header.indexOf('gbifTaxonID')
             const gbifAcceptedColumn = header.indexOf('gbifAcceptedTaxonID')
+            const colColumn = header.indexOf('colTaxonID')
+            const colAcceptedColumn = header.indexOf('colAcceptedTaxonID')
             for (const taxon of dwc) {
                 const gbifId = taxon[gbifColumn]
                 if (gbifId) {
@@ -65,6 +71,15 @@ async function main (args: string[]): Promise<void> {
                         addTaxon(gbifIndex, taxon[gbifAcceptedColumn], taxon)
                     }
                 }
+
+                const colId = taxon[colColumn]
+                if (colId) {
+                    addTaxon(colIndex, colId, taxon)
+                    if (taxon[colAcceptedColumn] !== taxon[colColumn]) {
+                        addTaxon(colIndex, taxon[colAcceptedColumn], taxon)
+                    }
+                }
+
                 amendedResource.taxonCount += 1
             }
 
@@ -74,6 +89,7 @@ async function main (args: string[]): Promise<void> {
 
     await Promise.all([
         fs.writeFile(path.join(REPO_ROOT, 'gbif.index.json'), JSON.stringify(sortObject(gbifIndex), null, 2)),
+        fs.writeFile(path.join(REPO_ROOT, 'col.index.json'), JSON.stringify(sortObject(colIndex, alphabeticSort), null, 2)),
         fs.writeFile(path.join(REPO_ROOT, 'index.json'), JSON.stringify(sortObject(resourceIndex), null, 2))
     ])
 }

package/src/module.d.ts

@@ -95,7 +95,8 @@ interface ResourceHistory {
 type ResourceDiff = ResourceDiffPart[]
 
 interface ResourceDiffPart {
-    text: string,
+    text?: string,
+    original?: string,
     type: ResourceDiffType
 }
 

package/src/resources/diff-resource.ts

@@ -5,7 +5,12 @@ export enum ResourceDiffType {
     Unchanged = '='
 }
 
-function LCS (X: string[], Y: string[]): ResourceDiffPart[] {
+interface DiffPart {
+    text: string,
+    type: ResourceDiffType
+}
+
+function LCS (X: string[], Y: string[]): DiffPart[] {
     const m = X.length
     const n = Y.length
 
@@ -57,7 +62,7 @@ function gitTokenize (text: string): string[] {
     return text.match(/\S+|\n|[\r\t\f\v \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+/g) as string[]
 }
 
-export function createDiff (a: string, b: string, tokenize: ResourceDiffTokenizer = gitTokenize): ResourceDiffPart[] {
+export function createDiff (a: string, b: string, tokenize: ResourceDiffTokenizer = gitTokenize): ResourceDiff {
     const X = tokenize(a.trimEnd())
     const Y = tokenize(b.trimEnd())
 
@@ -94,7 +99,7 @@ export function createDiff (a: string, b: string, tokenize: ResourceDiffTokenize
 
     // Convert word diff to line diff
     const lines: ResourceDiff = []
-    let line = null
+    let line: ResourceDiffPart|null = null
     let deletedNewlines = 0
     let nextLineNew = false
 
@@ -102,7 +107,7 @@ export function createDiff (a: string, b: string, tokenize: ResourceDiffTokenize
         // Start of line
         if (line === null) {
             deletedNewlines = 0
-            line = { text: '', type: change.type }
+            line = { type: change.type }
         }
 
         // End of line (could be same token)
@@ -132,7 +137,8 @@ export function createDiff (a: string, b: string, tokenize: ResourceDiffTokenize
             // Add placeholders for deleted lines
             while (deletedNewlines--) {
                 lines.push({
-                    text: '',
+                    text: undefined,
+                    original: '',
                     type: ResourceDiffType.Deleted
                 })
             }
@@ -147,7 +153,11 @@ export function createDiff (a: string, b: string, tokenize: ResourceDiffTokenize
         }
 
         if (change.type !== ResourceDiffType.Deleted) {
-            line.text += change.text
+            line.text = (line.text ?? '') + change.text
+        }
+
+        if (change.type !== ResourceDiffType.Added) {
+            line.original = (line.original ?? '') + change.text
         }
     }
 

package/src/resources/parse-text.ts

@@ -66,14 +66,6 @@ const TAXONOMIC_STATUS: Record<string, TaxonStatus> = {
     '=': 'synonym'
 }
 
-const INDET_SUFFIXES = new Set([
-    'sp.',
-    'spec.',
-    'indet.',
-    'sp. indet.',
-    'spec. indet.'
-])
-
 const RANK_LABELS: Record<Rank, string> = {
     'subspecies': 'subsp.',
     'variety': 'var.',
@@ -93,6 +85,8 @@ const RANK_LABELS_REVERSE: Record<string, Rank> = {
     'subsp': 'subspecies'
 }
 
+const HYBRID_SIGN = '\u00D7'
+
 /**
  *   1. Any number of
  *      - capitalized words
@@ -133,11 +127,14 @@ const NAME_PATTERN = new RegExp(
 /**
  * Structure
  *   $1 genus+subgenus (+ trailing space): (?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?
- *     $1.1 genus: ([A-Z]\S+)
+ *     $1.1 genus: ((?:x )?[A-Z]\S+)
  *     $1.2 subgenus: (?:\(([A-Z]\S+?)\) )?
- *   $2 species: ((?:x )?[a-z][^\s.]+)
+ *   $2 species: (x [a-z]+|[a-z][^\s.]+(?: x [a-z]+)?|[A-Z][a-z]+_[a-z]+ x [A-Z][a-z]+_[a-z]+)
+ *     $2a: x [a-z]+
+ *     $2b hybrid: [a-z][^\s.]+(?: x [a-z]+)?
+ *     $2c intergeneric hybrid: [A-Z][a-z]+_[a-z]+ x [A-Z][a-z]+_[a-z]+
  */
-const BINAME_PATTERN = /^(?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?((?:x )?[a-z][^\s.]+)(?= |$)/
+const BINAME_PATTERN = /^(?:((?:x )?[A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?(x [a-z]+|[a-z][^\s.]+(?: x [a-z]+)?|[A-Z][a-z]+_[a-z]+ x [A-Z][a-z]+_[a-z]+)(?= |$)/
 
 function compareRanks (a: Rank, b: Rank): number {
     return RANKS.indexOf(a) - RANKS.indexOf(b)
@@ -147,12 +144,19 @@ function capitalize (name: string): string {
     return name[0].toUpperCase() + name.slice(1).toLowerCase()
 }
 
+function capitalizeGenericName (name: string): string {
+    if (name[0] === HYBRID_SIGN) {
+        return HYBRID_SIGN + capitalize(name.slice(1))
+    }
+
+    return capitalize(name)
+}
+
 function isUpperCase (name: string): boolean {
     return name === name.toUpperCase()
 }
 
 function getSynonymRank (name: string, rank: Rank): Rank {
-    const BINAME_PATTERN = /^([A-Z]\S+ (\([A-Z]\S+\) )?)?(x )?[a-z0-9-]+(?= |$)/
     const rest = name.replace(BINAME_PATTERN, '')
     const rankPrefix = rest.match(/^(?: |^)(st|r|ab|f|var|ssp|subsp)\. /)
     if (rankPrefix) {
@@ -194,7 +198,7 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
     }
 
     // Set verbatim identification after subsequent syntax is removed.
-    item.verbatimIdentification = name
+    item.verbatimIdentification = name.replace(/(?<=^| )x(?=$| )/g, HYBRID_SIGN).replace(/_/g, ' ')
 
     // Parent context is used for parsing and formatting binomial names.
     // For formatting, it needs to match external databases (i.e. be correct).
@@ -218,7 +222,8 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
     if (isSynonym || !parentContext.genus || (compareRanks('species', rank) < 0 && !parentContext.specificEpithet)) {
         const [, genus, subgenus, species] = name.match(BINAME_PATTERN) || []
         if (genus) {
-            parentContext.genus = parentContext.incorrect.genus = capitalize(genus)
+            parentContext.incorrect.genus = genus
+            parentContext.genus = capitalizeGenericName(genus.replace(/(^| )x /, HYBRID_SIGN))
         }
         if (subgenus) {
             parentContext.subgenus = capitalize(subgenus)
@@ -227,8 +232,9 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
             // from the parent context.
             delete parentContext.subgenus
         }
-        if (species) {
-            parentContext.specificEpithet = parentContext.incorrect.specificEpithet = species
+        if (species && compareRanks('species', rank) < 0) {
+            parentContext.incorrect.specificEpithet = species
+            parentContext.specificEpithet = species.replace(/(^| )x /, HYBRID_SIGN)
         }
     }
 
@@ -259,8 +265,12 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
     }
 
     // Hybrids
+    if (rank === 'genus' && name.startsWith('x ')) {
+        name = HYBRID_SIGN + name.slice(2)
+    }
+
     if (rank === 'species' && /(^| )x /.test(name)) {
-        name = name.replace(/(^| )x /, '\u00D7')
+        name = name.replace(/(^| )x /, HYBRID_SIGN)
     }
 
     // Divide the name into the main scientific name (only the epithet for taxa
@@ -271,7 +281,9 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
     }
 
     // To encode old names with spaces (e.g. "Orsillus pini canariensis Lindberg, 1953")
-    // underscores are used, which are replaced here.
+    // underscores are used, which are replaced here. This is also used for undescribed
+    // species (e.g. "Leiobunum species A") and intergeneric hybrids (e.g. "×Festulpia
+    // Festuca rubra × Vulpia bromoides")
     if (nameParts[1].includes('_')) {
         nameParts[1] = nameParts[1].replace(/_/g, ' ')
     }
@@ -281,14 +293,19 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
     item.taxonRemarks = notes
     item.taxonRank = rank
 
-    if (/[^\p{L}0-9\u{00D7}\- ]/u.test(taxon) && !INDET_SUFFIXES.has(taxon)) {
+    if (/[^\p{L}0-9\u{00D7}\- ]/u.test(taxon)) {
         throw new Error(`Taxon name contains unexpected characters: "${taxon}"`)
     }
 
     // Validate names and recompose binomial and trinomial names
-    if (compareRanks('group', rank) > 0) {
+    if (rank === 'genus') {
+        item.scientificName = capitalizeGenericName(taxon)
+        if (taxon[0].toUpperCase() !== taxon[0] || (taxon[0] === HYBRID_SIGN && taxon[1].toUpperCase() !== taxon[1])) {
+            throw new Error(`Generic epithet should be capitalized: "${taxon}"`)
+        }
+    } else if (compareRanks('group', rank) > 0) {
         item.scientificName = capitalize(taxon)
-        if (item.scientificName[0] !== taxon[0]) {
+        if (taxon[0].toUpperCase() !== taxon[0]) {
             throw new Error(`Taxon name (${rank}) should be capitalized: "${taxon}"`)
         }
     } else if (rank === 'group') {
@@ -312,12 +329,12 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
     } else if (rank === 'species') {
         item.genericName = parentContext.genus
         item.infragenericEpithet = parentContext.subgenus
-        item.specificEpithet = taxon.toLowerCase()
-        item.scientificName = `${item.genericName} ${item.specificEpithet}`
-        if (item.specificEpithet !== taxon) {
+        if (taxon.toLowerCase() !== taxon && !/^[A-Z][a-z]+ [a-z]+\xD7[A-Z][a-z]+ [a-z]+$/.test(taxon)) {
             console.log(item, taxon)
             throw new Error(`Specific epithet should be lowercase: "${taxon}"`)
         }
+        item.specificEpithet = taxon
+        item.scientificName = `${item.genericName} ${item.specificEpithet}`
     } else if (compareRanks('species', rank) < 0) {
         item.genericName = parentContext.genus
         item.infragenericEpithet = parentContext.subgenus
@@ -463,6 +480,20 @@ function parseResource (resource: string): [ResourceMetadata, string] {
     return [config, content]
 }
 
+function getIndentation (line: string): number {
+    return (line.match(/^ */) as string[])[0].length
+}
+
+function isIndetLine (line: string, indent?: number): boolean {
+    if (indent === undefined) {
+        indent = getIndentation(line)
+    }
+
+    line = line.slice(indent)
+
+    return line.startsWith('[indet]')
+}
+
 function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds: number[]): Resource {
     const idBase = `${resource.id}:`
 
@@ -473,23 +504,19 @@ function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds
     let previousId = ''
     let newIdOffset = Math.max(...oldIds)
 
-    for (const { text: line, type } of content) {
-        if (type === ResourceDiffType.Deleted) {
-            id++
-            continue
-        }
-
-        // Do not process "indet" lines further, as they only serve to indicate
-        // that subtaxa are explicitely omitted
-        if (Array.from(INDET_SUFFIXES).some(suffix => line.endsWith(' ' + suffix))) {
+    for (const line of content) {
+        if (line.type === ResourceDiffType.Deleted) {
+            // Increase id counter for removed line unless it was an "indet line"
+            if (!isIndetLine(line.original as string)) {
+                id++
+            }
             continue
         }
 
-        const lineIndent = (line.match(/^ */) as string[])[0].length
-
+        const lineIndent = getIndentation(line.text as string)
         if (lineIndent > groupIndent) {
             // Do not count synonyms as parents (unless this is correcting a typo in the synonym)
-            if (data[previousId] && data[previousId].taxonomicStatus === 'accepted' || /^( {2})+> /.test(line)) {
+            if (data[previousId] && data[previousId].taxonomicStatus === 'accepted' || /^( {2})+> /.test(line.text as string)) {
                 parents.push(previousId)
             } else {
                 parents.push(null)
@@ -508,10 +535,19 @@ function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds
             groupIndent = lineIndent
         }
 
+        // Do not process "indet" lines further, as they only serve to indicate
+        // that subtaxa are explicitely omitted
+        if (isIndetLine(line.text as string, lineIndent)) {
+            // If the line was previously not and ndet line, increase the id counter
+            if (line.type === ResourceDiffType.Modified && !isIndetLine(line.original as string)) {
+                id++
+            }
+            continue
+        }
+
         const parentId = parents.reduce((grandparent, parent) => parent || grandparent, null)
         const parent = parentId === null ? {} as WorkingTaxon : data[parentId]
-
-        const name = line.slice(groupIndent)
+        const name = (line.text as string).slice(groupIndent)
         const rank = resource.metadata.levels[groupIndent / 2]
         const item = parseName(name, rank, parent)
         const isSynonym = item.taxonomicStatus !== 'accepted'
@@ -556,7 +592,10 @@ function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds
         }
 
         // Set identifiers
-        if (type === ResourceDiffType.Added) {
+        if (line.type === ResourceDiffType.Added) {
+            newIdOffset++
+            item.scientificNameID = idBase + newIdOffset.toString()
+        } else if (line.type === ResourceDiffType.Modified && isIndetLine(line.original as string)) {
             newIdOffset++
             item.scientificNameID = idBase + newIdOffset.toString()
         } else {

package/test/resources.js

@@ -26,9 +26,9 @@ levels: [family, genus, species]
 
     Dolichurus haemorrhous A. Costa, 1886
   Dolichurus A. Costa, 1886
-    Dolichurus indet.
+    [indet]
 Sphecidae A. Costa, 1886
-    Sphecidae indet.
+    [indet]
 `, 'T1')
         assert.deepStrictEqual(Object.values(resource.taxa).map(taxon => taxon.scientificNameAuthorship), Array(3).fill('A. Costa, 1886'))
     })
@@ -86,7 +86,7 @@ levels: [genus, species]
 ---
 
 Drymus
-  Unknown sp.
+  [indet]
 `, 'T1')
         assert.strictEqual(resource.taxa['T1:1:1'].scientificName, 'Drymus')
     })
@@ -149,4 +149,30 @@ Bogdania Kerzhner, 1964
         assert.strictEqual(resource.taxa['T1:1:1'].scientificName, 'Bogdiana Kerzhner, 1964')
         assert.strictEqual(resource.taxa['T1:1:2'].scientificName, 'Bogdiana myrmica Kerzhner, 1964')
     })
+
+    await t.test('parses cross-genus hybrids', (t) => {
+        const [resource] = resources.parseTextFile(`---
+levels: [genus, species]
+---
+
+x Triticosecale
+  [indet]
+x Festulpia
+  Festuca_rubra x Vulpia_bromoides
+`, 'T1')
+        assert.strictEqual(resource.taxa['T1:1:1'].scientificName, '×Triticosecale')
+        assert.strictEqual(resource.taxa['T1:1:2'].scientificName, '×Festulpia')
+        assert.strictEqual(resource.taxa['T1:1:3'].scientificName, '×Festulpia Festuca rubra×Vulpia bromoides')
+    })
+
+    await t.test('parses cross-genus hybrids without parent context', (t) => {
+        const [resource] = resources.parseTextFile(`---
+levels: [species]
+---
+
+x Festulpia Festuca_rubra x Vulpia_bromoides
+`, 'T1')
+        assert.strictEqual(resource.taxa['T1:1:1'].scientificName, '×Festulpia Festuca rubra×Vulpia bromoides')
+        assert.strictEqual(resource.taxa['T1:1:1'].verbatimIdentification, '× Festulpia Festuca rubra × Vulpia bromoides')
+    })
 })