@larsgw/formica 0.6.3 → 0.6.5

package/CHANGELOG.md

@@ -1,3 +1,23 @@
+## [0.6.5](https://github.com/identification-resources/formica/compare/v0.6.4...v0.6.5) (2024-12-20)
+
+
+### Bug Fixes
+
+* **resources:** ignore empty lines in diff mode ([3fb027e](https://github.com/identification-resources/formica/commit/3fb027e8968196910fff2e574e700ac0abee996c))
+* **resources:** improve (ICBN) author parsing ([d3e3faf](https://github.com/identification-resources/formica/commit/d3e3faf411d1a7dccd528a5ee78bd0823a5883b4)), closes [#13](https://github.com/identification-resources/formica/issues/13) [#14](https://github.com/identification-resources/formica/issues/14)
+* **resources:** support "et al." in author name ([82993bf](https://github.com/identification-resources/formica/commit/82993bffac5e7f3ed00d00c28c836f347b264a43))
+
+
+
+## [0.6.4](https://github.com/identification-resources/formica/compare/v0.6.3...v0.6.4) (2024-12-16)
+
+
+### Bug Fixes
+
+* **resources:** fix matching of some ICBN names ([3c18458](https://github.com/identification-resources/formica/commit/3c184586b64e2a31f1eb7298bcd7c80c1d119069))
+
+
+
 ## [0.6.3](https://github.com/identification-resources/formica/compare/v0.6.2...v0.6.3) (2024-04-12)
 
 

package/lib/bin/process-resources.js

@@ -398,9 +398,9 @@ var ResourceProcessor = /** @class */ (function () {
     };
     ResourceProcessor.prototype.processResourceDwc = function (resource, config) {
         return __awaiter(this, void 0, void 0, function () {
-            var file, _a, header_1, rows, _b, _c, _i, rows_1, row, oldTaxon, taxon, filteredResults, taxonNames, names, id, name_1, result, _d, _e, results, _f, name_2, matches, _g, matches_1, match, source, currentRank, _h, _j, loirId, taxon, _k, amendResource, groupNameMatches, groupedNameMatches, amendedResource, _l, _m, _o, _p, source, matches;
-            return __generator(this, function (_q) {
-                switch (_q.label) {
+            var file, _a, header_1, rows, _b, _c, _i, rows_1, row, oldTaxon, taxon, filteredResults, taxonNames, names, id, name_1, result, _loop_2, _d, _e, results, _f, amendResource, groupNameMatches, groupedNameMatches, amendedResource, _g, _h, _j, _k, source, matches;
+            return __generator(this, function (_l) {
+                switch (_l.label) {
                     case 0:
                         console.log("".concat(resource.workId, ": matching ").concat(resource.id));
                         if (!!config.updateMappings) return [3 /*break*/, 3];
@@ -409,7 +409,7 @@ var ResourceProcessor = /** @class */ (function () {
                         _c = (_b = index_1.csv).parseCsv;
                         return [4 /*yield*/, fs_1.promises.readFile(file, 'utf-8')];
                     case 1:
-                        _a = _c.apply(_b, [_q.sent()]), header_1 = _a[0], rows = _a.slice(1);
+                        _a = _c.apply(_b, [_l.sent()]), header_1 = _a[0], rows = _a.slice(1);
                         for (_i = 0, rows_1 = rows; _i < rows_1.length; _i++) {
                             row = rows_1[_i];
                             oldTaxon = row.reduce(function (taxon, value, index) {
@@ -424,7 +424,7 @@ var ResourceProcessor = /** @class */ (function () {
                                 taxon.gbifAcceptedTaxonID = oldTaxon.gbifAcceptedTaxonID;
                             }
                         }
-                        _q.label = 2;
+                        _l.label = 2;
                     case 2: return [2 /*return*/, resource];
                     case 3:
                         filteredResults = {};
@@ -441,17 +441,23 @@ var ResourceProcessor = /** @class */ (function () {
                         }
                         return [4 /*yield*/, runGnverifier(Array.from(names).join('\n'))];
                     case 4:
-                        result = _q.sent();
-                        for (_d = 0, _e = result.trim().split('\n'); _d < _e.length; _d++) {
-                            results = _e[_d];
-                            _f = JSON.parse(results), name_2 = _f.name, matches = _f.results;
+                        result = _l.sent();
+                        _loop_2 = function (results) {
+                            var _m = JSON.parse(results), name_2 = _m.name, matches = _m.results;
                             if (!matches) {
-                                continue;
+                                return "continue";
                             }
-                            for (_g = 0, matches_1 = matches; _g < matches_1.length; _g++) {
-                                match = matches_1[_g];
-                                source = match.dataSourceId;
-                                currentRank = match.classificationRanks.split('|').pop();
+                            // Fix author scoring for some species, see https://github.com/gnames/gnverifier/issues/129
+                            matches.sort(function (a, b) {
+                                if (a.sortScore !== b.sortScore) {
+                                    return b.sortScore - a.sortScore;
+                                }
+                                return name_2 === a.matchedName ? -1 : name_2 === b.matchedName ? 1 : 0;
+                            });
+                            for (var _o = 0, _p = matches; _o < _p.length; _o++) {
+                                var match = _p[_o];
+                                var source = match.dataSourceId;
+                                var currentRank = match.classificationRanks.split('|').pop();
                                 if (match.scoreDetails.cardinalityScore === 0) {
                                     // Rank mismatch
                                     continue;
@@ -460,9 +466,9 @@ var ResourceProcessor = /** @class */ (function () {
                                     // GBIF species like "Nomada spec"
                                     continue;
                                 }
-                                for (_h = 0, _j = taxonNames[name_2]; _h < _j.length; _h++) {
-                                    loirId = _j[_h];
-                                    taxon = resource.taxa[loirId];
+                                for (var _q = 0, _r = taxonNames[name_2]; _q < _r.length; _q++) {
+                                    var loirId = _r[_q];
+                                    var taxon = resource.taxa[loirId];
                                     if (source === 11 && !GBIF_RANKS.includes(taxon.taxonRank)) {
                                         // Exclude GBIF matches for ranks that are not in GBIF
                                         continue;
@@ -483,30 +489,34 @@ var ResourceProcessor = /** @class */ (function () {
                                     });
                                 }
                             }
+                        };
+                        for (_d = 0, _e = result.trim().split('\n'); _d < _e.length; _d++) {
+                            results = _e[_d];
+                            _loop_2(results);
                         }
                         return [4 /*yield*/, Promise.resolve().then(function () { return require('../index'); })];
                     case 5:
-                        _k = (_q.sent()).taxonNames, amendResource = _k.amendResource, groupNameMatches = _k.groupNameMatches;
+                        _f = (_l.sent()).taxonNames, amendResource = _f.amendResource, groupNameMatches = _f.groupNameMatches;
                         groupedNameMatches = groupNameMatches(filteredResults);
                         amendedResource = __assign(__assign({}, resource), { taxa: __assign({}, resource.taxa) });
-                        _l = groupedNameMatches;
-                        _m = [];
-                        for (_o in _l)
-                            _m.push(_o);
-                        _p = 0;
-                        _q.label = 6;
+                        _g = groupedNameMatches;
+                        _h = [];
+                        for (_j in _g)
+                            _h.push(_j);
+                        _k = 0;
+                        _l.label = 6;
                     case 6:
-                        if (!(_p < _m.length)) return [3 /*break*/, 9];
-                        _o = _m[_p];
-                        if (!(_o in _l)) return [3 /*break*/, 8];
-                        source = _o;
+                        if (!(_k < _h.length)) return [3 /*break*/, 9];
+                        _j = _h[_k];
+                        if (!(_j in _g)) return [3 /*break*/, 8];
+                        source = _j;
                         return [4 /*yield*/, this.selectPrefixes(resource, groupedNameMatches, source)];
                     case 7:
-                        matches = _q.sent();
+                        matches = _l.sent();
                         amendResource(amendedResource, source, matches);
-                        _q.label = 8;
+                        _l.label = 8;
                     case 8:
-                        _p++;
+                        _k++;
                         return [3 /*break*/, 6];
                     case 9: return [2 /*return*/, amendedResource];
                 }

package/lib/resources/parse-text.js

@@ -93,6 +93,18 @@ var RANK_LABELS_REVERSE = {
     'ssp': 'subspecies',
     'subsp': 'subspecies'
 };
+/**
+ *   1. Any number of
+ *      - capitalized words
+ *      - "&"
+ *      - " in "
+ *      - " ex "
+ *      - lowercase name particles
+ *   2. Followed by a capitalized word
+ *   3. Optionally, followed by "et al."
+ */
+var LOWERCASE_NAME_PARTICLES = ['y', 'der', 'den', 'de', 'van'].join('|');
+var SIMPLE_AUTHOR_PATTERN = '(?:(?:\\p{Lu}\\S*|&|in|ex|' + LOWERCASE_NAME_PARTICLES + ')\\s*)*\\p{Lu}\\S+(?:\\s+et\\s+al\\.)?';
 var NAME_PATTERN = new RegExp('^' +
     // $1 main name part
     '(\\S+)' +
@@ -101,20 +113,14 @@ var NAME_PATTERN = new RegExp('^' +
     // but not auct(t)., etc.
     '(?!auctt?\\.|(?:syn|comb|sp|spec)\\. n(?:ov)?\\.|s(?:ens[.u]|\\.))' +
     '(' +
-    // $2.1 anything in parentheses
-    '\\(.+?\\)' +
+    // $2.1 anything in parentheses, followed by optional revising author(s)
+    '\\(.+?\\)(?:\\s+' + SIMPLE_AUTHOR_PATTERN + ')?' +
     '|' +
     // $2.2 anything followed by a year
     '.+?\\d{4}\\)?' +
     '|' +
-    // $2.3 name(, name)* & name
-    '.+(?:, .+)* & \\S+' +
-    '|' +
-    // $2.4 name y name
-    '\\S+ [yY] \\S+' +
-    '|' +
-    // $2.5 name( in name)
-    '\\p{Lu}\\S*(?: in \\S+)?' +
+    // $2.3 author(s)
+    SIMPLE_AUTHOR_PATTERN +
     '))?' +
     // $3 optional notes
     '(?:,? (.+))?' +
@@ -541,6 +547,8 @@ function parseFile(file, id, old) {
         var diff;
         if (oldResources[index]) {
             diff = (0, diff_resource_1.createDiff)(content, parseResource(oldResources[index])[1]);
+            // Ignore empty lines
+            diff = diff.filter(function (line) { return line.text !== ''; });
         }
         else {
             diff = (0, diff_resource_1.createDiff)(content, content);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larsgw/formica",
-  "version": "0.6.3",
+  "version": "0.6.5",
   "description": "SDK and tools for data from the Library of Identification Resources",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",

package/src/bin/process-resources.ts

@@ -287,13 +287,38 @@ class ResourceProcessor {
 
         const result = await runGnverifier(Array.from(names).join('\n'))
         for (const results of result.trim().split('\n')) {
+            interface MatchScoreDetails {
+                cardinalityScore: number;
+            }
+
+            interface Match {
+                currentRecordId: string;
+                dataSourceId: number;
+                matchedName: string;
+                recordId: string;
+                sortScore: number;
+                isSynonym: boolean;
+                classificationPath: string;
+                classificationRanks: string;
+                scoreDetails: MatchScoreDetails;
+            }
+
             const { name, results: matches } = JSON.parse(results)
 
             if (!matches) {
                 continue
             }
 
-            for (const match of matches) {
+            // Fix author scoring for some species, see https://github.com/gnames/gnverifier/issues/129
+            matches.sort((a: Match, b: Match) => {
+                if (a.sortScore !== b.sortScore) {
+                    return b.sortScore - a.sortScore
+                }
+
+                return name === a.matchedName ? -1 : name === b.matchedName ? 1 : 0
+            })
+
+            for (const match of matches as Match[]) {
                 const source = match.dataSourceId
                 const currentRank = match.classificationRanks.split('|').pop()
 

package/src/resources/parse-text.ts

@@ -87,6 +87,19 @@ const RANK_LABELS_REVERSE: Record<string, Rank> = {
     'subsp': 'subspecies'
 }
 
+/**
+ *   1. Any number of
+ *      - capitalized words
+ *      - "&"
+ *      - " in "
+ *      - " ex "
+ *      - lowercase name particles
+ *   2. Followed by a capitalized word
+ *   3. Optionally, followed by "et al."
+ */
+const LOWERCASE_NAME_PARTICLES = ['y', 'der', 'den', 'de', 'van'].join('|')
+const SIMPLE_AUTHOR_PATTERN = '(?:(?:\\p{Lu}\\S*|&|in|ex|' + LOWERCASE_NAME_PARTICLES + ')\\s*)*\\p{Lu}\\S+(?:\\s+et\\s+al\\.)?'
+
 const NAME_PATTERN = new RegExp(
     '^' +
         // $1 main name part
@@ -96,20 +109,14 @@ const NAME_PATTERN = new RegExp(
             // but not auct(t)., etc.
             '(?!auctt?\\.|(?:syn|comb|sp|spec)\\. n(?:ov)?\\.|s(?:ens[.u]|\\.))' +
         '(' +
-            // $2.1 anything in parentheses
-            '\\(.+?\\)' +
+            // $2.1 anything in parentheses, followed by optional revising author(s)
+            '\\(.+?\\)(?:\\s+' + SIMPLE_AUTHOR_PATTERN + ')?' +
         '|' +
             // $2.2 anything followed by a year
             '.+?\\d{4}\\)?' +
         '|' +
-            // $2.3 name(, name)* & name
-            '.+(?:, .+)* & \\S+' +
-        '|' +
-            // $2.4 name y name
-            '\\S+ [yY] \\S+' +
-        '|' +
-            // $2.5 name( in name)
-            '\\p{Lu}\\S*(?: in \\S+)?' +
+            // $2.3 author(s)
+            SIMPLE_AUTHOR_PATTERN +
         '))?' +
         // $3 optional notes
         '(?:,? (.+))?' +
@@ -573,6 +580,8 @@ export function parseFile (file: string, id: WorkId, old?: ResourceHistory): Res
         let diff
         if (oldResources[index]) {
             diff = createDiff(content, parseResource(oldResources[index])[1])
+            // Ignore empty lines
+            diff = diff.filter(line => line.text !== '')
         } else {
             diff = createDiff(content, content)
         }