@larsgw/formica 0.6.6 → 0.6.7

package/CHANGELOG.md

@@ -1,3 +1,19 @@
+## [0.6.7](https://github.com/identification-resources/formica/compare/v0.6.6...v0.6.7) (2025-02-26)
+
+
+### Bug Fixes
+
+* **resources:** keep uncorrected name in verbatimIdentification ([dc4d5d1](https://github.com/identification-resources/formica/commit/dc4d5d17e7a412a94538f6162dfd839d808cbc30))
+* **resources:** update heuristics for recognizing author citations ([bca6a7f](https://github.com/identification-resources/formica/commit/bca6a7f5a6c2dc3dc421d885f6270f5464fc5ea5))
+* **resources:** use corrected generic name in output ([b472384](https://github.com/identification-resources/formica/commit/b4723842e87515f076230c603246082bc3bbec9d))
+
+
+### Features
+
+* **resources:** add support for completeness flags ([55b5427](https://github.com/identification-resources/formica/commit/55b5427125b4f1072191cadbc461ba31d37e6f2e))
+
+
+
 ## [0.6.6](https://github.com/identification-resources/formica/compare/v0.6.5...v0.6.6) (2025-02-16)
 
 

package/lib/bin/process-resources-problems.js

@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+"use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+var fs_1 = require("fs");
+var path = require("path");
+var util = require("util");
+var index_1 = require("../index");
+var GBIF_RANKS = [
+    'kingdom',
+    'phyllum',
+    'class',
+    'order',
+    'family',
+    'genus',
+    'species',
+    'subspecies',
+    'variety'
+];
+var DISPLAY_FIELDS = [
+    'scientificNameID',
+    'taxonRank',
+    'scientificName',
+    'taxonomicStatus',
+    'taxonRemarks',
+    'colTaxonID',
+    'gbifTaxonID'
+];
+var ResourceProblemsProcessor = /** @class */ (function () {
+    function ResourceProblemsProcessor(collectionPath) {
+        this.DIR_ROOT = path.resolve(collectionPath);
+        this.DIR_DWC = path.join(this.DIR_ROOT, 'dwc');
+        this.FILE_INDEX = path.join(this.DIR_ROOT, 'index.json');
+        this.FILE_PROBLEMS = path.join(this.DIR_ROOT, 'problems.csv');
+    }
+    ResourceProblemsProcessor.prototype.run = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var problems, _a, _b, resources, _c, _d, _i, problems_1, problem, _work, resourceId, _problemText, resource, missing;
+            return __generator(this, function (_e) {
+                switch (_e.label) {
+                    case 0:
+                        _b = (_a = index_1.csv).parseCsv;
+                        return [4 /*yield*/, fs_1.promises.readFile(this.FILE_PROBLEMS, 'utf8')];
+                    case 1:
+                        problems = _b.apply(_a, [_e.sent()]).slice(1);
+                        _d = (_c = JSON).parse;
+                        return [4 /*yield*/, fs_1.promises.readFile(this.FILE_INDEX, 'utf8')];
+                    case 2:
+                        resources = _d.apply(_c, [_e.sent()]);
+                        _i = 0, problems_1 = problems;
+                        _e.label = 3;
+                    case 3:
+                        if (!(_i < problems_1.length)) return [3 /*break*/, 6];
+                        problem = problems_1[_i];
+                        _work = problem[0], resourceId = problem[1], _problemText = problem[2];
+                        return [4 /*yield*/, this.readAmendedResource(resourceId, resources)];
+                    case 4:
+                        resource = _e.sent();
+                        missing = this.getMissingMappings(resource);
+                        if (missing.length === 0) {
+                            this.removeProblem(resourceId);
+                        }
+                        _e.label = 5;
+                    case 5:
+                        _i++;
+                        return [3 /*break*/, 3];
+                    case 6: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    ResourceProblemsProcessor.prototype.readAmendedResource = function (resourceId, resources) {
+        return __awaiter(this, void 0, void 0, function () {
+            var resource, resourceFileId, resourceFile, _a, header, taxonRows, _b, _c, taxa, _i, taxonRows_1, row, taxon;
+            return __generator(this, function (_d) {
+                switch (_d.label) {
+                    case 0:
+                        resource = resources[resourceId];
+                        resourceFileId = resourceId.split(':').join('-');
+                        resourceFile = path.join(this.DIR_DWC, "".concat(resourceFileId, ".csv"));
+                        _c = (_b = index_1.csv).parseCsv;
+                        return [4 /*yield*/, fs_1.promises.readFile(resourceFile, 'utf8')];
+                    case 1:
+                        _a = _c.apply(_b, [_d.sent()]), header = _a[0], taxonRows = _a.slice(1);
+                        taxa = {};
+                        for (_i = 0, taxonRows_1 = taxonRows; _i < taxonRows_1.length; _i++) {
+                            row = taxonRows_1[_i];
+                            taxon = row.reduce(function (taxon, value, index) {
+                                taxon[header[index]] = value;
+                                return taxon;
+                            }, {});
+                            taxa[taxon.scientificNameID] = taxon;
+                        }
+                        return [2 /*return*/, __assign(__assign({}, resource), { taxa: taxa })];
+                }
+            });
+        });
+    };
+    ResourceProblemsProcessor.prototype.getMissingMappings = function (resource) {
+        var missing = [];
+        for (var id in resource.taxa) {
+            var taxon = resource.taxa[id];
+            if (taxon.taxonomicStatus !== 'accepted') {
+                continue;
+            }
+            var missingCol = false; // !taxon.colTaxonID
+            var missingGbif = GBIF_RANKS.includes(taxon.taxonRank) && !taxon.gbifTaxonID;
+            if (missingCol || missingGbif) {
+                missing.push(taxon);
+            }
+        }
+        return missing;
+    };
+    ResourceProblemsProcessor.prototype.removeProblem = function (resourceId) {
+        resourceId;
+        //
+    };
+    return ResourceProblemsProcessor;
+}());
+function main() {
+    var args = util.parseArgs({ allowPositionals: true });
+    var processor = new ResourceProblemsProcessor(args.positionals[0]);
+    process.on('exit', function () {
+        process.stdout.write('\n');
+    });
+    processor.run().catch(function (error) {
+        console.error(error);
+        process.exit(1);
+    });
+}
+main();

package/lib/resources/parse-text.js

@@ -64,6 +64,11 @@ var DWC_RANKS = [
     'genus',
     'subgenus'
 ];
+var FLAGS = [
+    'MISSING_TAXA',
+    'MISSING_PARENT_TAXA',
+    'MISSING_SYNONYMS'
+];
 var TAXONOMIC_STATUS = {
     '>': 'incorrect',
     '+': 'heterotypic synonym',
@@ -103,7 +108,7 @@ var RANK_LABELS_REVERSE = {
  *   2. Followed by a capitalized word
  *   3. Optionally, followed by "et al."
  */
-var LOWERCASE_NAME_PARTICLES = ['y', 'der', 'den', 'de', 'van'].join('|');
+var LOWERCASE_NAME_PARTICLES = ['y', 'der', 'den', 'de', 'van', 'von'].join('|');
 var SIMPLE_AUTHOR_PATTERN = '(?:(?:\\p{Lu}\\S*|&|in|ex|' + LOWERCASE_NAME_PARTICLES + ')\\s*)*\\p{Lu}\\S+(?:\\s+et\\s+al\\.)?';
 var NAME_PATTERN = new RegExp('^' +
     // $1 main name part
@@ -111,7 +116,7 @@ var NAME_PATTERN = new RegExp('^' +
     // $2 optional author citation
     '(?: ' +
     // but not auct(t)., etc.
-    '(?!auctt?\\.|(?:syn|comb|sp|spec)\\. n(?:ov)?\\.|s(?:ens[.u]|\\.))' +
+    '(?!auctt?\\.|(?:syn|comb|sp|spec|nom)\\. n(?:ov)?\\.|s(?:ens[.u]|\\.)|in part|partim)' +
     '(' +
     // $2.1 anything in parentheses, followed by optional revising author(s)
     '\\(.+?\\)(?:\\s+' + SIMPLE_AUTHOR_PATTERN + ')?' +
@@ -326,17 +331,6 @@ function parseName(name, rank, parent) {
     if (item.scientificNameAuthorship) {
         item.scientificName += " ".concat(item.scientificNameAuthorship);
     }
-    // Amend "parent" with corrections
-    if (item.taxonomicStatus === 'incorrect') {
-        var itemAsObject = item;
-        var parentAsObject = parent;
-        parent.incorrect = __assign({}, parent);
-        for (var key in item) {
-            if (key !== 'taxonomicStatus') {
-                parentAsObject[key] = itemAsObject[key];
-            }
-        }
-    }
     return item;
 }
 function parseHeader(header) {
@@ -401,6 +395,16 @@ function parseHeader(header) {
             metadata.catalog[key] = work.get(key);
         }
     }
+    if ('flags' in config) {
+        if (!Array.isArray(config.flags)) {
+            throw new SyntaxError('"flags" should be an array if present');
+        }
+        var invalidFlags = config.flags.filter(function (flag) { return !FLAGS.includes(flag); });
+        if (invalidFlags.length) {
+            throw new SyntaxError("\"flags\" contains invalid values: ".concat(invalidFlags.join(', ')));
+        }
+        metadata.flags = config.flags;
+    }
     return metadata;
 }
 function validateResource(config, content) {
@@ -480,27 +484,11 @@ function parseResourceContent(content, resource, oldIds) {
         var rank = resource.metadata.levels[groupIndent / 2];
         var item = parseName(name_1, rank, parent_1);
         var isSynonym = item.taxonomicStatus !== 'accepted';
-        if (item.taxonomicStatus === 'incorrect') {
-            return "continue";
-        }
-        if (type === diff_resource_1.ResourceDiffType.Added) {
-            newIdOffset++;
-            item.scientificNameID = idBase + newIdOffset.toString();
-        }
-        else {
-            id++;
-            item.scientificNameID = idBase + (oldIds[id - 1] || id).toString();
-        }
-        previousId = item.scientificNameID;
-        item.parentNameUsageID = isSynonym ? undefined : parent_1.scientificNameID;
-        item.parentNameUsage = isSynonym ? undefined : parent_1.scientificName;
-        item.acceptedNameUsageID = isSynonym ? parent_1.scientificNameID : undefined;
-        item.acceptedNameUsage = isSynonym ? parent_1.scientificName : undefined;
-        item.collectionCode = idBase.slice(0, -1);
+        // Add higher classification info
+        var itemAsObject = item;
+        var parentAsObject = parent_1;
         for (var _b = 0, DWC_RANKS_1 = DWC_RANKS; _b < DWC_RANKS_1.length; _b++) {
             var rank_1 = DWC_RANKS_1[_b];
-            var itemAsObject = item;
-            var parentAsObject = parent_1;
             itemAsObject[rank_1] = undefined;
             if (parentAsObject[rank_1]) {
                 itemAsObject[rank_1] = parentAsObject[rank_1];
@@ -524,6 +512,31 @@ function parseResourceContent(content, resource, oldIds) {
         else if (parentId) {
             item.higherClassification = parent_1.scientificNameOnly;
         }
+        // Amend "parent" with corrections
+        if (item.taxonomicStatus === 'incorrect') {
+            parent_1.incorrect = __assign({}, parent_1);
+            for (var key in item) {
+                if (key !== 'taxonomicStatus' && key !== 'verbatimIdentification') {
+                    parentAsObject[key] = itemAsObject[key];
+                }
+            }
+            return "continue";
+        }
+        // Set identifiers
+        if (type === diff_resource_1.ResourceDiffType.Added) {
+            newIdOffset++;
+            item.scientificNameID = idBase + newIdOffset.toString();
+        }
+        else {
+            id++;
+            item.scientificNameID = idBase + (oldIds[id - 1] || id).toString();
+        }
+        previousId = item.scientificNameID;
+        item.parentNameUsageID = isSynonym ? undefined : parent_1.scientificNameID;
+        item.parentNameUsage = isSynonym ? undefined : parent_1.scientificName;
+        item.acceptedNameUsageID = isSynonym ? parent_1.scientificNameID : undefined;
+        item.acceptedNameUsage = isSynonym ? parent_1.scientificName : undefined;
+        item.collectionCode = idBase.slice(0, -1);
         data[item.scientificNameID] = item;
     };
     for (var _i = 0, content_1 = content; _i < content_1.length; _i++) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larsgw/formica",
-  "version": "0.6.6",
+  "version": "0.6.7",
   "description": "SDK and tools for data from the Library of Identification Resources",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",

package/src/bin/process-resources.ts

@@ -175,7 +175,6 @@ class ResourceProcessor {
             const skip = await this.shouldBeSkipped(resource.id)
 
             if (!skip) {
-                // TODO const correct = checkResults(results, classifications)
                 const correct = this.checkResults(results)
                 if (!correct) {
                     const choice = await promptForAnswers(

package/src/module.d.ts

@@ -71,8 +71,11 @@ interface Taxon extends WorkingTaxon {
     verbatimIdentification: string
 }
 
+type ResourceFlag = string;
+
 interface ResourceMetadata {
     levels: Rank[],
+    flags?: ResourceFlag[],
     catalog?: Record<string, Value>
 }
 

package/src/resources/parse-text.ts

@@ -54,6 +54,12 @@ const DWC_RANKS: DwcRank[] = [
     'subgenus'
 ]
 
+const FLAGS: ResourceFlag[] = [
+    'MISSING_TAXA',
+    'MISSING_PARENT_TAXA',
+    'MISSING_SYNONYMS'
+]
+
 const TAXONOMIC_STATUS: Record<string, TaxonStatus> = {
     '>': 'incorrect',
     '+': 'heterotypic synonym',
@@ -97,7 +103,7 @@ const RANK_LABELS_REVERSE: Record<string, Rank> = {
  *   2. Followed by a capitalized word
  *   3. Optionally, followed by "et al."
  */
-const LOWERCASE_NAME_PARTICLES = ['y', 'der', 'den', 'de', 'van'].join('|')
+const LOWERCASE_NAME_PARTICLES = ['y', 'der', 'den', 'de', 'van', 'von'].join('|')
 const SIMPLE_AUTHOR_PATTERN = '(?:(?:\\p{Lu}\\S*|&|in|ex|' + LOWERCASE_NAME_PARTICLES + ')\\s*)*\\p{Lu}\\S+(?:\\s+et\\s+al\\.)?'
 
 const NAME_PATTERN = new RegExp(
@@ -107,7 +113,7 @@ const NAME_PATTERN = new RegExp(
         // $2 optional author citation
         '(?: ' +
             // but not auct(t)., etc.
-            '(?!auctt?\\.|(?:syn|comb|sp|spec)\\. n(?:ov)?\\.|s(?:ens[.u]|\\.))' +
+            '(?!auctt?\\.|(?:syn|comb|sp|spec|nom)\\. n(?:ov)?\\.|s(?:ens[.u]|\\.)|in part|partim)' +
         '(' +
             // $2.1 anything in parentheses, followed by optional revising author(s)
             '\\(.+?\\)(?:\\s+' + SIMPLE_AUTHOR_PATTERN + ')?' +
@@ -342,19 +348,6 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
         item.scientificName += ` ${item.scientificNameAuthorship}`
     }
 
-    // Amend "parent" with corrections
-    if (item.taxonomicStatus === 'incorrect') {
-        const itemAsObject = item as { [index: string]: unknown }
-        const parentAsObject = parent as { [index: string]: unknown }
-
-        parent.incorrect = { ...parent }
-        for (const key in item) {
-            if (key !== 'taxonomicStatus') {
-                parentAsObject[key] = itemAsObject[key]
-            }
-        }
-    }
-
     return item
 }
 
@@ -418,6 +411,19 @@ function parseHeader (header: string): ResourceMetadata {
         }
     }
 
+    if ('flags' in config) {
+        if (!Array.isArray(config.flags)) {
+            throw new SyntaxError('"flags" should be an array if present')
+        }
+
+        const invalidFlags = config.flags.filter(flag => !FLAGS.includes(flag))
+        if (invalidFlags.length) {
+            throw new SyntaxError(`"flags" contains invalid values: ${invalidFlags.join(', ')}`)
+        }
+
+        metadata.flags = config.flags
+    }
+
     return metadata
 }
 
@@ -510,29 +516,10 @@ function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds
         const item = parseName(name, rank, parent)
         const isSynonym = item.taxonomicStatus !== 'accepted'
 
-        if (item.taxonomicStatus === 'incorrect') {
-            continue
-        }
-
-        if (type === ResourceDiffType.Added) {
-            newIdOffset++
-            item.scientificNameID = idBase + newIdOffset.toString()
-        } else {
-            id++
-            item.scientificNameID = idBase + (oldIds[id - 1] || id).toString()
-        }
-        previousId = item.scientificNameID
-
-        item.parentNameUsageID = isSynonym ? undefined : parent.scientificNameID
-        item.parentNameUsage = isSynonym ? undefined : parent.scientificName
-        item.acceptedNameUsageID = isSynonym ? parent.scientificNameID : undefined
-        item.acceptedNameUsage = isSynonym ? parent.scientificName : undefined
-        item.collectionCode = idBase.slice(0, -1)
-
+        // Add higher classification info
+        const itemAsObject = item as { [index: string]: unknown }
+        const parentAsObject = parent as { [index: string]: unknown }
         for (const rank of DWC_RANKS) {
-            const itemAsObject = item as { [index: string]: unknown }
-            const parentAsObject = parent as { [index: string]: unknown }
-
             itemAsObject[rank] = undefined
             if (parentAsObject[rank]) {
                 itemAsObject[rank] = parentAsObject[rank]
@@ -557,6 +544,33 @@ function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds
             item.higherClassification = parent.scientificNameOnly
         }
 
+        // Amend "parent" with corrections
+        if (item.taxonomicStatus === 'incorrect') {
+            parent.incorrect = { ...parent }
+            for (const key in item) {
+                if (key !== 'taxonomicStatus' && key !== 'verbatimIdentification') {
+                    parentAsObject[key] = itemAsObject[key]
+                }
+            }
+            continue
+        }
+
+        // Set identifiers
+        if (type === ResourceDiffType.Added) {
+            newIdOffset++
+            item.scientificNameID = idBase + newIdOffset.toString()
+        } else {
+            id++
+            item.scientificNameID = idBase + (oldIds[id - 1] || id).toString()
+        }
+        previousId = item.scientificNameID
+
+        item.parentNameUsageID = isSynonym ? undefined : parent.scientificNameID
+        item.parentNameUsage = isSynonym ? undefined : parent.scientificName
+        item.acceptedNameUsageID = isSynonym ? parent.scientificNameID : undefined
+        item.acceptedNameUsage = isSynonym ? parent.scientificName : undefined
+        item.collectionCode = idBase.slice(0, -1)
+
         data[item.scientificNameID] = item
     }
 

package/test/resources.js

@@ -136,4 +136,17 @@ Tilia x vulgaris
         assert.strictEqual(resource.taxa['T1:1:1'].scientificName, 'Rumex conglomeratus×maritimus')
         assert.strictEqual(resource.taxa['T1:1:2'].scientificName, 'Tilia ×vulgaris')
     })
+
+    await t.test('outputs corrected generic names', (t) => {
+        const [resource] = resources.parseTextFile(`---
+levels: [genus, species]
+---
+
+Bogdania Kerzhner, 1964
+  > Bogdiana Kerzhner, 1964
+  myrmica Kerzhner, 1964
+`, 'T1')
+        assert.strictEqual(resource.taxa['T1:1:1'].scientificName, 'Bogdiana Kerzhner, 1964')
+        assert.strictEqual(resource.taxa['T1:1:2'].scientificName, 'Bogdiana myrmica Kerzhner, 1964')
+    })
 })