@larsgw/formica 0.4.3 → 0.5.1

package/.gitattributes

@@ -0,0 +1 @@
+*.ts linguist-language=TypeScript

package/.github/workflows/ci.yml

@@ -0,0 +1,27 @@
+name: Node.js CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [18.x, 20.x]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+    - run: npm install
+    - run: npm run build
+    - run: npm run lint
+    - run: npm test

package/CHANGELOG.md

@@ -1,3 +1,44 @@
+## [0.5.1](https://github.com/identification-resources/formica/compare/v0.5.0...v0.5.1) (2023-11-03)
+
+
+### Bug Fixes
+
+* **resources:** fix file listing in processor ([009b980](https://github.com/identification-resources/formica/commit/009b98091b0f597e91213ace82252facbe3b5fed))
+
+
+
+# [0.5.0](https://github.com/identification-resources/formica/compare/v0.4.3...v0.5.0) (2023-10-24)
+
+
+### Bug Fixes
+
+* **catalog:** make validation more robust ([522db8a](https://github.com/identification-resources/formica/commit/522db8ad428c2d4645569a7f541fd203f5d628e8))
+* **csv:** fix csv formatting ([064e5e5](https://github.com/identification-resources/formica/commit/064e5e57b2a642d22850501ab2a0b95879779b85))
+* **resources:** do not count abbreviations as species ([ac0cc5a](https://github.com/identification-resources/formica/commit/ac0cc5a7cfd1f68df9b8d1d1c543a2f593839daa))
+* **resources:** fix typos in output column names ([0d02823](https://github.com/identification-resources/formica/commit/0d0282378b21e92fbe028a53a9d121f42e791cc5))
+
+
+### Features
+
+* **resources:** encode names with spaces with underscores ([9c7efd4](https://github.com/identification-resources/formica/commit/9c7efd4daf80660be98f2fec8ef9faa5d5b42d67))
+* **resources:** include verbatimIdentification column ([4a2e920](https://github.com/identification-resources/formica/commit/4a2e9201467347e900f5690d2301a3b231ded822))
+* **resources:** move "scope" metadata to "catalog" ([286c5b2](https://github.com/identification-resources/formica/commit/286c5b223ce41336f3ebd63c1797403aad21aef6))
+* **resources:** update processor configuration ([64daa26](https://github.com/identification-resources/formica/commit/64daa26d70c96df24298c7b78f2b93ef0f5b2a48))
+
+
+### BREAKING CHANGES
+
+* **resources:** - "intragenericEpithet" is now "infragenericEpithet"
+- "intraspecificEpithet" is now "infraspecificEpithet"
+* **resources:** Adds additional column, between "higherClassification" and "colTaxonID".
+* **resources:** - Instead of "-u, --update" use "-s modified"
+- Instead of "--update-mapping" use "-s all"
+* **resources:** Index will no longer contain "scope" array. "scope" field in YAML header
+does no longer pass validation. Use the "taxon_scope" and "scope" fields
+in "catalog" instead.
+
+
+
 ## [0.4.3](https://github.com/identification-resources/formica/compare/v0.4.2...v0.4.3) (2023-09-30)
 
 

package/lib/bin/process-resources.d.ts

@@ -1,2 +1,6 @@
 #!/usr/bin/env node
-export {};
+export declare enum ResourceProcessorSource {
+    All = "all",
+    Unprocessed = "unprocessed",
+    Modified = "modified"
+}

package/lib/bin/process-resources.js

@@ -48,20 +48,27 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
     }
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.ResourceProcessorSource = void 0;
 var fs_1 = require("fs");
 var path = require("path");
 var child_process_1 = require("child_process");
 var util = require("util");
 var index_1 = require("../index");
 var util_1 = require("./util");
+var ResourceProcessorSource;
+(function (ResourceProcessorSource) {
+    ResourceProcessorSource["All"] = "all";
+    ResourceProcessorSource["Unprocessed"] = "unprocessed";
+    ResourceProcessorSource["Modified"] = "modified";
+})(ResourceProcessorSource = exports.ResourceProcessorSource || (exports.ResourceProcessorSource = {}));
 var DWC_FIELDS = [
     'scientificNameID',
     'scientificName',
     'scientificNameAuthorship',
     'genericName',
-    'intragenericEpithet',
+    'infragenericEpithet',
     'specificEpithet',
-    'intraspecificEpithet',
+    'infraspecificEpithet',
     'taxonRank',
     'taxonRemarks',
     'collectionCode',
@@ -79,6 +86,7 @@ var DWC_FIELDS = [
     'genus',
     'subgenus',
     'higherClassification',
+    'verbatimIdentification',
     'colTaxonID',
     'gbifTaxonID',
     'colAcceptedTaxonID',
@@ -122,6 +130,36 @@ function runGnverifier(names) {
         proc.stdin.end();
     });
 }
+function listFiles(directory) {
+    return __awaiter(this, void 0, void 0, function () {
+        var input;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0: return [4 /*yield*/, fs_1.promises.readdir(directory)];
+                case 1:
+                    input = _a.sent();
+                    return [2 /*return*/, input.map(function (file) { return path.basename(file, '.txt'); }).sort(util_1.numericSort)];
+            }
+        });
+    });
+}
+function listUnprocessedFiles(directory, outputDirectory) {
+    return __awaiter(this, void 0, void 0, function () {
+        var input, output, _a;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
+                case 0: return [4 /*yield*/, listFiles(directory)];
+                case 1:
+                    input = _b.sent();
+                    _a = Set.bind;
+                    return [4 /*yield*/, fs_1.promises.readdir(outputDirectory)];
+                case 2:
+                    output = new (_a.apply(Set, [void 0, _b.sent()]))();
+                    return [2 /*return*/, input.filter(function (file) { return !output.has(file + '-1.csv'); })];
+            }
+        });
+    });
+}
 function listChangedFiles(directory) {
     return __awaiter(this, void 0, void 0, function () {
         var output;
@@ -132,7 +170,7 @@ function listChangedFiles(directory) {
                     })];
                 case 1:
                     output = _a.sent();
-                    return [2 /*return*/, output.trimEnd().split('\n').sort(util_1.numericSort)];
+                    return [2 /*return*/, output.trimEnd().split('\n').map(function (file) { return path.basename(file, '.txt'); }).sort(util_1.numericSort)];
             }
         });
     });
@@ -162,72 +200,23 @@ var ResourceProcessor = /** @class */ (function () {
         this.DIR_DWC = path.join(this.DIR_ROOT, 'dwc');
         this.FILE_PROBLEMS = path.join(this.DIR_ROOT, 'problems.csv');
     }
-    ResourceProcessor.prototype.run = function () {
+    ResourceProcessor.prototype.run = function (source, config) {
         return __awaiter(this, void 0, void 0, function () {
-            var input, output, ids, _loop_1, this_1, _i, ids_1, id;
+            var ids, _i, ids_1, id;
             return __generator(this, function (_a) {
                 switch (_a.label) {
-                    case 0: return [4 /*yield*/, fs_1.promises.readdir(this.DIR_TXT)];
+                    case 0: return [4 /*yield*/, this.listWorks(source)];
                     case 1:
-                        input = _a.sent();
-                        return [4 /*yield*/, fs_1.promises.readdir(this.DIR_DWC)];
-                    case 2:
-                        output = _a.sent();
-                        ids = input
-                            .map(function (file) { return path.basename(file, '.txt'); })
-                            .sort(function (a, b) { return parseInt(a.slice(1)) - parseInt(b.slice(1)); });
-                        _loop_1 = function (id) {
-                            return __generator(this, function (_b) {
-                                switch (_b.label) {
-                                    case 0:
-                                        // Skip existing files
-                                        if (output.some(function (file) { return file.startsWith(id + '-'); })) {
-                                            return [2 /*return*/, "continue"];
-                                        }
-                                        return [4 /*yield*/, this_1.processWork(id)];
-                                    case 1:
-                                        _b.sent();
-                                        return [2 /*return*/];
-                                }
-                            });
-                        };
-                        this_1 = this;
+                        ids = _a.sent();
                         _i = 0, ids_1 = ids;
-                        _a.label = 3;
-                    case 3:
-                        if (!(_i < ids_1.length)) return [3 /*break*/, 6];
-                        id = ids_1[_i];
-                        return [5 /*yield**/, _loop_1(id)];
-                    case 4:
-                        _a.sent();
-                        _a.label = 5;
-                    case 5:
-                        _i++;
-                        return [3 /*break*/, 3];
-                    case 6: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    ResourceProcessor.prototype.runUpdate = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            var _i, _a, file, id;
-            return __generator(this, function (_b) {
-                switch (_b.label) {
-                    case 0:
-                        _i = 0;
-                        return [4 /*yield*/, listChangedFiles(this.DIR_TXT)];
-                    case 1:
-                        _a = _b.sent();
-                        _b.label = 2;
+                        _a.label = 2;
                     case 2:
-                        if (!(_i < _a.length)) return [3 /*break*/, 5];
-                        file = _a[_i];
-                        id = path.basename(file, '.txt');
-                        return [4 /*yield*/, this.processWork(id, true)];
+                        if (!(_i < ids_1.length)) return [3 /*break*/, 5];
+                        id = ids_1[_i];
+                        return [4 /*yield*/, this.processWork(id, config)];
                     case 3:
-                        _b.sent();
-                        _b.label = 4;
+                        _a.sent();
+                        _a.label = 4;
                     case 4:
                         _i++;
                         return [3 /*break*/, 2];
@@ -236,52 +225,41 @@ var ResourceProcessor = /** @class */ (function () {
             });
         });
     };
-    ResourceProcessor.prototype.runMappingsUpdate = function () {
+    ResourceProcessor.prototype.listWorks = function (source) {
         return __awaiter(this, void 0, void 0, function () {
-            var input, ids, _i, ids_2, id;
             return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0: return [4 /*yield*/, fs_1.promises.readdir(this.DIR_TXT)];
-                    case 1:
-                        input = _a.sent();
-                        ids = input
-                            .map(function (file) { return path.basename(file, '.txt'); })
-                            .sort(function (a, b) { return parseInt(a.slice(1)) - parseInt(b.slice(1)); });
-                        _i = 0, ids_2 = ids;
-                        _a.label = 2;
-                    case 2:
-                        if (!(_i < ids_2.length)) return [3 /*break*/, 5];
-                        id = ids_2[_i];
-                        return [4 /*yield*/, this.processWork(id, true)];
-                    case 3:
-                        _a.sent();
-                        _a.label = 4;
-                    case 4:
-                        _i++;
-                        return [3 /*break*/, 2];
-                    case 5: return [2 /*return*/];
+                switch (source) {
+                    case ResourceProcessorSource.All:
+                        return [2 /*return*/, listFiles(this.DIR_TXT)];
+                    case ResourceProcessorSource.Unprocessed:
+                        return [2 /*return*/, listUnprocessedFiles(this.DIR_TXT, this.DIR_DWC)];
+                    case ResourceProcessorSource.Modified:
+                        return [2 /*return*/, listChangedFiles(this.DIR_TXT)];
+                    default:
+                        return [2 /*return*/, []];
                 }
+                return [2 /*return*/];
             });
         });
     };
-    ResourceProcessor.prototype.processWork = function (id, update) {
+    ResourceProcessor.prototype.processWork = function (id, config) {
         return __awaiter(this, void 0, void 0, function () {
             var resources;
             var _this = this;
             return __generator(this, function (_a) {
                 switch (_a.label) {
-                    case 0: return [4 /*yield*/, this.processResources(id, update)];
+                    case 0: return [4 /*yield*/, this.processResources(id, config)];
                     case 1:
                         resources = _a.sent();
                         return [4 /*yield*/, Promise.all(resources.map(function (resource) {
                                 var header = DWC_FIELDS;
                                 var table = [header];
-                                var _loop_2 = function (id_1) {
+                                var _loop_1 = function (id_1) {
                                     var taxon = resource.taxa[id_1];
                                     table.push(header.map(function (column) { return taxon[column] || ''; }));
                                 };
                                 for (var id_1 in resource.taxa) {
-                                    _loop_2(id_1);
+                                    _loop_1(id_1);
                                 }
                                 return fs_1.promises.writeFile(path.join(_this.DIR_DWC, "".concat(resource.file, ".csv")), index_1.csv.formatCsv(table, ',').trim());
                             }))];
@@ -292,12 +270,12 @@ var ResourceProcessor = /** @class */ (function () {
             });
         });
     };
-    ResourceProcessor.prototype.processResources = function (id, update) {
+    ResourceProcessor.prototype.processResources = function (id, config) {
         return __awaiter(this, void 0, void 0, function () {
             var resources, amendedResources, _i, resources_1, resource, results, skip, correct, choice, _a, reason;
             return __generator(this, function (_b) {
                 switch (_b.label) {
-                    case 0: return [4 /*yield*/, this.processResourceText(id, update)];
+                    case 0: return [4 /*yield*/, this.processResourceText(id, config)];
                     case 1:
                         resources = _b.sent();
                         amendedResources = [];
@@ -306,7 +284,7 @@ var ResourceProcessor = /** @class */ (function () {
                     case 2:
                         if (!(_i < resources_1.length)) return [3 /*break*/, 11];
                         resource = resources_1[_i];
-                        return [4 /*yield*/, this.processResourceDwc(resource)];
+                        return [4 /*yield*/, this.processResourceDwc(resource, config)];
                     case 3:
                         results = _b.sent();
                         return [4 /*yield*/, this.shouldBeSkipped(resource.id)];
@@ -339,7 +317,7 @@ var ResourceProcessor = /** @class */ (function () {
                     case 8:
                         {
                             console.log("".concat(resource.workId, ": retrying ").concat(resource.id));
-                            return [2 /*return*/, this.processResources(id, update)];
+                            return [2 /*return*/, this.processResources(id, config)];
                         }
                         _b.label = 9;
                     case 9:
@@ -353,7 +331,7 @@ var ResourceProcessor = /** @class */ (function () {
             });
         });
     };
-    ResourceProcessor.prototype.processResourceText = function (id, update) {
+    ResourceProcessor.prototype.processResourceText = function (id, config) {
         return __awaiter(this, void 0, void 0, function () {
             var filePath, file, old, dwc, _i, _a, file_1, filePath_1, _b, _c, _d, _e, resources, error_1, prefix, file;
             var _f;
@@ -367,7 +345,7 @@ var ResourceProcessor = /** @class */ (function () {
                     case 1:
                         file = _g.sent();
                         old = undefined;
-                        if (!update) return [3 /*break*/, 8];
+                        if (!config.update) return [3 /*break*/, 8];
                         dwc = [];
                         _i = 0;
                         return [4 /*yield*/, fs_1.promises.readdir(this.DIR_DWC)];
@@ -412,19 +390,43 @@ var ResourceProcessor = /** @class */ (function () {
                                 delete require.cache[file];
                             }
                         }
-                        return [2 /*return*/, this.processResourceText(id, update)];
+                        return [2 /*return*/, this.processResourceText(id, config)];
                     case 12: return [2 /*return*/];
                 }
             });
         });
     };
-    ResourceProcessor.prototype.processResourceDwc = function (resource) {
+    ResourceProcessor.prototype.processResourceDwc = function (resource, config) {
         return __awaiter(this, void 0, void 0, function () {
-            var filteredResults, taxonNames, names, id, name_1, result, _i, _a, results, _b, name_2, matches, _c, matches_1, match, source, currentRank, _d, _e, loirId, taxon, _f, amendResource, groupNameMatches, groupedNameMatches, amendedResource, _g, _h, _j, _k, source, matches;
-            return __generator(this, function (_l) {
-                switch (_l.label) {
+            var file, _a, header_1, rows, _b, _c, _i, rows_1, row, oldTaxon, taxon, filteredResults, taxonNames, names, id, name_1, result, _d, _e, results, _f, name_2, matches, _g, matches_1, match, source, currentRank, _h, _j, loirId, taxon, _k, amendResource, groupNameMatches, groupedNameMatches, amendedResource, _l, _m, _o, _p, source, matches;
+            return __generator(this, function (_q) {
+                switch (_q.label) {
                     case 0:
                         console.log("".concat(resource.workId, ": matching ").concat(resource.id));
+                        if (!!config.updateMappings) return [3 /*break*/, 3];
+                        file = path.join(this.DIR_DWC, resource.file + '.csv');
+                        if (!(0, fs_1.existsSync)(file)) return [3 /*break*/, 2];
+                        _c = (_b = index_1.csv).parseCsv;
+                        return [4 /*yield*/, fs_1.promises.readFile(file, 'utf-8')];
+                    case 1:
+                        _a = _c.apply(_b, [_q.sent()]), header_1 = _a[0], rows = _a.slice(1);
+                        for (_i = 0, rows_1 = rows; _i < rows_1.length; _i++) {
+                            row = rows_1[_i];
+                            oldTaxon = row.reduce(function (taxon, value, index) {
+                                taxon[header_1[index]] = value;
+                                return taxon;
+                            }, {});
+                            taxon = resource.taxa[oldTaxon.scientificNameID];
+                            if (taxon) {
+                                taxon.colTaxonID = oldTaxon.colTaxonID;
+                                taxon.colAcceptedTaxonID = oldTaxon.colAcceptedTaxonID;
+                                taxon.gbifTaxonID = oldTaxon.gbifTaxonID;
+                                taxon.gbifAcceptedTaxonID = oldTaxon.gbifAcceptedTaxonID;
+                            }
+                        }
+                        _q.label = 2;
+                    case 2: return [2 /*return*/, resource];
+                    case 3:
                         filteredResults = {};
                         taxonNames = {};
                         names = new Set();
@@ -438,16 +440,16 @@ var ResourceProcessor = /** @class */ (function () {
                             filteredResults[id] = [];
                         }
                         return [4 /*yield*/, runGnverifier(Array.from(names).join('\n'))];
-                    case 1:
-                        result = _l.sent();
-                        for (_i = 0, _a = result.trim().split('\n'); _i < _a.length; _i++) {
-                            results = _a[_i];
-                            _b = JSON.parse(results), name_2 = _b.name, matches = _b.results;
+                    case 4:
+                        result = _q.sent();
+                        for (_d = 0, _e = result.trim().split('\n'); _d < _e.length; _d++) {
+                            results = _e[_d];
+                            _f = JSON.parse(results), name_2 = _f.name, matches = _f.results;
                             if (!matches) {
                                 continue;
                             }
-                            for (_c = 0, matches_1 = matches; _c < matches_1.length; _c++) {
-                                match = matches_1[_c];
+                            for (_g = 0, matches_1 = matches; _g < matches_1.length; _g++) {
+                                match = matches_1[_g];
                                 source = match.dataSourceId;
                                 currentRank = match.classificationRanks.split('|').pop();
                                 if (match.scoreDetails.cardinalityScore === 0) {
@@ -458,8 +460,8 @@ var ResourceProcessor = /** @class */ (function () {
                                     // GBIF species like "Nomada spec"
                                     continue;
                                 }
-                                for (_d = 0, _e = taxonNames[name_2]; _d < _e.length; _d++) {
-                                    loirId = _e[_d];
+                                for (_h = 0, _j = taxonNames[name_2]; _h < _j.length; _h++) {
+                                    loirId = _j[_h];
                                     taxon = resource.taxa[loirId];
                                     if (source === 11 && !GBIF_RANKS.includes(taxon.taxonRank)) {
                                         // Exclude GBIF matches for ranks that are not in GBIF
@@ -483,30 +485,30 @@ var ResourceProcessor = /** @class */ (function () {
                             }
                         }
                         return [4 /*yield*/, Promise.resolve().then(function () { return require('../index'); })];
-                    case 2:
-                        _f = (_l.sent()).taxonNames, amendResource = _f.amendResource, groupNameMatches = _f.groupNameMatches;
+                    case 5:
+                        _k = (_q.sent()).taxonNames, amendResource = _k.amendResource, groupNameMatches = _k.groupNameMatches;
                         groupedNameMatches = groupNameMatches(filteredResults);
                         amendedResource = __assign(__assign({}, resource), { taxa: __assign({}, resource.taxa) });
-                        _g = groupedNameMatches;
-                        _h = [];
-                        for (_j in _g)
-                            _h.push(_j);
-                        _k = 0;
-                        _l.label = 3;
-                    case 3:
-                        if (!(_k < _h.length)) return [3 /*break*/, 6];
-                        _j = _h[_k];
-                        if (!(_j in _g)) return [3 /*break*/, 5];
-                        source = _j;
+                        _l = groupedNameMatches;
+                        _m = [];
+                        for (_o in _l)
+                            _m.push(_o);
+                        _p = 0;
+                        _q.label = 6;
+                    case 6:
+                        if (!(_p < _m.length)) return [3 /*break*/, 9];
+                        _o = _m[_p];
+                        if (!(_o in _l)) return [3 /*break*/, 8];
+                        source = _o;
                         return [4 /*yield*/, this.selectPrefixes(resource, groupedNameMatches, source)];
-                    case 4:
-                        matches = _l.sent();
+                    case 7:
+                        matches = _q.sent();
                         amendResource(amendedResource, source, matches);
-                        _l.label = 5;
-                    case 5:
-                        _k++;
-                        return [3 /*break*/, 3];
-                    case 6: return [2 /*return*/, amendedResource];
+                        _q.label = 8;
+                    case 8:
+                        _p++;
+                        return [3 /*break*/, 6];
+                    case 9: return [2 /*return*/, amendedResource];
                 }
             });
         });
@@ -635,12 +637,14 @@ var ResourceProcessor = /** @class */ (function () {
 function main() {
     var args = util.parseArgs({
         options: {
-            update: {
-                type: 'boolean',
-                short: 'u'
+            source: {
+                type: 'string',
+                short: 's',
+                default: 'unprocessed'
             },
-            'update-mappings': {
-                type: 'boolean'
+            'keep-mappings': {
+                type: 'boolean',
+                short: 'k'
             }
         },
         allowPositionals: true
@@ -649,17 +653,12 @@ function main() {
     process.on('exit', function () {
         process.stdout.write('\n');
     });
-    var task;
-    if (args.values.update) {
-        task = processor.runUpdate();
-    }
-    else if (args.values['update-mappings']) {
-        task = processor.runMappingsUpdate();
-    }
-    else {
-        task = processor.run();
-    }
-    task.catch(function (error) {
+    var source = args.values.source;
+    var config = {
+        update: source !== 'unprocessed',
+        updateMappings: !args.values['keep-mappings']
+    };
+    processor.run(source, config).catch(function (error) {
         console.error(error);
         process.exit(1);
     });

package/lib/catalog/value.js

@@ -22,10 +22,10 @@ exports.FORMATS = {
 };
 exports.CHECK = {
     MULTILANG: function (entry) {
-        return entry.language.length > 1;
+        return Array.isArray(entry.language) && entry.language.length > 1;
     },
     ISBN: function (entry) {
-        if (entry.ISBN.length < 2) {
+        if (!Array.isArray(entry.ISBN) || entry.ISBN.length < 2) {
             return false;
         }
         var a = entry.ISBN[0].length === 10;

package/lib/csv.js

@@ -25,7 +25,7 @@ function formatCsv(table, delimiter) {
         .map(function (row) {
         return row.map(function (value) {
             if (/["\n]/.test(value) || value.includes(delimiter)) {
-                return "\"".concat(value.replace(/"/, '"""'), "\"");
+                return "\"".concat(value.replace(/"/g, '""'), "\"");
             }
             else {
                 return value;