@larsgw/formica 0.9.2 → 0.10.1

package/lib/bin/process-resources.d.ts

@@ -1,6 +1,11 @@
 #!/usr/bin/env node
+export declare enum ResourceTaxonIdSource {
+    COL = "col",
+    GBIF = "gbif"
+}
 export declare enum ResourceProcessorSource {
     All = "all",
     Unprocessed = "unprocessed",
-    Modified = "modified"
+    Modified = "modified",
+    Dwc = "dwc"
 }

package/lib/bin/process-resources.js

@@ -43,18 +43,24 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ResourceProcessorSource = void 0;
+exports.ResourceProcessorSource = exports.ResourceTaxonIdSource = void 0;
 const fs_1 = require("fs");
 const path = __importStar(require("path"));
 const child_process_1 = require("child_process");
 const util = __importStar(require("util"));
 const index_1 = require("../index");
 const util_1 = require("./util");
+var ResourceTaxonIdSource;
+(function (ResourceTaxonIdSource) {
+    ResourceTaxonIdSource["COL"] = "col";
+    ResourceTaxonIdSource["GBIF"] = "gbif";
+})(ResourceTaxonIdSource || (exports.ResourceTaxonIdSource = ResourceTaxonIdSource = {}));
 var ResourceProcessorSource;
 (function (ResourceProcessorSource) {
     ResourceProcessorSource["All"] = "all";
     ResourceProcessorSource["Unprocessed"] = "unprocessed";
     ResourceProcessorSource["Modified"] = "modified";
+    ResourceProcessorSource["Dwc"] = "dwc";
 })(ResourceProcessorSource || (exports.ResourceProcessorSource = ResourceProcessorSource = {}));
 const DWC_FIELDS = [
     'scientificNameID',
@@ -108,9 +114,14 @@ const GBIF_RANKS = [
     'subspecies',
     'variety'
 ];
-function runGnverifier(names) {
+const GNVERIFIER_SOURCES = {
+    gbif: 11,
+    col: 13
+};
+function runGnverifier(names, sources) {
+    const sourceArg = sources.map(source => GNVERIFIER_SOURCES[source]).join();
     return new Promise((resolve, reject) => {
-        const proc = (0, child_process_1.spawn)('gnverifier', ['-s', '1,11', '-f', 'compact', '-M']);
+        const proc = (0, child_process_1.spawn)('gnverifier', ['-s', sourceArg, '-f', 'compact', '-M']);
         let stdout = '';
         proc.stdout.on('data', data => { stdout += data; });
         proc.stderr.pipe(process.stdout);
@@ -165,16 +176,44 @@ class ResourceProcessor {
     }
     run(source, config) {
         return __awaiter(this, void 0, void 0, function* () {
+            if (source === ResourceProcessorSource.Dwc) {
+                return this.runDwc(config);
+            }
             const ids = yield this.listWorks(source);
             for (const id of ids) {
                 yield this.processWork(id, config);
             }
         });
     }
+    runDwc(config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const files = yield fs_1.promises.readdir(this.DIR_DWC);
+            files.sort(util_1.numericSort);
+            for (const file of files) {
+                const [workId, index] = path.basename(file, '.csv').split('-');
+                const resource = {
+                    id: `${workId}:${index}`,
+                    file: `${workId}-${index}`,
+                    workId,
+                    metadata: { levels: [] },
+                    taxa: {}
+                };
+                const filePath = path.join(this.DIR_DWC, file);
+                const [header, ...rows] = index_1.csv.parseCsv(yield fs_1.promises.readFile(filePath, 'utf-8'));
+                for (const row of rows) {
+                    const taxon = Object.fromEntries(header.map((h, i) => [h, row[i]]));
+                    resource.taxa[taxon.scientificNameID] = taxon;
+                }
+                yield this.processResourceDwc(resource, config);
+                yield this.writeResourceDwc(resource);
+            }
+        });
+    }
     listWorks(source) {
         return __awaiter(this, void 0, void 0, function* () {
             switch (source) {
                 case ResourceProcessorSource.All:
+                case ResourceProcessorSource.Dwc:
                     return listFiles(this.DIR_TXT);
                 case ResourceProcessorSource.Unprocessed:
                     return listUnprocessedFiles(this.DIR_TXT, this.DIR_DWC);
@@ -188,15 +227,18 @@ class ResourceProcessor {
     processWork(id, config) {
         return __awaiter(this, void 0, void 0, function* () {
             const resources = yield this.processResources(id, config);
-            yield Promise.all(resources.map(resource => {
-                const header = DWC_FIELDS;
-                const table = [header];
-                for (const id in resource.taxa) {
-                    const taxon = resource.taxa[id];
-                    table.push(header.map(column => taxon[column] || ''));
-                }
-                return fs_1.promises.writeFile(path.join(this.DIR_DWC, `${resource.file}.csv`), index_1.csv.formatCsv(table, ',').trim());
-            }));
+            yield Promise.all(resources.map(resource => this.writeResourceDwc(resource)));
+        });
+    }
+    writeResourceDwc(resource) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const header = DWC_FIELDS;
+            const table = [header];
+            for (const id in resource.taxa) {
+                const taxon = resource.taxa[id];
+                table.push(header.map(column => taxon[column] || ''));
+            }
+            return fs_1.promises.writeFile(path.join(this.DIR_DWC, `${resource.file}.csv`), index_1.csv.formatCsv(table, ',').trim());
         });
     }
     processResources(id, config) {
@@ -243,20 +285,18 @@ class ResourceProcessor {
                 const file = yield fs_1.promises.readFile(filePath, 'utf-8');
                 let old = undefined;
                 if (config.update) {
-                    const dwc = [];
-                    for (const file of yield fs_1.promises.readdir(this.DIR_DWC)) {
-                        if (file.startsWith(id + '-')) {
-                            const filePath = path.join(this.DIR_DWC, file);
-                            dwc.push(index_1.csv.parseCsv(yield getOldFile(filePath)));
-                        }
-                    }
-                    old = { txt: yield getOldFile(filePath), dwc };
+                    old = {
+                        txt: yield getOldFile(filePath),
+                        dwc: yield this.readResourceDwc(id)
+                    };
                 }
                 const { resources } = yield Promise.resolve().then(() => __importStar(require('../index')));
                 return resources.parseTextFile(file, id, old);
             }
             catch (error) {
-                console.log(error.message);
+                if (error instanceof Error) {
+                    console.log(error.message);
+                }
                 yield (0, util_1.prompt)(`${id}: generating Darwin Core failed, retry? `);
                 // Clear cache to re-import
                 const prefix = path.dirname(require.resolve('../index'));
@@ -269,10 +309,25 @@ class ResourceProcessor {
             }
         });
     }
+    readResourceDwc(id) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const dwc = [];
+            for (const file of yield fs_1.promises.readdir(this.DIR_DWC)) {
+                if (file.startsWith(id + '-')) {
+                    const filePath = path.join(this.DIR_DWC, file);
+                    dwc.push(index_1.csv.parseCsv(yield getOldFile(filePath)));
+                }
+            }
+            return dwc;
+        });
+    }
     processResourceDwc(resource, config) {
         return __awaiter(this, void 0, void 0, function* () {
             console.log(`${resource.workId}: matching ${resource.id}`);
-            if (!config.updateMappings) {
+            // If not all mappings will be rerun and the mappings are not already provided,
+            // read the existing DwC files and add the corresponding mappings.
+            const hasMappings = Object.values(resource.taxa).some(taxon => 'gbifAcceptedTaxonID' in taxon || 'colAcceptedTaxonID' in taxon);
+            if (!hasMappings && config.updateMappings.length < 2) {
                 const file = path.join(this.DIR_DWC, resource.file + '.csv');
                 if ((0, fs_1.existsSync)(file)) {
                     const [header, ...rows] = index_1.csv.parseCsv(yield fs_1.promises.readFile(file, 'utf-8'));
@@ -290,6 +345,8 @@ class ResourceProcessor {
                         }
                     }
                 }
+            }
+            if (config.updateMappings.length === 0) {
                 return resource;
             }
             const filteredResults = {};
@@ -304,7 +361,7 @@ class ResourceProcessor {
                 names.add(name);
                 filteredResults[id] = [];
             }
-            const result = yield runGnverifier(Array.from(names).join('\n'));
+            const result = yield runGnverifier(Array.from(names).join('\n'), config.updateMappings);
             for (const results of result.trim().split('\n')) {
                 const { name, results: matches } = JSON.parse(results);
                 if (!matches) {
@@ -339,6 +396,16 @@ class ResourceProcessor {
                             // for non-synonyms).
                             continue;
                         }
+                        // https://github.com/gnames/gnverifier/issues/156
+                        if (source === 1 || source === 13 && match.recordId !== match.currentRecordId) {
+                            const [a, b, c] = match.recordId.split('|');
+                            if (b && c) {
+                                if (a !== c || b !== match.currentRecordId) {
+                                    throw new Error(`Unexpected recordId format ${match.recordId}`);
+                                }
+                                match.recordId = a;
+                            }
+                        }
                         if (!filteredResults[loirId]) {
                             filteredResults[loirId] = [];
                         }
@@ -468,18 +535,42 @@ function main() {
             'keep-mappings': {
                 type: 'boolean',
                 short: 'k'
+            },
+            'update-mappings': {
+                type: 'string',
+                short: 'u',
+                multiple: true
             }
         },
         allowPositionals: true
     });
+    if (!Object.values(ResourceProcessorSource).includes(args.values.source)) {
+        throw new Error(`Unknown argument "--source ${args.values.source}"`);
+    }
+    const mappingSources = Object.values(ResourceTaxonIdSource);
+    let updateMappings = mappingSources;
+    if (args.values['keep-mappings']) {
+        if (args.values['update-mappings']) {
+            throw new Error('Cannot provide both --keep-mappings (-k) and --update-mappings (-u)');
+        }
+        updateMappings = [];
+    }
+    else if (args.values['update-mappings']) {
+        for (const source of args.values['update-mappings']) {
+            if (!mappingSources.includes(source)) {
+                throw new Error(`Unknown argument "--update-mappings ${args.values.source}"`);
+            }
+        }
+        updateMappings = args.values['update-mappings'];
+    }
     const processor = new ResourceProcessor(args.positionals[0]);
     process.on('exit', () => {
         process.stdout.write('\n');
     });
     const source = args.values.source;
     const config = {
-        update: source !== 'unprocessed',
-        updateMappings: !args.values['keep-mappings']
+        update: source === ResourceProcessorSource.All || source === ResourceProcessorSource.Modified,
+        updateMappings
     };
     processor.run(source, config).catch(error => {
         console.error(error);

package/lib/bin/validate-catalog.js

@@ -47,37 +47,38 @@ const fs_1 = require("fs");
 const path = __importStar(require("path"));
 const index_1 = require("../index");
 class TaxaValidator {
-    constructor(taxa) {
-        this.taxa = Array.from(taxa).filter(taxon => taxon.has('ancestors_gbif'));
+    constructor(taxa, fields) {
+        this.taxa = Array.from(taxa).filter(taxon => taxon.has(fields.ancestors));
+        this.fields = fields;
         this.errors = [];
         this.parentIndex = {};
     }
     validate() {
         for (const taxon of this.taxa) {
-            const parents = taxon.get('ancestors_gbif');
+            const parents = taxon.get(this.fields.ancestors);
             if (parents.length > 1) {
                 const id = taxon.get('id');
                 for (let i = 1; i < parents.length; i++) {
-                    this.addToIndex(id, 'ancestors_gbif', parents[i], parents[i - 1]);
+                    this.addToIndex(id, this.fields.ancestors, parents[i], parents[i - 1]);
                 }
             }
         }
         for (const taxon of this.taxa) {
-            if (taxon.has('gbif')) {
+            if (taxon.has(this.fields.id)) {
                 const id = taxon.get('id');
-                const parents = taxon.get('ancestors_gbif');
+                const parents = taxon.get(this.fields.ancestors);
                 const parent = parents[parents.length - 1];
-                this.addToIndex(id, 'gbif', taxon.get('gbif'), parent);
+                this.addToIndex(id, this.fields.id, taxon.get(this.fields.id), parent);
             }
         }
         for (const taxon of this.taxa) {
-            if (taxon.has('children_gbif')) {
+            if (taxon.has(this.fields.children)) {
                 const id = taxon.get('id');
-                const parents = taxon.get('ancestors_gbif');
+                const parents = taxon.get(this.fields.ancestors);
                 const parent = parents[parents.length - 1];
-                const childIds = taxon.get('children_gbif');
+                const childIds = taxon.get(this.fields.children);
                 for (const childId of childIds) {
-                    this.addToIndex(id, 'gbif', childId, parent);
+                    this.addToIndex(id, this.fields.id, childId, parent);
                 }
             }
         }
@@ -94,6 +95,24 @@ class TaxaValidator {
         }
     }
 }
+class GbifTaxaValidator extends TaxaValidator {
+    constructor(taxa) {
+        super(taxa, {
+            id: 'gbif',
+            children: 'children_gbif',
+            ancestors: 'ancestors_gbif'
+        });
+    }
+}
+class ColTaxaValidator extends TaxaValidator {
+    constructor(taxa) {
+        super(taxa, {
+            id: 'col',
+            children: 'children_col',
+            ancestors: 'ancestors_col'
+        });
+    }
+}
 function validateFile(arg) {
     return __awaiter(this, void 0, void 0, function* () {
         const filePath = path.resolve(arg);
@@ -116,8 +135,10 @@ function validateFile(arg) {
             }
         }
         if (sheet === 'taxa') {
-            const validator = new TaxaValidator(entities);
-            errors.push(...validator.validate());
+            // const colValidator = new ColTaxaValidator(entities)
+            // errors.push(...colValidator.validate())
+            const gbifValidator = new GbifTaxaValidator(entities);
+            errors.push(...gbifValidator.validate());
         }
         return errors;
     });

package/lib/bin/validate-resources-text.js

@@ -57,7 +57,9 @@ function main(args) {
                 index_1.resources.parseTextFile(file, id);
             }
             catch (error) {
-                console.error(filePath + '\n' + error.message.replace(/^/gm, '    ') + '\n');
+                if (error instanceof Error) {
+                    console.error(filePath + '\n' + error.message.replace(/^/gm, '    ') + '\n');
+                }
                 exitStatus = 1;
             }
         }

package/lib/catalog/tables/taxon.js

@@ -10,6 +10,10 @@ class Taxon extends entity_1.Entity {
             id: { required: true, multiple: false, format: value_1.FORMATS.TAXON_ID },
             qid: { required: false, multiple: false, format: value_1.FORMATS.QID },
             rank: { required: false, multiple: false },
+            col: { required: false, multiple: false, format: value_1.FORMATS.COL_ID },
+            accepted_col: { required: false, multiple: false, format: value_1.FORMATS.COL_ID },
+            children_col: { required: false, multiple: true, format: value_1.FORMATS.COL_ID },
+            ancestors_col: { required: false, multiple: true, format: value_1.FORMATS.COL_ID },
             gbif: { required: false, multiple: false, format: value_1.FORMATS.INTEGER },
             children_gbif: { required: false, multiple: true, format: value_1.FORMATS.INTEGER },
             ancestors_gbif: { required: false, multiple: true, format: value_1.FORMATS.INTEGER },

package/lib/catalog/value.d.ts

@@ -14,6 +14,7 @@ export declare const FORMATS: {
     QID: RegExp;
     URL: RegExp;
     INTEGER: RegExp;
+    COL_ID: RegExp;
     LICENSE(value: SingleValue): true;
     LANGUAGE(value: SingleValue): boolean;
 };

package/lib/catalog/value.js

@@ -24,6 +24,7 @@ exports.FORMATS = {
     QID: /^Q[1-9][0-9]*$/,
     URL: /^(ftp|http|https):\/\/((?:[a-z0-9][a-z0-9-_]*?[a-z0-9]?\.)+(?:xn--)?[a-z0-9]+)(:\d*)?((?:\/(?:%\d\d|[!$&'()*+,\-.0-9";=@A-Z_a-z~])*)*)(\?(?:%\d\d|[!$&'()*+,\-./0-9:;=?@A-Z_a-z~])*)?(#(?:%\d\d|[!$&'()*+,\-./0-9:;=?@A-Z_a-z~])*)?/i,
     INTEGER: /^(0|[1-9]\d*)$/,
+    COL_ID: /^(0|[1-9A-Z][0-9A-Z]*)$/,
     LICENSE(value) { return /^<(public domain|.+\?)>$/.test(value) || !!spdx_license_list_1.default[value]; },
     LANGUAGE(value) { return ietfTagList.test(value); }
 };

package/lib/resources/parse-text.js

@@ -147,7 +147,7 @@ function parseResource(resource) {
         config = parseHeader(header);
     }
     catch (error) {
-        throw makeParseError(error.message, resource.offsetLine + 1);
+        throw makeParseError(error instanceof Error ? error.message : 'Error', resource.offsetLine + 1);
     }
     const content = rest.join('');
     const offsetLine = resource.offsetLine + (header + _).split('\n').length - 1;
@@ -257,7 +257,7 @@ function parseResourceContent(content, resource, oldIds, offsetLine) {
                 item = error.result;
             }
             else {
-                errors.push(makeParseError(error.message, lineNumber));
+                errors.push(makeParseError(error instanceof Error ? error.message : 'Error', lineNumber));
                 continue;
             }
         }
@@ -382,7 +382,10 @@ function parseFile(file, id, old) {
             resources.push(parseResourceContent(diff, template, oldIds, content.offsetLine));
         }
         catch (error) {
-            errors.push(error);
+            if (error instanceof SyntaxError) {
+                errors.push(error);
+            }
+            throw error;
         }
     }
     if (errors.length) {

package/lib/taxon-names/index.js

@@ -57,7 +57,7 @@ function groupNameMatches(results) {
 function amendResource(resource, source, matches) {
     for (const id in matches) {
         const match = matches[id];
-        if (source === '1') {
+        if (source === '1' || source === '13') {
             resource.taxa[id].colTaxonID = match.id;
             if (match.currentId) {
                 resource.taxa[id].colAcceptedTaxonID = match.currentId;

package/package.json

@@ -1,16 +1,18 @@
 {
   "name": "@larsgw/formica",
-  "version": "0.9.2",
+  "version": "0.10.1",
   "description": "SDK and tools for data from the Library of Identification Resources",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
+  "files": [
+    "lib"
+  ],
   "bin": {
-    "loir-validate-catalog": "./lib/bin/validate-catalog.js",
-    "loir-validate-resources": "./lib/bin/validate-resources-text.js",
-    "loir-resources-process": "./lib/bin/process-resources.js",
-    "loir-resources-index": "./lib/bin/process-resources-index.js",
-    "loir-generate-linked-data": "./lib/bin/generate-linked-data.js",
-    "loir-catalog-clean-links": "./lib/bin/clean-links.js"
+    "loir-validate-catalog": "lib/bin/validate-catalog.js",
+    "loir-validate-resources": "lib/bin/validate-resources-text.js",
+    "loir-resources-process": "lib/bin/process-resources.js",
+    "loir-resources-index": "lib/bin/process-resources-index.js",
+    "loir-generate-linked-data": "lib/bin/generate-linked-data.js"
   },
   "scripts": {
     "test": "node --test --test-reporter spec",
@@ -33,22 +35,22 @@
   "homepage": "https://github.com/identification-resources/formica#readme",
   "dependencies": {
     "ietf-language-tag-regex": "^0.0.5",
-    "js-yaml": "^4.1.0",
+    "js-yaml": "^5.2.1",
     "jsonld": "^9.0.0",
-    "n3": "^1.25.2",
+    "n3": "^2.1.1",
     "spdx-license-list": "^6.6.0"
   },
   "devDependencies": {
     "@eslint/eslintrc": "^3.3.1",
-    "@eslint/js": "^9.26.0",
+    "@eslint/js": "^10.0.1",
     "@types/js-yaml": "^4.0.5",
     "@types/jsonld": "^1.5.15",
     "@types/n3": "^1.24.2",
-    "@types/node": "^22.15.14",
+    "@types/node": "^26.1.1",
     "@typescript-eslint/eslint-plugin": "^8.32.0",
     "@typescript-eslint/parser": "^8.32.0",
     "conventional-changelog-cli": "^5.0.0",
-    "eslint": "^9.26.0",
-    "typescript": "^5.8.3"
+    "eslint": "^10.6.0",
+    "typescript": "^6.0.3"
   }
 }

package/.gitattributes

@@ -1 +0,0 @@
-*.ts linguist-language=TypeScript

package/.github/workflows/ci.yml

@@ -1,27 +0,0 @@
-name: Node.js CI
-
-on:
-  push:
-    branches: [ "main" ]
-  pull_request:
-    branches: [ "main" ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        node-version: [20.x, 22.x]
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
-      with:
-        node-version: ${{ matrix.node-version }}
-    - run: npm install
-    - run: npm run build
-    - run: npm run lint
-    - run: npm test