@larsgw/formica 0.10.1 → 0.10.3

package/README.md

@@ -20,13 +20,25 @@ Run locally:
   - `loir-validate-resources [./B1234.txt]`:
     Validate the `.txt` files containing information on the taxonomic scope of keys
     ([documentation](https://github.com/identification-resources/catalog/blob/main/docs/resources-txt.md)).
+  - `loir-validate-integrity .`:
+    Check if all values are mapped as expected.
   - `loir-resources-process [./resources]`:
     Interactive tool to convert the aforementioned `.txt` files to Darwin Core archives
     ([documentation](https://github.com/identification-resources/catalog/blob/main/docs/resources-dwc.md)),
-    and link the taxa to GBIF and Catalogue of Life identifiers. Use the `-u` or
-    `--update` flag in a Git repo to update existing Darwin Core files while keeping
-    the identifiers stable (in most cases).
+    and link the taxa to GBIF and Catalogue of Life identifiers.
+      - Use `-s` or `--source` to specify which files should be processed: `unprocessed`
+        (default) for `.txt` files without corresponding Darwin Core files; `modified`
+        if in a Git repo to update existing Darwin Core files while keeping the identifiers
+        stable (in most cases); `all` for all `.txt` files; or `dwc` for only update
+        identifier mappings.
+      - Use `-u` or `--update-mappings` to specify which mappings to update: `col` or
+        `gbif`. This argument can be repeated. By default, both COL and GBIF are updated.
+      - Use `-k` or `--keep-mappings` to not update any mappings. Cannot be used in
+        conjunction with `-u`.
+      - Use `--mapping-api` to use either `gnverifier` (default) or `clb`, the ChecklistBank
+        API, to update the mappings.
   - `loir-resources-index [./resources]`: Create indices of the Darwin Core archives.
+  - `loir-resources-check-diff [./resources/dwc]`: Determine which columns changed.
 
 ## API
 

package/lib/bin/check-diff.js

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+// import { promises as fs, existsSync as doesFileExist } from 'fs'
+const path = __importStar(require("path"));
+const util = __importStar(require("util"));
+const util_1 = require("./util");
+function listChangedFiles(directory) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const output = yield (0, util_1.runCommand)('git', ['diff', '--name-only', 'HEAD', '--', directory], {
+            cwd: directory
+        });
+        return output.trimEnd().split('\n');
+    });
+}
+function getOldFile(file) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const options = {
+            cwd: path.dirname(file)
+        };
+        const gitRoot = (yield (0, util_1.runCommand)('git', ['rev-parse', '--show-toplevel'], options)).trim();
+        return yield (0, util_1.runCommand)('git', ['show', 'HEAD:' + path.relative(gitRoot, file)], options);
+    });
+}
+function main() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const args = util.parseArgs({
+            allowPositionals: true
+        });
+        const files = yield listChangedFiles(path.resolve(args.positionals[0]));
+        console.log(files);
+    });
+}
+main().catch(console.error);

package/lib/bin/check-resources-diff.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/lib/bin/check-resources-diff.js

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = require("fs");
+const path = __importStar(require("path"));
+const util = __importStar(require("util"));
+const index_1 = require("../index");
+const util_1 = require("./util");
+function listChangedFiles(directory) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const output = yield (0, util_1.runCommand)('git', ['diff', '--name-only', 'HEAD', '--', directory], {
+            cwd: directory
+        });
+        return output.trimEnd().split('\n');
+    });
+}
+function getOldFile(file, gitRoot) {
+    return __awaiter(this, void 0, void 0, function* () {
+        return (0, util_1.runCommand)('git', ['show', 'HEAD:' + path.relative(gitRoot, file)], {
+            cwd: path.dirname(file)
+        });
+    });
+}
+function main() {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a;
+        const args = util.parseArgs({
+            allowPositionals: true
+        });
+        const base = path.resolve(args.positionals[0]);
+        const files = yield listChangedFiles(base);
+        const gitRoot = (yield (0, util_1.runCommand)('git', ['rev-parse', '--show-toplevel'], {
+            cwd: args.positionals[0]
+        })).trim();
+        const columnIndex = {};
+        for (const fileName of files) {
+            const filePath = path.join(gitRoot, fileName);
+            const b = index_1.csv.parseCsv(yield fs_1.promises.readFile(filePath, 'utf8'));
+            const a = index_1.csv.parseCsv(yield getOldFile(filePath, gitRoot));
+            const header = a[0];
+            for (let i = 1; i < Math.max(a.length, b.length); i++) {
+                for (let j = 0; j < header.length; j++) {
+                    if (!a[i] || !b[i] || a[i][j] !== b[i][j]) {
+                        ((_a = columnIndex[header[j]]) !== null && _a !== void 0 ? _a : (columnIndex[header[j]] = new Set())).add(path.basename(fileName, '.csv'));
+                    }
+                }
+            }
+        }
+        const output = Object.entries(columnIndex)
+            .sort((a, b) => b[1].size - a[1].size)
+            .map(([column, files]) => ({
+            column,
+            count: files.size,
+            files: (files.size > 10 ? [...files].slice(0, 10).concat('...') : [...files]).join(', ')
+        }));
+        console.table(output);
+    });
+}
+main().catch(console.error);

package/lib/bin/diff-check.js

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+// import { promises as fs, existsSync as doesFileExist } from 'fs'
+const path = __importStar(require("path"));
+const util = __importStar(require("util"));
+const util_1 = require("./util");
+function listChangedFiles(directory) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const output = yield (0, util_1.runCommand)('git', ['diff', '--name-only', 'HEAD', '--', directory], {
+            cwd: directory
+        });
+        return output.trimEnd().split('\n');
+    });
+}
+function getOldFile(file) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const options = {
+            cwd: path.dirname(file)
+        };
+        const gitRoot = (yield (0, util_1.runCommand)('git', ['rev-parse', '--show-toplevel'], options)).trim();
+        return yield (0, util_1.runCommand)('git', ['show', 'HEAD:' + path.relative(gitRoot, file)], options);
+    });
+}
+function main() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const args = util.parseArgs({
+            allowPositionals: true
+        });
+        const files = yield listChangedFiles(args.positionals[0]);
+        console.log(files);
+    });
+}
+main().catch(console.error);

package/lib/bin/process-resources.d.ts

@@ -1,8 +1,4 @@
 #!/usr/bin/env node
-export declare enum ResourceTaxonIdSource {
-    COL = "col",
-    GBIF = "gbif"
-}
 export declare enum ResourceProcessorSource {
     All = "all",
     Unprocessed = "unprocessed",

package/lib/bin/process-resources.js

@@ -43,18 +43,19 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ResourceProcessorSource = exports.ResourceTaxonIdSource = void 0;
+exports.ResourceProcessorSource = void 0;
 const fs_1 = require("fs");
 const path = __importStar(require("path"));
 const child_process_1 = require("child_process");
 const util = __importStar(require("util"));
 const index_1 = require("../index");
 const util_1 = require("./util");
-var ResourceTaxonIdSource;
-(function (ResourceTaxonIdSource) {
-    ResourceTaxonIdSource["COL"] = "col";
-    ResourceTaxonIdSource["GBIF"] = "gbif";
-})(ResourceTaxonIdSource || (exports.ResourceTaxonIdSource = ResourceTaxonIdSource = {}));
+const { ResourceTaxonIdSource, formatClassificationPath } = index_1.taxonNames;
+var ResourceTaxonIdApi;
+(function (ResourceTaxonIdApi) {
+    ResourceTaxonIdApi["GNVerifier"] = "gnverifier";
+    ResourceTaxonIdApi["ChecklistBank"] = "clb";
+})(ResourceTaxonIdApi || (ResourceTaxonIdApi = {}));
 var ResourceProcessorSource;
 (function (ResourceProcessorSource) {
     ResourceProcessorSource["All"] = "all";
@@ -137,6 +138,55 @@ function runGnverifier(names, sources) {
         proc.stdin.end();
     });
 }
+const CLB_SOURCES = {
+    gbif: '53147',
+    col: '3LXR'
+};
+function getBatchedFetch(limit = 20) {
+    const queue = [];
+    let count = 0;
+    return function (...args) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (count >= limit) {
+                yield new Promise(resolve => { queue.push(resolve); });
+            }
+            count++;
+            try {
+                return yield fetch(...args);
+            }
+            finally {
+                count--;
+                if (count < limit) {
+                    const resolver = queue.shift();
+                    if (resolver) {
+                        resolver();
+                    }
+                }
+            }
+        });
+    };
+}
+function runClbNameParser(taxa, sources) {
+    const fetchBatched = getBatchedFetch(50);
+    const promises = [];
+    for (const id in taxa) {
+        const { scientificName, scientificNameAuthorship: authorship, taxonRank: rank } = taxa[id];
+        const params = new URLSearchParams({
+            scientificName: authorship ? scientificName.slice(0, -1 - authorship.length) : scientificName,
+            authorship: authorship !== null && authorship !== void 0 ? authorship : '',
+            rank: rank !== null && rank !== void 0 ? rank : ''
+        });
+        const queryString = params.toString();
+        for (const source of sources) {
+            const url = `https://api.checklistbank.org/dataset/${CLB_SOURCES[source]}/match/nameusage?verbose=true&${queryString}`;
+            const promise = fetchBatched(url)
+                .then(response => response.text())
+                .then(response => [taxa[id].scientificNameID, source, response]);
+            promises.push(promise);
+        }
+    }
+    return Promise.all(promises);
+}
 function listFiles(directory) {
     return __awaiter(this, void 0, void 0, function* () {
         const input = yield fs_1.promises.readdir(directory);
@@ -349,6 +399,22 @@ class ResourceProcessor {
             if (config.updateMappings.length === 0) {
                 return resource;
             }
+            const filteredResults = config.mappingsApi === ResourceTaxonIdApi.ChecklistBank
+                ? yield this.getClbNameParserResults(resource, config)
+                : yield this.getGnverifierResults(resource, config);
+            const { taxonNames: { amendResource, groupNameMatches } } = yield Promise.resolve().then(() => __importStar(require('../index')));
+            const groupedNameMatches = groupNameMatches(filteredResults);
+            const amendedResource = Object.assign(Object.assign({}, resource), { taxa: Object.assign({}, resource.taxa) });
+            for (const key in groupedNameMatches) {
+                const source = key;
+                const matches = yield this.selectPrefixes(resource, groupedNameMatches[source], source);
+                amendResource(amendedResource, source, matches);
+            }
+            return amendedResource;
+        });
+    }
+    getGnverifierResults(resource, config) {
+        return __awaiter(this, void 0, void 0, function* () {
             const filteredResults = {};
             const taxonNames = {};
             const names = new Set();
@@ -385,6 +451,8 @@ class ResourceProcessor {
                         // GBIF species like "Nomada spec"
                         continue;
                     }
+                    const classificationRanks = match.classificationRanks.split('|');
+                    const classificationPath = match.classificationPath.split('|').map((name, i) => ({ name, rank: classificationRanks[i] }));
                     for (const loirId of taxonNames[name]) {
                         const taxon = resource.taxa[loirId];
                         if (source === 11 && !GBIF_RANKS.includes(taxon.taxonRank)) {
@@ -410,44 +478,82 @@ class ResourceProcessor {
                             filteredResults[loirId] = [];
                         }
                         filteredResults[loirId].push({
-                            source,
+                            source: source === 11 ? ResourceTaxonIdSource.GBIF : ResourceTaxonIdSource.COL,
                             id: match.recordId,
                             currentId: match.currentRecordId,
-                            classificationPath: match.classificationPath.split('|')
+                            classificationPath
                         });
                     }
                 }
             }
-            const { taxonNames: { amendResource, groupNameMatches } } = yield Promise.resolve().then(() => __importStar(require('../index')));
-            const groupedNameMatches = groupNameMatches(filteredResults);
-            const amendedResource = Object.assign(Object.assign({}, resource), { taxa: Object.assign({}, resource.taxa) });
-            for (const source in groupedNameMatches) {
-                const matches = yield this.selectPrefixes(resource, groupedNameMatches, source);
-                amendResource(amendedResource, source, matches);
+            return filteredResults;
+        });
+    }
+    getClbNameParserResults(resource, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const filteredResults = {};
+            const result = yield runClbNameParser(resource.taxa, config.updateMappings);
+            for (const [loirId, source, results] of result) {
+                const { match, type: matchType, usage, alternatives = [] } = JSON.parse(results);
+                if (!match) {
+                    continue;
+                }
+                filteredResults[loirId] = [];
+                const matches = [usage, ...alternatives];
+                for (const match of matches) {
+                    const isSynonym = match.status !== 'accepted' && match.status !== 'provisionally accepted';
+                    const currentRank = isSynonym ? match.classification[0].rank : match.rank;
+                    const currentName = isSynonym ? match.classification[0].name : match.name;
+                    const currentId = isSynonym ? match.classification[0].id : match.id;
+                    if (matchType === 'higherrank') {
+                        // Rank mismatch
+                        continue;
+                    }
+                    if (source === ResourceTaxonIdSource.GBIF && currentRank === 'species' && currentName.endsWith(' spec')) {
+                        // GBIF species like "Nomada spec"
+                        continue;
+                    }
+                    const taxon = resource.taxa[loirId];
+                    if (source === ResourceTaxonIdSource.GBIF && !GBIF_RANKS.includes(taxon.taxonRank)) {
+                        // Exclude GBIF matches for ranks that are not in GBIF
+                        continue;
+                    }
+                    else if (source === ResourceTaxonIdSource.GBIF && !isSynonym && currentRank !== taxon.taxonRank) {
+                        // Exclude matches with rank mismatches (only possible
+                        // for non-synonyms).
+                        continue;
+                    }
+                    filteredResults[loirId].push({
+                        source,
+                        id: match.id,
+                        currentId: currentId,
+                        classificationPath: match.classification.map(({ name, rank }) => ({ name, rank })).reverse()
+                    });
+                }
             }
-            return amendedResource;
+            return filteredResults;
         });
     }
     selectPrefixes(resource, groupedNameMatches, source) {
         return __awaiter(this, void 0, void 0, function* () {
-            const prefixes = Object.keys(groupedNameMatches[source]);
+            const prefixes = Object.keys(groupedNameMatches);
             if (prefixes.length === 0) {
                 return {};
             }
             else if (prefixes.length === 1) {
-                return groupedNameMatches[source][prefixes[0]];
+                return groupedNameMatches[prefixes[0]];
             }
             // Count total mapped taxa
             const mappedTaxa = {};
             for (const prefix of prefixes) {
-                for (const taxon in groupedNameMatches[source][prefix]) {
+                for (const taxon in groupedNameMatches[prefix]) {
                     mappedTaxa[taxon] = true;
                 }
             }
-            const missedTaxonCount = Object.keys(mappedTaxa).length - Object.keys(groupedNameMatches[source][prefixes[0]]).length;
+            const missedTaxonCount = Object.keys(mappedTaxa).length - Object.keys(groupedNameMatches[prefixes[0]]).length;
             if (missedTaxonCount === 0) {
                 // Multiple prefixes but the first one maps all taxa (not counting that are unmapped in all prefixes)
-                return groupedNameMatches[source][prefixes[0]];
+                return groupedNameMatches[prefixes[0]];
             }
             console.error(`${resource.workId}: source ${source} results in multiple prefixes`);
             let choice;
@@ -455,21 +561,17 @@ class ResourceProcessor {
                 console.error(`  Most common prefix misses ${missedTaxonCount} taxa: automatically selecting most common prefix...`);
                 choice = '1';
             }
-            else if (source === '1') {
-                console.error(`  Catalogue of Life: automatically selecting most common prefix...`);
-                choice = '1';
-            }
             else {
                 for (let i = 0; i < prefixes.length; i++) {
                     const prefix = prefixes[i];
-                    const taxa = groupedNameMatches[source][prefix];
+                    const taxa = groupedNameMatches[prefix];
                     const taxonIds = Object.keys(taxa);
                     console.error(`  [${i + 1}] ${prefix} (${taxonIds.length} taxa)`);
                     for (let j = 0; j < Math.min(9, taxonIds.length); j++) {
                         const taxonId = taxonIds[j];
                         const taxon = resource.taxa[taxonId];
                         const match = taxa[taxonId];
-                        console.error(`      taxon: ${taxonId} "${taxon.scientificName}" - ${match.classificationPath.join('|')}`);
+                        console.error(`      taxon: ${taxonId} "${taxon.scientificName}" - ${formatClassificationPath(match.classificationPath)}`);
                     }
                     if (taxonIds.length > 9) {
                         console.error(`      ...`);
@@ -486,7 +588,7 @@ class ResourceProcessor {
             const matches = {};
             for (const i of choice.split(',')) {
                 const prefix = prefixes[parseInt(i) - 1];
-                const taxa = groupedNameMatches[source][prefix];
+                const taxa = groupedNameMatches[prefix];
                 for (const id in taxa) {
                     if (id in matches) {
                         continue;
@@ -540,6 +642,10 @@ function main() {
                 type: 'string',
                 short: 'u',
                 multiple: true
+            },
+            'mapping-api': {
+                type: 'string',
+                default: 'gnverifier'
             }
         },
         allowPositionals: true
@@ -563,6 +669,9 @@ function main() {
         }
         updateMappings = args.values['update-mappings'];
     }
+    if (!Object.values(ResourceTaxonIdApi).includes(args.values['mapping-api'])) {
+        throw new Error(`Unknown argument "--mapping-api ${args.values['mapping-api']}"`);
+    }
     const processor = new ResourceProcessor(args.positionals[0]);
     process.on('exit', () => {
         process.stdout.write('\n');
@@ -570,7 +679,8 @@ function main() {
     const source = args.values.source;
     const config = {
         update: source === ResourceProcessorSource.All || source === ResourceProcessorSource.Modified,
-        updateMappings
+        updateMappings,
+        mappingsApi: args.values['mapping-api']
     };
     processor.run(source, config).catch(error => {
         console.error(error);

package/lib/bin/validate-catalog.js

@@ -104,15 +104,15 @@ class GbifTaxaValidator extends TaxaValidator {
         });
     }
 }
-class ColTaxaValidator extends TaxaValidator {
-    constructor(taxa) {
-        super(taxa, {
-            id: 'col',
-            children: 'children_col',
-            ancestors: 'ancestors_col'
-        });
-    }
-}
+// class ColTaxaValidator extends TaxaValidator {
+//     constructor (taxa: catalog.Entities) {
+//         super(taxa, {
+//             id: 'col',
+//             children: 'children_col',
+//             ancestors: 'ancestors_col'
+//         })
+//     }
+// }
 function validateFile(arg) {
     return __awaiter(this, void 0, void 0, function* () {
         const filePath = path.resolve(arg);

package/lib/bin/validate-integrity.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/lib/bin/validate-integrity.js

@@ -0,0 +1,133 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = require("fs");
+const path = __importStar(require("path"));
+const index_1 = require("../index");
+function loadCatalogFile(file, base) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const contents = yield fs_1.promises.readFile(path.join(base, `${file}.csv`), 'utf8');
+        return index_1.catalog.loadData(contents, file);
+    });
+}
+function loadResourcesIndex(base) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const contents = yield fs_1.promises.readFile(path.join(base, 'resources', 'index.json'), 'utf8');
+        return JSON.parse(contents);
+    });
+}
+function getAll(entity, field) {
+    const value = entity.get(field);
+    if (Array.isArray(value)) {
+        return value;
+    }
+    else if (value) {
+        return [value];
+    }
+    else {
+        return [];
+    }
+}
+function checkIntegrity(catalog, resources, field, sheet) {
+    const errors = [];
+    const index = new Set();
+    for (const entity of sheet) {
+        for (const value of getAll(entity, 'name')) {
+            index.add(value);
+        }
+    }
+    for (const entity of catalog) {
+        const id = entity.get('id');
+        for (const value of getAll(entity, field)) {
+            if (!index.has(value)) {
+                errors.push({
+                    entity: id,
+                    field: field,
+                    error: `Value "${value}" not mapped`
+                });
+            }
+        }
+    }
+    for (const id in resources) {
+        const resource = resources[id];
+        if (!resource.catalog || !Object.prototype.hasOwnProperty.call(resource.catalog, field)) {
+            continue;
+        }
+        const value = resource.catalog[field];
+        const values = Array.isArray(value) ? value : [value];
+        for (const value of values) {
+            if (!index.has(value)) {
+                errors.push({
+                    entity: id,
+                    field: field,
+                    error: `Value "${value}" is not mapped`
+                });
+            }
+        }
+    }
+    return errors;
+}
+function main(args) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const base = path.resolve(args[0]);
+        const [catalog, authors, places, publishers, taxa, resources] = yield Promise.all([
+            loadCatalogFile('catalog', base),
+            loadCatalogFile('authors', base),
+            loadCatalogFile('places', base),
+            loadCatalogFile('publishers', base),
+            loadCatalogFile('taxa', base),
+            loadResourcesIndex(base)
+        ]);
+        const errors = [
+            ...checkIntegrity(catalog, resources, 'author', authors),
+            ...checkIntegrity(catalog, resources, 'publisher', publishers),
+            ...checkIntegrity(catalog, resources, 'taxon', taxa),
+            ...checkIntegrity(catalog, resources, 'region', places)
+        ];
+        console.table(errors);
+        process.exit(errors.length ? 1 : 0);
+    });
+}
+main(process.argv.slice(2));

package/lib/taxon-names/index.d.ts

@@ -1,2 +1,7 @@
+export declare enum ResourceTaxonIdSource {
+    COL = "col",
+    GBIF = "gbif"
+}
+export declare function formatClassificationPath(path: TaxonMatchClassificationPath): string;
 export declare function groupNameMatches(results: Record<TaxonId, TaxonMatch[]>): GroupedNameMatches;
-export declare function amendResource(resource: AmendedResource, source: string, matches: Record<TaxonId, TaxonMatch>): void;
+export declare function amendResource(resource: AmendedResource, source: ResourceTaxonIdSource, matches: Record<TaxonId, TaxonMatch>): void;

package/lib/taxon-names/index.js

@@ -1,38 +1,108 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.ResourceTaxonIdSource = void 0;
+exports.formatClassificationPath = formatClassificationPath;
 exports.groupNameMatches = groupNameMatches;
 exports.amendResource = amendResource;
-const MINIMUM_PREFIX_LENGTH = 3;
-const VALID_COMMON_PREFIXES = new Set([
-    'Plantae|Tracheophyta',
-    'Fungi',
-    'Fungi|Ascomycota',
-    'Fungi|Basidiomycota',
-    'Fungi|Zygomycota'
-]);
+const RANKS = [
+    'superdomain',
+    'domain',
+    'kingdom',
+    'subkingdom',
+    'phylum',
+    'subphylum',
+    'infraphylum',
+    'parvphylum',
+    'gigaclass',
+    'megaclass',
+    'superclass',
+    'class',
+    'subclass',
+    'infraclass',
+    'subterclass',
+    'superorder',
+    'order',
+    'suborder',
+    'infraorder',
+    'parvorder',
+    'section zoology',
+    'subsection zoology',
+    'series zoology',
+    'superfamily',
+    'epifamily',
+    'family',
+    'subfamily',
+    'tribe',
+    'subtribe',
+    'genus',
+    'subgenus',
+    'section',
+    'group',
+    'aggregate',
+    'complex',
+    'species',
+    'subspecies',
+    'variety',
+    'form'
+];
+const MINIMUM_PREFIX_RANK = 'class';
+const VALID_COMMON_PREFIXES = {
+    col: new Set([
+        'Eukaryota|Animalia|Mollusca',
+        'Eukaryota|Plantae|Bryobiotina|Bryophyta',
+        'Eukaryota|Plantae|Pteridobiotina|Tracheophyta',
+        'Eukaryota|Fungi',
+        'Eukaryota|Fungi|Ascomycota',
+        'Eukaryota|Fungi|Basidiomycota',
+        'Eukaryota|Fungi|Zygomycota'
+    ]),
+    gbif: new Set([
+        'Plantae|Tracheophyta',
+        'Fungi',
+        'Fungi|Ascomycota',
+        'Fungi|Basidiomycota',
+        'Fungi|Zygomycota'
+    ])
+};
+function compareRanks(a, b) {
+    return RANKS.indexOf(a) - RANKS.indexOf(b);
+}
 function getCommonPrefix(a, b) {
     for (let i = 0; i < Math.max(a.length, b.length); i++) {
-        if (a[i] !== b[i]) {
+        if (a[i] && b[i] && a[i].name !== b[i].name) {
             return a.slice(0, i);
         }
     }
     return a.slice();
 }
-function isValidPrefix(a, b) {
+function isValidPrefix(a, b, source) {
     const prefix = getCommonPrefix(a, b);
-    return VALID_COMMON_PREFIXES.has(prefix.join('|')) || prefix.length >= MINIMUM_PREFIX_LENGTH;
+    if (prefix.length === 0) {
+        return false;
+    }
+    if (VALID_COMMON_PREFIXES[source].has(formatClassificationPath(prefix))) {
+        return true;
+    }
+    return compareRanks(prefix[prefix.length - 1].rank, MINIMUM_PREFIX_RANK) >= 0;
+}
+var ResourceTaxonIdSource;
+(function (ResourceTaxonIdSource) {
+    ResourceTaxonIdSource["COL"] = "col";
+    ResourceTaxonIdSource["GBIF"] = "gbif";
+})(ResourceTaxonIdSource || (exports.ResourceTaxonIdSource = ResourceTaxonIdSource = {}));
+function formatClassificationPath(path) {
+    return path.map(part => part.name).join('|');
 }
 function groupNameMatches(results) {
+    var _a;
     const prefixes = {};
     for (const scientificNameID in results) {
         for (const result of results[scientificNameID]) {
-            if (!prefixes[result.source]) {
-                prefixes[result.source] = [];
-            }
-            let prefix = prefixes[result.source].find(prefix => isValidPrefix(prefix[0], result.classificationPath));
+            const matchGroups = (_a = prefixes[result.source]) !== null && _a !== void 0 ? _a : (prefixes[result.source] = []);
+            let prefix = matchGroups.find(prefix => isValidPrefix(prefix[0], result.classificationPath, result.source));
             if (!prefix) {
                 prefix = [result.classificationPath, {}];
-                prefixes[result.source].push(prefix);
+                matchGroups.push(prefix);
             }
             else {
                 prefix[0] = getCommonPrefix(prefix[0], result.classificationPath);
@@ -44,11 +114,11 @@ function groupNameMatches(results) {
         }
     }
     const groupedNameMatches = {};
-    for (const source in prefixes) {
-        groupedNameMatches[source] = prefixes[source]
+    for (const [source, matchGroups] of Object.entries(prefixes)) {
+        groupedNameMatches[source] = matchGroups
             .sort((a, b) => Object.keys(b[1]).length - Object.keys(a[1]).length)
             .reduce((map, [prefix, taxa]) => {
-            map[prefix.join('|')] = taxa;
+            map[formatClassificationPath(prefix)] = taxa;
             return map;
         }, {});
     }
@@ -57,13 +127,13 @@ function groupNameMatches(results) {
 function amendResource(resource, source, matches) {
     for (const id in matches) {
         const match = matches[id];
-        if (source === '1' || source === '13') {
+        if (source === ResourceTaxonIdSource.COL) {
             resource.taxa[id].colTaxonID = match.id;
             if (match.currentId) {
                 resource.taxa[id].colAcceptedTaxonID = match.currentId;
             }
         }
-        else if (source === '11') {
+        else if (source === ResourceTaxonIdSource.GBIF) {
             resource.taxa[id].gbifTaxonID = match.id;
             if (match.currentId) {
                 resource.taxa[id].gbifAcceptedTaxonID = match.currentId;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larsgw/formica",
-  "version": "0.10.1",
+  "version": "0.10.3",
   "description": "SDK and tools for data from the Library of Identification Resources",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
@@ -10,8 +10,10 @@
   "bin": {
     "loir-validate-catalog": "lib/bin/validate-catalog.js",
     "loir-validate-resources": "lib/bin/validate-resources-text.js",
+    "loir-validate-integrity": "lib/bin/validate-integrity.js",
     "loir-resources-process": "lib/bin/process-resources.js",
     "loir-resources-index": "lib/bin/process-resources-index.js",
+    "loir-resources-check-diff": "lib/bin/check-resources-diff.js",
     "loir-generate-linked-data": "lib/bin/generate-linked-data.js"
   },
   "scripts": {