@larsgw/formica 0.2.0 → 0.3.0

package/lib/resources/parse-text.js

@@ -43,6 +43,15 @@ var RANKS = [
     'race',
     'stirps' // not ICZN
 ];
+var MAIN_RANKS = [
+    'kingdom',
+    'phylum',
+    'class',
+    'order',
+    'family',
+    'genus',
+    'species'
+];
 var DWC_RANKS = [
     'kingdom',
     'phylum',
@@ -350,10 +359,7 @@ function parseHeader(header) {
     }
     return metadata;
 }
-function parseResource(resource) {
-    var _a = resource.split(/(\n---\n+)/), header = _a[0], _ = _a[1], rest = _a.slice(2);
-    var config = parseHeader(header);
-    var content = rest.join('');
+function validateResource(config, content) {
     // Check for too much indentation
     var longerIndent = new RegExp("^(  ){".concat(config.levels.length - 1, "}(?!  [+=>] ) "), 'm');
     var longerIndentMatch = content.match(longerIndent);
@@ -363,13 +369,22 @@ function parseResource(resource) {
         throw new SyntaxError("Too much indentation at ".concat(line, ":0\n").concat(content.slice(offset).split('\n', 1), "\n^"));
     }
     // Check for missing leaf taxa
-    var missingLeafTaxa = new RegExp("^(  ){0,".concat(config.levels.length - 2, "}(?![+=>] ).*\\n(?!\\1  )"), 'm');
-    var missingLeafTaxaMatch = content.match(missingLeafTaxa);
-    if (missingLeafTaxaMatch !== null) {
-        var offset = missingLeafTaxaMatch.index;
-        var line = (content.slice(0, offset).match(/\n/g) || []).length + 1;
-        throw new SyntaxError("Missing leaf taxon at ".concat(line, ":0\n").concat(content.slice(offset).split('\n', 1), "\n^"));
+    var leafTaxonRank = config.levels.filter(function (rank) { return MAIN_RANKS.includes(rank); }).pop();
+    var leafTaxonParentIndent = config.levels.indexOf(leafTaxonRank) - 1;
+    if (leafTaxonRank && leafTaxonParentIndent >= 0) {
+        var missingLeafTaxa = new RegExp("^((?:  ){0,".concat(leafTaxonParentIndent, "})(?![+=> ] ).*\\n(\\1(  )+[+=>].*\\n)*(?!\\1  )"), 'm');
+        var missingLeafTaxaMatch = content.match(missingLeafTaxa);
+        if (missingLeafTaxaMatch !== null) {
+            var offset = missingLeafTaxaMatch.index;
+            var line = (content.slice(0, offset).match(/\n/g) || []).length + 1;
+            throw new SyntaxError("Missing leaf taxon at ".concat(line, ":0\n").concat(content.slice(offset).split('\n', 1), "\n^"));
+        }
     }
+}
+function parseResource(resource) {
+    var _a = resource.split(/(\n---\n+)/), header = _a[0], _ = _a[1], rest = _a.slice(2);
+    var config = parseHeader(header);
+    var content = rest.join('');
     return [config, content];
 }
 function parseResourceContent(content, resource, oldIds) {
@@ -476,6 +491,7 @@ function parseFile(file, id, old) {
     var oldResources = old ? splitResources(old.txt) : [];
     return splitResources(file).map(function (resource, index) {
         var _a = parseResource(resource), config = _a[0], content = _a[1];
+        validateResource(config, content);
         var template = {
             id: "".concat(id, ":").concat(index + 1),
             file: "".concat(id, "-").concat(index + 1),

package/lib/taxon-names/index.d.ts

@@ -0,0 +1,2 @@
+export declare function groupNameMatches(results: Record<TaxonId, TaxonMatch[]>): GroupedNameMatches;
+export declare function amendResource(resource: AmendedResource, source: string, matches: Record<TaxonId, TaxonMatch>): void;

package/lib/taxon-names/index.js

@@ -0,0 +1,79 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.amendResource = exports.groupNameMatches = void 0;
+var MINIMUM_PREFIX_LENGTH = 3;
+var VALID_COMMON_PREFIXES = new Set([
+    'Plantae|Tracheophyta',
+    'Fungi',
+    'Fungi|Ascomycota',
+    'Fungi|Basidiomycota',
+    'Fungi|Zygomycota'
+]);
+function getCommonPrefix(a, b) {
+    for (var i = 0; i < Math.max(a.length, b.length); i++) {
+        if (a[i] !== b[i]) {
+            return a.slice(0, i);
+        }
+    }
+    return a.slice();
+}
+function isValidPrefix(a, b) {
+    var prefix = getCommonPrefix(a, b);
+    return VALID_COMMON_PREFIXES.has(prefix.join('|')) || prefix.length >= MINIMUM_PREFIX_LENGTH;
+}
+function groupNameMatches(results) {
+    var prefixes = {};
+    for (var scientificNameID in results) {
+        var _loop_1 = function (result) {
+            if (!prefixes[result.source]) {
+                prefixes[result.source] = [];
+            }
+            var prefix = prefixes[result.source].find(function (prefix) { return isValidPrefix(prefix[0], result.classificationPath); });
+            if (!prefix) {
+                prefix = [result.classificationPath, {}];
+                prefixes[result.source].push(prefix);
+            }
+            else {
+                prefix[0] = getCommonPrefix(prefix[0], result.classificationPath);
+            }
+            if (scientificNameID in prefix[1]) {
+                return "continue";
+            }
+            prefix[1][scientificNameID] = result;
+        };
+        for (var _i = 0, _a = results[scientificNameID]; _i < _a.length; _i++) {
+            var result = _a[_i];
+            _loop_1(result);
+        }
+    }
+    var groupedNameMatches = {};
+    for (var source in prefixes) {
+        groupedNameMatches[source] = prefixes[source]
+            .sort(function (a, b) { return Object.keys(b[1]).length - Object.keys(a[1]).length; })
+            .reduce(function (map, _a) {
+            var prefix = _a[0], taxa = _a[1];
+            map[prefix.join('|')] = taxa;
+            return map;
+        }, {});
+    }
+    return groupedNameMatches;
+}
+exports.groupNameMatches = groupNameMatches;
+function amendResource(resource, source, matches) {
+    for (var id in matches) {
+        var match = matches[id];
+        if (source === '1') {
+            resource.taxa[id].colTaxonID = match.id;
+            if (match.currentId) {
+                resource.taxa[id].colAcceptedTaxonID = match.currentId;
+            }
+        }
+        else if (source === '11') {
+            resource.taxa[id].gbifTaxonID = match.id;
+            if (match.currentId) {
+                resource.taxa[id].gbifAcceptedTaxonID = match.currentId;
+            }
+        }
+    }
+}
+exports.amendResource = amendResource;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larsgw/formica",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "SDK and tools for data from the Library of Identification Resources",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
@@ -14,7 +14,9 @@
     "test": "node --test --test-reporter spec",
     "lint": "eslint src",
     "build": "tsc -d",
+    "changelog": "conventional-changelog -p angular -i CHANGELOG.md -s -r 0",
     "preversion": "npm run lint",
+    "version": "npm run changelog && git add CHANGELOG.md",
     "prepublishOnly": "npm run build"
   },
   "repository": {
@@ -37,6 +39,7 @@
     "@types/node": "^18.14.1",
     "@typescript-eslint/eslint-plugin": "^5.54.0",
     "@typescript-eslint/parser": "^5.54.0",
+    "conventional-changelog-cli": "^3.0.0",
     "eslint": "^8.35.0",
     "typescript": "^4.9.5"
   }

package/src/bin/process-resources-index.ts

@@ -21,6 +21,14 @@ function sortObject (object: Record<string, any>): Record<string, any> {
 }
 /* eslint-enable @typescript-eslint/no-explicit-any */
 
+function addTaxon (gbifIndex: Record<string, TaxonId[]>, gbifId: string, taxon: string[]) {
+    if (!(gbifId in gbifIndex)) {
+        gbifIndex[gbifId] = []
+    }
+    gbifIndex[gbifId].push(taxon[0])
+    gbifIndex[gbifId].sort(numericSort)
+}
+
 async function main (args: string[]): Promise<void> {
     const REPO_ROOT = path.resolve(args[0])
 
@@ -50,11 +58,10 @@ async function main (args: string[]): Promise<void> {
             for (const taxon of dwc) {
                 const gbifId = taxon[25]
                 if (gbifId) {
-                    if (!(gbifId in gbifIndex)) {
-                        gbifIndex[gbifId] = []
+                    addTaxon(gbifIndex, gbifId, taxon)
+                    if (taxon[27] !== taxon[25]) {
+                        addTaxon(gbifIndex, taxon[27], taxon)
                     }
-                    gbifIndex[gbifId].push(taxon[0])
-                    gbifIndex[gbifId].sort(numericSort)
                 }
                 amendedResource.taxonCount += 1
             }

package/src/bin/process-resources.ts

@@ -8,17 +8,6 @@ import * as util from 'util'
 import { csv } from '../index'
 import { prompt, promptForAnswers, numericSort, runCommand } from './util'
 
-interface AmendedTaxon extends Taxon {
-    colTaxonID?: string,
-    gbifTaxonID?: string
-}
-
-interface AmendedResource extends Resource {
-    taxa: Record<TaxonId, AmendedTaxon>
-}
-
-type Classifications = Record<string, Array<[AmendedTaxon, string]>>
-
 const DWC_FIELDS: string[] = [
     'scientificNameID',
     'scientificName',
@@ -49,7 +38,9 @@ const DWC_FIELDS: string[] = [
     'higherClassification',
 
     'colTaxonID',
-    'gbifTaxonID'
+    'gbifTaxonID',
+    'colAcceptedTaxonID',
+    'gbifAcceptedTaxonID'
 ]
 
 const DISPLAY_FIELDS: string[] = [
@@ -74,17 +65,9 @@ const GBIF_RANKS: Rank[] = [
     'variety'
 ]
 
-const VALID_COMMON_PREFIXES = [
-    'Plantae|Tracheophyta',
-    'Fungi',
-    'Fungi|Ascomycota',
-    'Fungi|Basidiomycota',
-    'Fungi|Zygomycota'
-]
-
 function runGnverifier (names: string): Promise<string> {
     return new Promise((resolve, reject) => {
-        const proc = spawn('gnverifier', ['-s', '1,11', '-M'])
+        const proc = spawn('gnverifier', ['-s', '1,11', '-f', 'compact', '-M'])
         let stdout = ''
         proc.stdout.on('data', data => { stdout += data })
         proc.stderr.pipe(process.stdout)
@@ -153,6 +136,18 @@ class ResourceProcessor {
         }
     }
 
+    async runMappingsUpdate (): Promise<void> {
+        const input = await fs.readdir(this.DIR_TXT)
+
+        const ids = input
+            .map(file => path.basename(file, '.txt'))
+            .sort((a, b) => parseInt(a.slice(1)) - parseInt(b.slice(1)))
+
+        for (const id of ids) {
+            await this.processWork(id, true)
+        }
+    }
+
     async processWork (id: WorkId, update?: boolean): Promise<void> {
         const resources = await this.processResources(id, update)
 
@@ -174,11 +169,7 @@ class ResourceProcessor {
 
         const amendedResources = []
         for (const resource of resources) {
-            const [results, classifications] = await this.processResourceDwc(resource)
-
-            for (const source in classifications) {
-                await this.checkPrefix(resource, classifications, source)
-            }
+            const results = await this.processResourceDwc(resource)
 
             const skip = await this.shouldBeSkipped(resource.id)
 
@@ -256,100 +247,153 @@ class ResourceProcessor {
         }
     }
 
-    async processResourceDwc (resource: Resource): Promise<[AmendedResource, Classifications]> {
+    async processResourceDwc (resource: Resource): Promise<AmendedResource> {
         console.log(`${resource.workId}: matching ${resource.id}`)
-        const taxa: Record<string, Record<TaxonId, AmendedTaxon>> = {}
-        const names = []
+
+        const filteredResults: Record<TaxonId, TaxonMatch[]> = {}
+        const taxonNames: Record<string, TaxonId[]> = {}
+        const names = new Set()
         for (const id in resource.taxa) {
             const name = resource.taxa[id].scientificName
 
-            if (!taxa[name]) { taxa[name] = {} }
-            taxa[name][id] = { ...resource.taxa[id] }
+            if (!taxonNames[name]) { taxonNames[name] = [] }
+            taxonNames[name].push(id)
 
-            names.push(name)
+            names.add(name)
+            filteredResults[id] = []
         }
 
-        const result = await runGnverifier(names.join('\n'))
-        const classifications: Classifications = { '1': [], '11': [] }
-
-        const [header, ...matches] = csv.parseCsv(result)
-        for (const match of matches) {
-            const name = match[header.indexOf('ScientificName')]
-            const source = match[header.indexOf('DataSourceId')]
-            const id = match[header.indexOf('TaxonId')]
-            const classification = match[header.indexOf('ClassificationPath')]
-
-            for (const loirId in taxa[name]) {
-                const taxon = taxa[name][loirId]
-                if (source === '1' && !taxon.colTaxonID) {
-                    taxon.colTaxonID = id
-                    classifications[source].push([taxon, classification])
+        const result = await runGnverifier(Array.from(names).join('\n'))
+        for (const results of result.trim().split('\n')) {
+            const { name, results: matches } = JSON.parse(results)
+
+            if (!matches) {
+                continue
+            }
+
+            for (const match of matches) {
+                const source = match.dataSourceId
+                const currentRank = match.classificationRanks.split('|').pop()
+
+                if (match.scoreDetails.cardinalityScore === 0) {
+                    // Rank mismatch
+                    continue
+                } else if (source === 11 && currentRank === 'species' && match.classificationPath.endsWith(' spec')) {
+                    // GBIF species like "Nomada spec"
+                    continue
                 }
-                if (source === '11' && GBIF_RANKS.includes(taxon.taxonRank) && !taxon.gbifTaxonID) {
-                    taxon.gbifTaxonID = id
-                    classifications[source].push([taxon, classification])
+
+                for (const loirId of taxonNames[name]) {
+                    const taxon = resource.taxa[loirId]
+
+                    if (source === 11 && !GBIF_RANKS.includes(taxon.taxonRank)) {
+                        // Exclude GBIF matches for ranks that are not in GBIF
+                        continue
+                    } else if (source === 11 && !match.isSynonym && currentRank !== taxon.taxonRank) {
+                        // Exclude matches with rank mismatches (only possible
+                        // for non-synonyms).
+                        continue
+                    }
+
+                    if (!filteredResults[loirId]) {
+                        filteredResults[loirId] = []
+                    }
+
+                    filteredResults[loirId].push({
+                        source,
+                        id: match.recordId,
+                        currentId: match.currentRecordId,
+                        classificationPath: match.classificationPath.split('|')
+                    })
                 }
             }
         }
 
-        const results = {
-            ...resource,
-            taxa: Object.fromEntries(Object.values(resource.taxa).map(taxon => [
-                taxon.scientificNameID,
-                taxa[taxon.scientificName][taxon.scientificNameID]
-            ]))
+        const { taxonNames: { amendResource, groupNameMatches } } = await import('../index')
+        const groupedNameMatches = groupNameMatches(filteredResults)
+
+        const amendedResource: AmendedResource = { ...resource, taxa: { ...resource.taxa } }
+        for (const source in groupedNameMatches) {
+            const matches = await this.selectPrefixes(resource, groupedNameMatches, source)
+            amendResource(amendedResource, source, matches)
         }
 
-        return [results, classifications]
+        return amendedResource
     }
 
-    async checkPrefix (resource: AmendedResource, classifications: Classifications, source: string): Promise<void> {
-        const lists = classifications[source]
-        if (!lists.length) { return }
-        const prefix = lists[0][1].split('|')
-
-        for (const [taxon, list] of lists.slice(1)) {
-            const parts = list.split('|')
-            for (let i = 0; i < parts.length; i++) {
-                if (parts[i] !== prefix[i] && i < 3) {
-                    let choice
-
-                    if (source === '1' || taxon.taxonomicStatus !== 'accepted') {
-                        choice = 'd'
-                    } else if (VALID_COMMON_PREFIXES.includes(prefix.slice(0, i).join('|'))) {
-                        choice = 'k'
-                    } else {
-                        console.log(`${resource.workId}: source ${source} results in short prefix "${prefix.slice(0, i).join('|')}" (${i} taxa)`)
-                        console.log(`  taxon: ${taxon.scientificNameID} "${taxon.scientificName}"`)
-                        console.log(`  class: ${parts.join('|')}`)
-                        console.log(`  prefx: ${prefix.join('|')}`)
-
-                        choice = await promptForAnswers(`  Keep or delete (k/d)? `, ['k', 'K', 'd', 'D'])
-                    }
+    async selectPrefixes (resource: Resource, groupedNameMatches: GroupedNameMatches, source: string): Promise<Record<TaxonId, TaxonMatch>> {
+        const prefixes = Object.keys(groupedNameMatches[source])
+        if (prefixes.length === 0) {
+            return {}
+        } else if (prefixes.length === 1) {
+            return groupedNameMatches[source][prefixes[0]]
+        }
 
-                    switch (choice) {
-                        case 'k':
-                        case 'K': {
-                            console.log(`  keeping...`)
-                            break
-                        }
+        // Count total mapped taxa
+        const mappedTaxa: Record<TaxonId, boolean> = {}
+        for (const prefix of prefixes) {
+            for (const taxon in groupedNameMatches[source][prefix]) {
+                mappedTaxa[taxon] = true
+            }
+        }
+        const missedTaxonCount = Object.keys(mappedTaxa).length - Object.keys(groupedNameMatches[source][prefixes[0]]).length
 
-                        case 'd':
-                        case 'D': {
-                            console.log(`  deleting...`)
-                            if (source === '1') {
-                                delete taxon.colTaxonID
-                            } else if (source === '11') {
-                                delete taxon.gbifTaxonID
-                            }
-                            break
-                        }
-                    }
+        if (missedTaxonCount === 0) {
+            // Multiple prefixes but the first one maps all taxa (not counting that are unmapped in all prefixes)
+            return groupedNameMatches[source][prefixes[0]]
+        }
+
+        console.error(`${resource.workId}: source ${source} results in multiple prefixes`)
+
+        let choice
+        if (missedTaxonCount <= 5) {
+            console.error(`  Most common prefix misses ${missedTaxonCount} taxa: automatically selecting most common prefix...`)
+            choice = '1'
+        } else if (source === '1') {
+            console.error(`  Catalogue of Life: automatically selecting most common prefix...`)
+            choice = '1'
+        } else {
+            for (let i = 0; i < prefixes.length; i++) {
+                const prefix = prefixes[i]
+                const taxa = groupedNameMatches[source][prefix]
+                const taxonIds = Object.keys(taxa)
+
+                console.error(`  [${i + 1}] ${prefix} (${taxonIds.length} taxa)`)
+                for (let j = 0; j < Math.min(9, taxonIds.length); j++) {
+                    const taxonId = taxonIds[j]
+                    const taxon = resource.taxa[taxonId]
+                    const match = taxa[taxonId]
+                    console.error(`      taxon: ${taxonId} "${taxon.scientificName}" - ${match.classificationPath.join('|')}`)
+                }
+                if (taxonIds.length > 9) {
+                    console.error(`      ...`)
+                }
+            }
+
+            do {
+                choice = await prompt(`  Select prefixes (1-${prefixes.length})? `)
+            } while (!/^(|\d+(,\d+)*)$/.test(choice))
+        }
+
+        console.error(`  Applying selection...`)
+
+        if (choice === '') {
+            return {}
+        }
 
-                    break
+        const matches: Record<TaxonId, TaxonMatch> = {}
+        for (const i of choice.split(',')) {
+            const prefix = prefixes[parseInt(i) - 1]
+            const taxa = groupedNameMatches[source][prefix]
+            for (const id in taxa) {
+                if (id in matches) {
+                    continue
                 }
+                matches[id] = taxa[id]
             }
         }
+
+        return matches
     }
 
     checkResults (resource: AmendedResource): boolean {
@@ -388,6 +432,9 @@ function main (): void {
             update: {
                 type: 'boolean',
                 short: 'u'
+            },
+            'update-mappings': {
+                type: 'boolean'
             }
         },
         allowPositionals: true
@@ -398,7 +445,15 @@ function main (): void {
         process.stdout.write('\n')
     })
 
-    const task = args.values.update ? processor.runUpdate() : processor.run()
+    let task
+    if (args.values.update) {
+        task = processor.runUpdate()
+    } else if (args.values['update-mappings']) {
+        task = processor.runMappingsUpdate()
+    } else {
+        task = processor.run()
+    }
+
     task.catch(error => {
         console.error(error)
         process.exit(1)

package/src/index.ts

@@ -1,3 +1,4 @@
 export * as catalog from './catalog/index'
 export * as resources from './resources/index'
+export * as taxonNames from './taxon-names/index'
 export * as csv from './csv'