@larsgw/formica 0.2.1 → 0.3.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larsgw/formica",
-  "version": "0.2.1",
+  "version": "0.3.1",
   "description": "SDK and tools for data from the Library of Identification Resources",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",

package/src/bin/process-resources-index.ts

@@ -21,6 +21,14 @@ function sortObject (object: Record<string, any>): Record<string, any> {
 }
 /* eslint-enable @typescript-eslint/no-explicit-any */
 
+function addTaxon (gbifIndex: Record<string, TaxonId[]>, gbifId: string, taxon: string[]) {
+    if (!(gbifId in gbifIndex)) {
+        gbifIndex[gbifId] = []
+    }
+    gbifIndex[gbifId].push(taxon[0])
+    gbifIndex[gbifId].sort(numericSort)
+}
+
 async function main (args: string[]): Promise<void> {
     const REPO_ROOT = path.resolve(args[0])
 
@@ -50,11 +58,10 @@ async function main (args: string[]): Promise<void> {
             for (const taxon of dwc) {
                 const gbifId = taxon[25]
                 if (gbifId) {
-                    if (!(gbifId in gbifIndex)) {
-                        gbifIndex[gbifId] = []
+                    addTaxon(gbifIndex, gbifId, taxon)
+                    if (taxon[27] !== taxon[25]) {
+                        addTaxon(gbifIndex, taxon[27], taxon)
                     }
-                    gbifIndex[gbifId].push(taxon[0])
-                    gbifIndex[gbifId].sort(numericSort)
                 }
                 amendedResource.taxonCount += 1
             }

package/src/bin/process-resources.ts

@@ -8,17 +8,6 @@ import * as util from 'util'
 import { csv } from '../index'
 import { prompt, promptForAnswers, numericSort, runCommand } from './util'
 
-interface AmendedTaxon extends Taxon {
-    colTaxonID?: string,
-    gbifTaxonID?: string
-}
-
-interface AmendedResource extends Resource {
-    taxa: Record<TaxonId, AmendedTaxon>
-}
-
-type Classifications = Record<string, Array<[AmendedTaxon, string]>>
-
 const DWC_FIELDS: string[] = [
     'scientificNameID',
     'scientificName',
@@ -49,7 +38,9 @@ const DWC_FIELDS: string[] = [
     'higherClassification',
 
     'colTaxonID',
-    'gbifTaxonID'
+    'gbifTaxonID',
+    'colAcceptedTaxonID',
+    'gbifAcceptedTaxonID'
 ]
 
 const DISPLAY_FIELDS: string[] = [
@@ -74,17 +65,9 @@ const GBIF_RANKS: Rank[] = [
     'variety'
 ]
 
-const VALID_COMMON_PREFIXES = [
-    'Plantae|Tracheophyta',
-    'Fungi',
-    'Fungi|Ascomycota',
-    'Fungi|Basidiomycota',
-    'Fungi|Zygomycota'
-]
-
 function runGnverifier (names: string): Promise<string> {
     return new Promise((resolve, reject) => {
-        const proc = spawn('gnverifier', ['-s', '1,11', '-M'])
+        const proc = spawn('gnverifier', ['-s', '1,11', '-f', 'compact', '-M'])
         let stdout = ''
         proc.stdout.on('data', data => { stdout += data })
         proc.stderr.pipe(process.stdout)
@@ -153,6 +136,18 @@ class ResourceProcessor {
         }
     }
 
+    async runMappingsUpdate (): Promise<void> {
+        const input = await fs.readdir(this.DIR_TXT)
+
+        const ids = input
+            .map(file => path.basename(file, '.txt'))
+            .sort((a, b) => parseInt(a.slice(1)) - parseInt(b.slice(1)))
+
+        for (const id of ids) {
+            await this.processWork(id, true)
+        }
+    }
+
     async processWork (id: WorkId, update?: boolean): Promise<void> {
         const resources = await this.processResources(id, update)
 
@@ -174,11 +169,7 @@ class ResourceProcessor {
 
         const amendedResources = []
         for (const resource of resources) {
-            const [results, classifications] = await this.processResourceDwc(resource)
-
-            for (const source in classifications) {
-                await this.checkPrefix(resource, classifications, source)
-            }
+            const results = await this.processResourceDwc(resource)
 
             const skip = await this.shouldBeSkipped(resource.id)
 
@@ -256,100 +247,153 @@ class ResourceProcessor {
         }
     }
 
-    async processResourceDwc (resource: Resource): Promise<[AmendedResource, Classifications]> {
+    async processResourceDwc (resource: Resource): Promise<AmendedResource> {
         console.log(`${resource.workId}: matching ${resource.id}`)
-        const taxa: Record<string, Record<TaxonId, AmendedTaxon>> = {}
-        const names = []
+
+        const filteredResults: Record<TaxonId, TaxonMatch[]> = {}
+        const taxonNames: Record<string, TaxonId[]> = {}
+        const names = new Set()
         for (const id in resource.taxa) {
             const name = resource.taxa[id].scientificName
 
-            if (!taxa[name]) { taxa[name] = {} }
-            taxa[name][id] = { ...resource.taxa[id] }
+            if (!taxonNames[name]) { taxonNames[name] = [] }
+            taxonNames[name].push(id)
 
-            names.push(name)
+            names.add(name)
+            filteredResults[id] = []
         }
 
-        const result = await runGnverifier(names.join('\n'))
-        const classifications: Classifications = { '1': [], '11': [] }
-
-        const [header, ...matches] = csv.parseCsv(result)
-        for (const match of matches) {
-            const name = match[header.indexOf('ScientificName')]
-            const source = match[header.indexOf('DataSourceId')]
-            const id = match[header.indexOf('TaxonId')]
-            const classification = match[header.indexOf('ClassificationPath')]
-
-            for (const loirId in taxa[name]) {
-                const taxon = taxa[name][loirId]
-                if (source === '1' && !taxon.colTaxonID) {
-                    taxon.colTaxonID = id
-                    classifications[source].push([taxon, classification])
+        const result = await runGnverifier(Array.from(names).join('\n'))
+        for (const results of result.trim().split('\n')) {
+            const { name, results: matches } = JSON.parse(results)
+
+            if (!matches) {
+                continue
+            }
+
+            for (const match of matches) {
+                const source = match.dataSourceId
+                const currentRank = match.classificationRanks.split('|').pop()
+
+                if (match.scoreDetails.cardinalityScore === 0) {
+                    // Rank mismatch
+                    continue
+                } else if (source === 11 && currentRank === 'species' && match.classificationPath.endsWith(' spec')) {
+                    // GBIF species like "Nomada spec"
+                    continue
                 }
-                if (source === '11' && GBIF_RANKS.includes(taxon.taxonRank) && !taxon.gbifTaxonID) {
-                    taxon.gbifTaxonID = id
-                    classifications[source].push([taxon, classification])
+
+                for (const loirId of taxonNames[name]) {
+                    const taxon = resource.taxa[loirId]
+
+                    if (source === 11 && !GBIF_RANKS.includes(taxon.taxonRank)) {
+                        // Exclude GBIF matches for ranks that are not in GBIF
+                        continue
+                    } else if (source === 11 && !match.isSynonym && currentRank !== taxon.taxonRank) {
+                        // Exclude matches with rank mismatches (only possible
+                        // for non-synonyms).
+                        continue
+                    }
+
+                    if (!filteredResults[loirId]) {
+                        filteredResults[loirId] = []
+                    }
+
+                    filteredResults[loirId].push({
+                        source,
+                        id: match.recordId,
+                        currentId: match.currentRecordId,
+                        classificationPath: match.classificationPath.split('|')
+                    })
                 }
             }
         }
 
-        const results = {
-            ...resource,
-            taxa: Object.fromEntries(Object.values(resource.taxa).map(taxon => [
-                taxon.scientificNameID,
-                taxa[taxon.scientificName][taxon.scientificNameID]
-            ]))
+        const { taxonNames: { amendResource, groupNameMatches } } = await import('../index')
+        const groupedNameMatches = groupNameMatches(filteredResults)
+
+        const amendedResource: AmendedResource = { ...resource, taxa: { ...resource.taxa } }
+        for (const source in groupedNameMatches) {
+            const matches = await this.selectPrefixes(resource, groupedNameMatches, source)
+            amendResource(amendedResource, source, matches)
         }
 
-        return [results, classifications]
+        return amendedResource
     }
 
-    async checkPrefix (resource: AmendedResource, classifications: Classifications, source: string): Promise<void> {
-        const lists = classifications[source]
-        if (!lists.length) { return }
-        const prefix = lists[0][1].split('|')
-
-        for (const [taxon, list] of lists.slice(1)) {
-            const parts = list.split('|')
-            for (let i = 0; i < parts.length; i++) {
-                if (parts[i] !== prefix[i] && i < 3) {
-                    let choice
-
-                    if (source === '1' || taxon.taxonomicStatus !== 'accepted') {
-                        choice = 'd'
-                    } else if (VALID_COMMON_PREFIXES.includes(prefix.slice(0, i).join('|'))) {
-                        choice = 'k'
-                    } else {
-                        console.log(`${resource.workId}: source ${source} results in short prefix "${prefix.slice(0, i).join('|')}" (${i} taxa)`)
-                        console.log(`  taxon: ${taxon.scientificNameID} "${taxon.scientificName}"`)
-                        console.log(`  class: ${parts.join('|')}`)
-                        console.log(`  prefx: ${prefix.join('|')}`)
-
-                        choice = await promptForAnswers(`  Keep or delete (k/d)? `, ['k', 'K', 'd', 'D'])
-                    }
+    async selectPrefixes (resource: Resource, groupedNameMatches: GroupedNameMatches, source: string): Promise<Record<TaxonId, TaxonMatch>> {
+        const prefixes = Object.keys(groupedNameMatches[source])
+        if (prefixes.length === 0) {
+            return {}
+        } else if (prefixes.length === 1) {
+            return groupedNameMatches[source][prefixes[0]]
+        }
 
-                    switch (choice) {
-                        case 'k':
-                        case 'K': {
-                            console.log(`  keeping...`)
-                            break
-                        }
+        // Count total mapped taxa
+        const mappedTaxa: Record<TaxonId, boolean> = {}
+        for (const prefix of prefixes) {
+            for (const taxon in groupedNameMatches[source][prefix]) {
+                mappedTaxa[taxon] = true
+            }
+        }
+        const missedTaxonCount = Object.keys(mappedTaxa).length - Object.keys(groupedNameMatches[source][prefixes[0]]).length
 
-                        case 'd':
-                        case 'D': {
-                            console.log(`  deleting...`)
-                            if (source === '1') {
-                                delete taxon.colTaxonID
-                            } else if (source === '11') {
-                                delete taxon.gbifTaxonID
-                            }
-                            break
-                        }
-                    }
+        if (missedTaxonCount === 0) {
+            // Multiple prefixes but the first one maps all taxa (not counting that are unmapped in all prefixes)
+            return groupedNameMatches[source][prefixes[0]]
+        }
+
+        console.error(`${resource.workId}: source ${source} results in multiple prefixes`)
+
+        let choice
+        if (missedTaxonCount <= 5) {
+            console.error(`  Most common prefix misses ${missedTaxonCount} taxa: automatically selecting most common prefix...`)
+            choice = '1'
+        } else if (source === '1') {
+            console.error(`  Catalogue of Life: automatically selecting most common prefix...`)
+            choice = '1'
+        } else {
+            for (let i = 0; i < prefixes.length; i++) {
+                const prefix = prefixes[i]
+                const taxa = groupedNameMatches[source][prefix]
+                const taxonIds = Object.keys(taxa)
+
+                console.error(`  [${i + 1}] ${prefix} (${taxonIds.length} taxa)`)
+                for (let j = 0; j < Math.min(9, taxonIds.length); j++) {
+                    const taxonId = taxonIds[j]
+                    const taxon = resource.taxa[taxonId]
+                    const match = taxa[taxonId]
+                    console.error(`      taxon: ${taxonId} "${taxon.scientificName}" - ${match.classificationPath.join('|')}`)
+                }
+                if (taxonIds.length > 9) {
+                    console.error(`      ...`)
+                }
+            }
+
+            do {
+                choice = await prompt(`  Select prefixes (1-${prefixes.length})? `)
+            } while (!/^(|\d+(,\d+)*)$/.test(choice))
+        }
+
+        console.error(`  Applying selection...`)
+
+        if (choice === '') {
+            return {}
+        }
 
-                    break
+        const matches: Record<TaxonId, TaxonMatch> = {}
+        for (const i of choice.split(',')) {
+            const prefix = prefixes[parseInt(i) - 1]
+            const taxa = groupedNameMatches[source][prefix]
+            for (const id in taxa) {
+                if (id in matches) {
+                    continue
                 }
+                matches[id] = taxa[id]
             }
         }
+
+        return matches
     }
 
     checkResults (resource: AmendedResource): boolean {
@@ -388,6 +432,9 @@ function main (): void {
             update: {
                 type: 'boolean',
                 short: 'u'
+            },
+            'update-mappings': {
+                type: 'boolean'
             }
         },
         allowPositionals: true
@@ -398,7 +445,15 @@ function main (): void {
         process.stdout.write('\n')
     })
 
-    const task = args.values.update ? processor.runUpdate() : processor.run()
+    let task
+    if (args.values.update) {
+        task = processor.runUpdate()
+    } else if (args.values['update-mappings']) {
+        task = processor.runMappingsUpdate()
+    } else {
+        task = processor.run()
+    }
+
     task.catch(error => {
         console.error(error)
         process.exit(1)

package/src/index.ts

@@ -1,3 +1,4 @@
 export * as catalog from './catalog/index'
 export * as resources from './resources/index'
+export * as taxonNames from './taxon-names/index'
 export * as csv from './csv'

package/src/module.d.ts

@@ -1,8 +1,8 @@
 type Schema = Record<string, FieldSpecification>
 type FieldSpecification = {
-  required: boolean,
-  multiple: boolean | FieldSpecificationCallbackMultiple
-  format?: string[] | RegExp | FieldSpecificationCallbackFormat
+    required: boolean,
+    multiple: boolean | FieldSpecificationCallbackMultiple
+    format?: string[] | RegExp | FieldSpecificationCallbackFormat
 }
 
 type Value = string[] | string
@@ -11,12 +11,12 @@ type FieldSpecificationCallbackMultiple = (entry: Record<string, Value>) => bool
 type FieldSpecificationCallbackFormat = (value: SingleValue) => boolean;
 
 interface FieldError {
-  field: string,
-  error: string
+    field: string,
+    error: string
 }
 
 interface WorkError extends FieldError {
-  entity: WorkId
+    entity: WorkId
 }
 
 type Rank = string
@@ -28,59 +28,59 @@ type ResourceId = string
 type WorkId = string
 
 interface WorkingTaxon {
-  scientificNameID?: TaxonId,
-  scientificName?: string,
-  scientificNameAuthorship?: string,
-  genericName?: string,
-  infragenericEpithet?: string,
-  specificEpithet?: string,
-  intraspecificEpithet?: string,
-
-  taxonRank?: Rank,
-  taxonRemarks?: string,
-  collectionCode?: ResourceId,
-
-  taxonomicStatus?: TaxonStatus,
-  acceptedNameUsageID?: TaxonId,
-  acceptedNameUsage?: string,
-
-  parentNameUsageID?: TaxonId,
-  parentNameUsage?: string,
-  kingdom?: string,
-  phylum?: string,
-  class?: string,
-  order?: string,
-  family?: string,
-  subfamily?: string,
-  genus?: string,
-  subgenus?: string,
-  higherClassification?: string,
-
-  // Non-standard
-  scientificNameOnly?: string,
-  incorrect?: WorkingTaxon
+    scientificNameID?: TaxonId,
+    scientificName?: string,
+    scientificNameAuthorship?: string,
+    genericName?: string,
+    infragenericEpithet?: string,
+    specificEpithet?: string,
+    intraspecificEpithet?: string,
+
+    taxonRank?: Rank,
+    taxonRemarks?: string,
+    collectionCode?: ResourceId,
+
+    taxonomicStatus?: TaxonStatus,
+    acceptedNameUsageID?: TaxonId,
+    acceptedNameUsage?: string,
+
+    parentNameUsageID?: TaxonId,
+    parentNameUsage?: string,
+    kingdom?: string,
+    phylum?: string,
+    class?: string,
+    order?: string,
+    family?: string,
+    subfamily?: string,
+    genus?: string,
+    subgenus?: string,
+    higherClassification?: string,
+
+    // Non-standard
+    scientificNameOnly?: string,
+    incorrect?: WorkingTaxon
 }
 
 interface Taxon extends WorkingTaxon {
-  scientificNameID: TaxonId,
-  scientificName: string,
-  taxonRank: Rank,
-  collectionCode: ResourceId,
-  taxonomicStatus: string
+    scientificNameID: TaxonId,
+    scientificName: string,
+    taxonRank: Rank,
+    collectionCode: ResourceId,
+    taxonomicStatus: string
 }
 
 interface ResourceMetadata {
-  levels: Rank[],
-  scope: string[],
-  catalog?: object
+    levels: Rank[],
+    scope: string[],
+    catalog?: object
 }
 
 interface Resource {
-  id: string,
-  file: string,
-  workId: string,
-  metadata: ResourceMetadata,
-  taxa: Record<TaxonId, Taxon>
+    id: string,
+    file: string,
+    workId: string,
+    metadata: ResourceMetadata,
+    taxa: Record<TaxonId, Taxon>
 }
 
 interface ResourceHistory {
@@ -103,3 +103,23 @@ declare enum ResourceDiffType {
     Modified = '~',
     Unchanged = '='
 }
+
+interface AmendedTaxon extends Taxon {
+    colTaxonID?: string,
+    colAcceptedTaxonID?: string,
+    gbifTaxonID?: string,
+    gbifAcceptedTaxonID?: string
+}
+
+interface AmendedResource extends Resource {
+    taxa: Record<TaxonId, AmendedTaxon>
+}
+
+interface TaxonMatch {
+    source: number,
+    id: string,
+    currentId?: string,
+    classificationPath: string[]
+}
+
+type GroupedNameMatches = Record<string, Record<string, Record<TaxonId, TaxonMatch>>>

package/src/resources/parse-text.ts

@@ -372,7 +372,7 @@ function parseHeader (header: string): ResourceMetadata {
 
 function validateResource (config: ResourceMetadata, content: string) {
     // Check for too much indentation
-    const longerIndent = new RegExp(`^(  ){${config.levels.length - 1}}(?!  [+=>] ) `, 'm')
+    const longerIndent = new RegExp(`^(  ){${config.levels.length - 1}}(?!  [+=>] |    > ) `, 'm')
     const longerIndentMatch = content.match(longerIndent)
     if (longerIndentMatch !== null) {
         const offset = longerIndentMatch.index
@@ -425,8 +425,8 @@ function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds
         const lineIndent = (line.match(/^ */) as string[])[0].length
 
         if (lineIndent > groupIndent) {
-            // Do not count synonyms as parents
-            if (data[previousId] && data[previousId].taxonomicStatus === 'accepted') {
+            // Do not count synonyms as parents (unless this is correcting a typo in the synonym)
+            if (data[previousId] && data[previousId].taxonomicStatus === 'accepted' || /^( {2})+> /.test(line)) {
                 parents.push(previousId)
             } else {
                 parents.push(null)

package/src/taxon-names/index.ts

@@ -0,0 +1,79 @@
+const MINIMUM_PREFIX_LENGTH = 3
+const VALID_COMMON_PREFIXES = new Set([
+    'Plantae|Tracheophyta',
+    'Fungi',
+    'Fungi|Ascomycota',
+    'Fungi|Basidiomycota',
+    'Fungi|Zygomycota'
+])
+
+function getCommonPrefix (a: string[], b: string[]): string[] {
+    for (let i = 0; i < Math.max(a.length, b.length); i++) {
+        if (a[i] !== b[i]) {
+            return a.slice(0, i)
+        }
+    }
+    return a.slice()
+}
+
+function isValidPrefix (a: string[], b: string[]): boolean {
+    const prefix = getCommonPrefix(a, b)
+    return VALID_COMMON_PREFIXES.has(prefix.join('|')) || prefix.length >= MINIMUM_PREFIX_LENGTH
+}
+
+export function groupNameMatches (results: Record<TaxonId, TaxonMatch[]>): GroupedNameMatches {
+    const prefixes: Record<string, [string[], Record<TaxonId, TaxonMatch>][]> = {}
+
+    for (const scientificNameID in results) {
+        for (const result of results[scientificNameID]) {
+            if (!prefixes[result.source]) {
+                prefixes[result.source] = []
+            }
+
+            let prefix = prefixes[result.source].find(prefix => isValidPrefix(prefix[0], result.classificationPath))
+
+            if (!prefix) {
+                prefix = [result.classificationPath, {}]
+                prefixes[result.source].push(prefix)
+            } else {
+                prefix[0] = getCommonPrefix(prefix[0], result.classificationPath)
+            }
+
+            if (scientificNameID in prefix[1]) {
+                continue
+            }
+
+            prefix[1][scientificNameID] = result
+        }
+    }
+
+    const groupedNameMatches: GroupedNameMatches = {}
+    for (const source in prefixes) {
+        groupedNameMatches[source] = prefixes[source]
+            .sort((a, b) => Object.keys(b[1]).length - Object.keys(a[1]).length)
+            .reduce((map: Record<string, Record<TaxonId, TaxonMatch>>, [prefix, taxa]) => {
+                map[prefix.join('|')] = taxa
+                return map
+            }, {})
+    }
+
+    return groupedNameMatches
+}
+
+export function amendResource (resource: AmendedResource, source: string, matches: Record<TaxonId, TaxonMatch>) {
+    for (const id in matches) {
+        const match = matches[id]
+
+        if (source === '1') {
+            resource.taxa[id].colTaxonID = match.id
+            if (match.currentId) {
+                resource.taxa[id].colAcceptedTaxonID = match.currentId
+            }
+        } else if (source === '11') {
+            resource.taxa[id].gbifTaxonID = match.id
+            if (match.currentId) {
+                resource.taxa[id].gbifAcceptedTaxonID = match.currentId
+            }
+        }
+    }
+}

package/test/resources.js

@@ -4,7 +4,33 @@ const assert = require('assert')
 const { resources } = require('../lib')
 
 test('resources', async (t) => {
-    await t.test('missing leaf taxa', (t) => {
+    await t.test('parses author with initials', (t) => {
+        const [resource] = resources.parseTextFile(`---
+levels: [family, genus, species]
+---
+
+    Dolichurus haemorrhous A. Costa, 1886
+  Dolichurus A. Costa, 1886
+    Dolichurus indet.
+Sphecidae A. Costa, 1886
+    Sphecidae indet.
+`, 'T1')
+        assert.deepStrictEqual(Object.values(resource.taxa).map(taxon => taxon.scientificNameAuthorship), Array(3).fill('A. Costa, 1886'))
+    })
+
+    await t.test('does not validate name with correction', (t) => {
+        const [resource] = resources.parseTextFile(`---
+levels: [species]
+---
+
+Clytochrysus lapidarius (Panzer, 1804)
+  = Crabo chrysostomus Lepeletier & Brullé, 1835
+    > Crabro chrysostomus Lepeletier & Brullé, 1835
+`, 'T1')
+        assert.strictEqual(resource.taxa['T1:1:1'].scientificName, 'Clytochrysus lapidarius (Panzer, 1804)')
+    })
+
+    await t.test('errors for missing leaf taxa', (t) => {
         assert.throws(() => {
             resources.parseTextFile(`---
 levels: [family, genus, species]