@larsgw/formica 0.1.0

package/src/bin/process-resources.ts

@@ -0,0 +1,406 @@
+import { promises as fs } from 'fs'
+import * as path from 'path'
+import { spawn } from 'child_process'
+import * as util from 'util'
+
+import { csv } from '../index'
+import { prompt, promptForAnswers, numericSort, runCommand } from './util'
+
+interface AmendedTaxon extends Taxon {
+    colTaxonID?: string,
+    gbifTaxonID?: string
+}
+
+interface AmendedResource extends Resource {
+    taxa: Record<TaxonId, AmendedTaxon>
+}
+
+type Classifications = Record<string, Array<[AmendedTaxon, string]>>
+
+const DWC_FIELDS: string[] = [
+    'scientificNameID',
+    'scientificName',
+    'scientificNameAuthorship',
+    'genericName',
+    'intragenericEpithet',
+    'specificEpithet',
+    'intraspecificEpithet',
+
+    'taxonRank',
+    'taxonRemarks',
+    'collectionCode',
+
+    'taxonomicStatus',
+    'acceptedNameUsageID',
+    'acceptedNameUsage',
+
+    'parentNameUsageID',
+    'parentNameUsage',
+    'kingdom',
+    'phylum',
+    'class',
+    'order',
+    'family',
+    'subfamily',
+    'genus',
+    'subgenus',
+    'higherClassification',
+
+    'colTaxonID',
+    'gbifTaxonID'
+]
+
+const DISPLAY_FIELDS: string[] = [
+    'scientificNameID',
+    'taxonRank',
+    'scientificName',
+    'taxonomicStatus',
+    'taxonRemarks',
+    'colTaxonID',
+    'gbifTaxonID'
+]
+
+const GBIF_RANKS: Rank[] = [
+    'kingdom',
+    'phyllum',
+    'class',
+    'order',
+    'family',
+    'genus',
+    'species',
+    'subspecies',
+    'variety'
+]
+
+const VALID_COMMON_PREFIXES = [
+    'Plantae|Tracheophyta',
+    'Fungi',
+    'Fungi|Ascomycota',
+    'Fungi|Basidiomycota',
+    'Fungi|Zygomycota'
+]
+
+function runGnverifier (names: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const proc = spawn('gnverifier', ['-s', '1,11', '-M'])
+        let stdout = ''
+        proc.stdout.on('data', data => { stdout += data })
+        proc.stderr.pipe(process.stdout)
+        proc.on('close', code => {
+            if (code === 0) {
+                resolve(stdout)
+            } else {
+                reject()
+            }
+        })
+        proc.stdin.write(names)
+        proc.stdin.end()
+    })
+}
+
+async function listChangedFiles (directory: string): Promise<string[]> {
+    const output = await runCommand('git', ['diff', '--name-only', 'HEAD', '--', directory], {
+        cwd: directory
+    })
+    return output.trimEnd().split('\n').sort(numericSort)
+}
+
+async function getOldFile (file: string): Promise<string> {
+    const options = {
+        cwd: path.dirname(file)
+    }
+    const gitRoot = (await runCommand('git', ['rev-parse', '--show-toplevel'], options)).trim()
+    return await runCommand('git', ['show', 'HEAD:' + path.relative(gitRoot, file)], options)
+}
+
+class ResourceProcessor {
+    DIR_ROOT: string;
+    DIR_TXT: string;
+    DIR_DWC: string;
+    FILE_PROBLEMS: string;
+
+    constructor (collectionPath: string) {
+        this.DIR_ROOT = path.resolve(collectionPath)
+        this.DIR_TXT = path.join(this.DIR_ROOT, 'txt')
+        this.DIR_DWC = path.join(this.DIR_ROOT, 'dwc')
+        this.FILE_PROBLEMS = path.join(this.DIR_ROOT, 'problems.csv')
+    }
+
+    async run (): Promise<void> {
+        const input = await fs.readdir(this.DIR_TXT)
+        const output = await fs.readdir(this.DIR_DWC)
+
+        const ids = input
+            .map(file => path.basename(file, '.txt'))
+            .sort((a, b) => parseInt(a.slice(1)) - parseInt(b.slice(1)))
+
+        for (const id of ids) {
+            // Skip existing files
+            if (output.some(file => file.startsWith(id + '-'))) {
+                continue
+            }
+
+            await this.processWork(id)
+        }
+    }
+
+    async runUpdate (): Promise<void> {
+        for (const file of await listChangedFiles(this.DIR_TXT)) {
+            const id = path.basename(file, '.txt')
+            await this.processWork(id, true)
+        }
+    }
+
+    async processWork (id: WorkId, update?: boolean): Promise<void> {
+        const resources = await this.processResources(id, update)
+
+        await Promise.all(resources.map(resource => {
+            const header = DWC_FIELDS
+            const table = [header]
+
+            for (const id in resource.taxa) {
+                const taxon = resource.taxa[id] as unknown as Record<string, string | undefined>
+                table.push(header.map(column => taxon[column] || ''))
+            }
+
+            return fs.writeFile(path.join(this.DIR_DWC, `${resource.file}.csv`), csv.formatCsv(table, ',').trim())
+        }))
+    }
+
+    async processResources (id: WorkId, update?: boolean): Promise<AmendedResource[]> {
+        const resources = await this.processResourceText(id, update)
+
+        const amendedResources = []
+        for (const resource of resources) {
+            const [results, classifications] = await this.processResourceDwc(resource)
+
+            for (const source in classifications) {
+                await this.checkPrefix(resource, classifications, source)
+            }
+
+            const skip = await this.shouldBeSkipped(resource.id)
+
+            if (!skip) {
+                // TODO const correct = checkResults(results, classifications)
+                const correct = this.checkResults(results)
+                if (!correct) {
+                    const choice = await promptForAnswers(
+                        `${resource.workId}: problems found in ${resource.id}. Skip or retry (s/r)? `,
+                        ['s', 'S', 'r', 'R']
+                    )
+
+                    switch (choice) {
+                        case 's':
+                        case 'S': {
+                            const reason = await prompt('Reason for skipping? ')
+                            fs.appendFile(this.FILE_PROBLEMS, csv.formatCsv([[
+                                resource.workId,
+                                resource.id,
+                                reason
+                            ]]))
+                            console.log(`${resource.workId}: skipping ${resource.id}`)
+                            break
+                        }
+
+                        case 'r':
+                        case 'R': {
+                            console.log(`${resource.workId}: retrying ${resource.id}`)
+                            return this.processResources(id, update)
+                        }
+                    }
+                }
+            }
+
+            amendedResources.push(results)
+        }
+
+        return amendedResources
+    }
+
+    async processResourceText (id: WorkId, update?: boolean): Promise<Resource[]> {
+        try {
+            console.log(`${id}: generating Darwin Core`)
+            const filePath = path.join(this.DIR_TXT, id + '.txt')
+            const file = await fs.readFile(filePath, 'utf-8')
+
+            let old = undefined
+            if (update) {
+                const dwc = []
+                for (const file of await fs.readdir(this.DIR_DWC)) {
+                    if (file.startsWith(id + '-')) {
+                        const filePath = path.join(this.DIR_DWC, file)
+                        dwc.push(csv.parseCsv(await getOldFile(filePath)))
+                    }
+                }
+
+                old = { txt: await getOldFile(filePath), dwc }
+            }
+
+            const { resources } = await import('../index')
+            return resources.parseTextFile(file, id, old)
+        } catch (error) {
+            console.log(error)
+            await prompt(`${id}: generating Darwin Core failed, retry? `)
+
+            // Clear cache to re-import
+            const prefix = path.dirname(require.resolve('../index'))
+            for (const file in require.cache) {
+                if (file.startsWith(prefix)) {
+                    delete require.cache[file]
+                }
+            }
+
+            return this.processResourceText(id, update)
+        }
+    }
+
+    async processResourceDwc (resource: Resource): Promise<[AmendedResource, Classifications]> {
+        console.log(`${resource.workId}: matching ${resource.id}`)
+        const taxa: Record<string, Record<TaxonId, AmendedTaxon>> = {}
+        const names = []
+        for (const id in resource.taxa) {
+            const name = resource.taxa[id].scientificName
+
+            if (!taxa[name]) { taxa[name] = {} }
+            taxa[name][id] = { ...resource.taxa[id] }
+
+            names.push(name)
+        }
+
+        const result = await runGnverifier(names.join('\n'))
+        const classifications: Classifications = { '1': [], '11': [] }
+
+        const [header, ...matches] = csv.parseCsv(result)
+        for (const match of matches) {
+            const name = match[header.indexOf('ScientificName')]
+            const source = match[header.indexOf('DataSourceId')]
+            const id = match[header.indexOf('TaxonId')]
+            const classification = match[header.indexOf('ClassificationPath')]
+
+            for (const loirId in taxa[name]) {
+                const taxon = taxa[name][loirId]
+                if (source === '1' && !taxon.colTaxonID) {
+                    taxon.colTaxonID = id
+                    classifications[source].push([taxon, classification])
+                }
+                if (source === '11' && GBIF_RANKS.includes(taxon.taxonRank) && !taxon.gbifTaxonID) {
+                    taxon.gbifTaxonID = id
+                    classifications[source].push([taxon, classification])
+                }
+            }
+        }
+
+        const results = {
+            ...resource,
+            taxa: Object.fromEntries(Object.values(resource.taxa).map(taxon => [
+                taxon.scientificNameID,
+                taxa[taxon.scientificName][taxon.scientificNameID]
+            ]))
+        }
+
+        return [results, classifications]
+    }
+
+    async checkPrefix (resource: AmendedResource, classifications: Classifications, source: string): Promise<void> {
+        const lists = classifications[source]
+        if (!lists.length) { return }
+        const prefix = lists[0][1].split('|')
+
+        for (const [taxon, list] of lists.slice(1)) {
+            const parts = list.split('|')
+            for (let i = 0; i < parts.length; i++) {
+                if (parts[i] !== prefix[i] && i < 3) {
+                    let choice
+
+                    if (source === '1' || taxon.taxonomicStatus !== 'accepted') {
+                        choice = 'd'
+                    } else if (VALID_COMMON_PREFIXES.includes(prefix.slice(0, i).join('|'))) {
+                        choice = 'k'
+                    } else {
+                        console.log(`${resource.workId}: source ${source} results in short prefix "${prefix.slice(0, i).join('|')}" (${i} taxa)`)
+                        console.log(`  taxon: ${taxon.scientificNameID} "${taxon.scientificName}"`)
+                        console.log(`  class: ${parts.join('|')}`)
+                        console.log(`  prefx: ${prefix.join('|')}`)
+
+                        choice = await promptForAnswers(`  Keep or delete (k/d)? `, ['k', 'K', 'd', 'D'])
+                    }
+
+                    switch (choice) {
+                        case 'k':
+                        case 'K': {
+                            console.log(`  keeping...`)
+                            break
+                        }
+
+                        case 'd':
+                        case 'D': {
+                            console.log(`  deleting...`)
+                            if (source === '1') {
+                                delete taxon.colTaxonID
+                            } else if (source === '11') {
+                                delete taxon.gbifTaxonID
+                            }
+                            break
+                        }
+                    }
+
+                    break
+                }
+            }
+        }
+    }
+
+    checkResults (resource: AmendedResource): boolean {
+        let correct = true
+        const missing = []
+
+        for (const id in resource.taxa) {
+            const taxon = resource.taxa[id]
+            if (taxon.taxonomicStatus !== 'accepted') { continue }
+
+            const missingCol = false // !taxon.colTaxonID
+            const missingGbif = GBIF_RANKS.includes(taxon.taxonRank) && !taxon.gbifTaxonID
+
+            if (missingCol || missingGbif) {
+                correct = false
+                missing.push(taxon)
+            }
+        }
+
+        if (missing.length) {
+            console.table(missing, DISPLAY_FIELDS)
+        }
+
+        return correct
+    }
+
+    async shouldBeSkipped (id: ResourceId): Promise<boolean> {
+        const problems = csv.parseCsv(await fs.readFile(this.FILE_PROBLEMS, 'utf8'))
+        return problems.some(([_work, resource, _problem]) => resource === id)
+    }
+}
+
+function main (): void {
+    const args = util.parseArgs({
+        options: {
+            update: {
+                type: 'boolean',
+                short: 'u'
+            }
+        },
+        allowPositionals: true
+    })
+
+    const processor = new ResourceProcessor(args.positionals[0])
+    process.on('exit', () => {
+        process.stdout.write('\n')
+    })
+
+    const task = args.values.update ? processor.runUpdate() : processor.run()
+    task.catch(error => {
+        console.error(error)
+        process.exit(1)
+    })
+}
+
+main()

package/src/bin/util.ts

@@ -0,0 +1,74 @@
+import * as readline from 'readline'
+import { spawn } from 'child_process'
+
+/**
+ * Comparison function to sort strings with numerical components.
+ */
+export function numericSort (a: string, b: string): number {
+    const as = a.split(/(\d+)/)
+    const bs = b.split(/(\d+)/)
+    for (let i = 0; i < Math.max(as.length, bs.length); i++) {
+        const ai = as[i]
+        const bi = bs[i]
+
+        if (ai === bi) {
+            continue
+        } else if (!ai) {
+            return 1
+        } else if (!bi) {
+            return -1
+        } else if (i % 2) {
+            return parseInt(ai) - parseInt(bi)
+        } else {
+            return ai > bi ? -1 : 1
+        }
+    }
+    return 0
+}
+
+/**
+ * Create a CLI prompt and return any answer.
+ */
+export function prompt (question: string): Promise<string> {
+    const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout
+    })
+    return new Promise(resolve => {
+        rl.question(question, (answer: string) => {
+            rl.close()
+            resolve(answer)
+        })
+    })
+}
+
+/**
+ * Create a CLI prompt and only accept certain answers.
+ */
+export async function promptForAnswers (question: string, answers: string[]) {
+    let answer
+
+    do {
+        answer = (await prompt(question))[0]
+    } while (!answers.includes(answer))
+
+    return answer
+}
+
+/**
+ * Run a command and return stdout.
+ */
+export function runCommand (command: string, args: string[], options?: Record<string, unknown>): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const proc = spawn(command, args, options)
+        let stdout = ''
+        proc.stdout.on('data', data => { stdout += data })
+        proc.on('close', code => {
+            if (code === 0) {
+                resolve(stdout)
+            } else {
+                reject()
+            }
+        })
+    })
+}

package/src/bin/validate-catalog.ts

@@ -0,0 +1,37 @@
+#!/usr/bin/env node
+
+import { promises as fs } from 'fs'
+import * as path from 'path'
+import { catalog } from '../index'
+
+async function validateFile (arg: string): Promise<{ filePath: string, errors: WorkError[]}> {
+    const filePath = path.resolve(arg)
+    const file = await fs.readFile(filePath, 'utf8')
+    const sheet = path.basename(filePath, '.csv')
+    return {
+        filePath,
+        errors: catalog.loadData(file, sheet).validate()
+    }
+}
+
+async function main (args: string[]): Promise<void> {
+    let exitStatus = 0
+
+    const results = await Promise.allSettled(args.map(validateFile))
+    for (const result of results) {
+        if (result.status === 'rejected') {
+            console.error(result.reason)
+            console.error()
+            exitStatus = 1
+        } else if (result.value.errors.length > 0) {
+            console.error(`${result.value.filePath}:`)
+            console.table(result.value.errors)
+            console.error()
+            exitStatus = 1
+        }
+    }
+
+    process.exit(exitStatus)
+}
+
+main(process.argv.slice(2))

package/src/bin/validate-resources-text.ts

@@ -0,0 +1,25 @@
+#!/usr/bin/env node
+
+import { promises as fs } from 'fs'
+import * as path from 'path'
+import { resources } from '../index'
+
+async function main (args: string[]): Promise<void> {
+    let exitStatus = 0
+
+    for (const arg of args) {
+        const filePath = path.resolve(arg)
+        const file = await fs.readFile(filePath, 'utf8')
+        const id = path.basename(filePath, '.txt')
+        try {
+            resources.parseTextFile(file, id)
+        } catch (error) {
+            console.error(filePath + '\n    ' + error.message + '\n')
+            exitStatus = 1
+        }
+    }
+
+    process.exit(exitStatus)
+}
+
+main(process.argv.slice(2))

package/src/catalog/entities.ts

@@ -0,0 +1,62 @@
+import { Entity } from './entity'
+
+export class Entities {
+    entities: Entity[];
+    indexField: string;
+    index: Record<string, Entity>;
+
+    constructor (entities: Entity[], indexField: string) {
+        this.entities = entities
+        this.indexField = indexField
+        this.index = {}
+        for (const entity of this.entities) {
+            const key = entity.get(indexField)
+            if (typeof key !== 'string') {
+                throw new TypeError('Entity missing index field')
+            }
+            this.index[key] = entity
+        }
+    }
+
+    *[Symbol.iterator] () {
+        for (const entity of this.entities) {
+            yield entity
+        }
+    }
+
+    get (key: string): Entity {
+        return this.index[key]
+    }
+
+    has (key: string): boolean {
+        return key in this.index
+    }
+
+    validate (): WorkError[] {
+        const errors: WorkError[] = []
+        for (const entity of this.entities) {
+            for (const { field, error } of entity.validate()) {
+                errors.push({
+                    entity: (entity.get(this.indexField) || '[missing]') as string,
+                    field,
+                    error
+                })
+            }
+        }
+        return errors
+    }
+
+    toTable (): string[][] {
+        if (this.entities.length === 0) {
+            return [[]]
+        }
+        const header = Object.keys(this.entities[0].fields)
+        const table = this.entities.map((entity: Entity) => {
+            return header.map((field: string): string => {
+                const value = entity.get(field) || ''
+                return Array.isArray(value) ? value.join('; ') : value
+            })
+        })
+        return [header, ...table]
+    }
+}