@larsgw/formica 0.4.2 → 0.5.0

package/lib/resources/parse-text.js

@@ -13,6 +13,7 @@ var __assign = (this && this.__assign) || function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.parseFileHeader = exports.parseFile = void 0;
 var yaml = require("js-yaml");
+var work_1 = require("../catalog/tables/work");
 var diff_resource_1 = require("./diff-resource");
 var RANKS = [
     'class',
@@ -122,9 +123,9 @@ var NAME_PATTERN = new RegExp('^' +
  *   $1 genus+subgenus (+ trailing space): (?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?
  *     $1.1 genus: ([A-Z]\S+)
  *     $1.2 subgenus: (?:\(([A-Z]\S+?)\) )?
- *   $2 species: ((?:x )?[a-z0-9-]+)
+ *   $2 species: ((?:x )?[a-z][^\s.]+)
  */
-var BINAME_PATTERN = /^(?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?((?:x )?[a-z0-9-]+)(?= |$)/;
+var BINAME_PATTERN = /^(?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?((?:x )?[a-z][^\s.]+)(?= |$)/;
 function compareRanks(a, b) {
     return RANKS.indexOf(a) - RANKS.indexOf(b);
 }
@@ -174,55 +175,62 @@ function parseName(name, rank, parent) {
     if (/^\[(_|\d+)\] /.test(name)) {
         name = name.replace(/^\[(_|\d+)\] /, '');
     }
+    // Set verbatim identification after subsequent syntax is removed.
+    item.verbatimIdentification = name;
     // Parent context is used for parsing and formatting binomial names.
-    var parentContext = __assign({}, parent);
-    if (parent.incorrect) {
-        parentContext.incorrect = __assign({}, parent.incorrect);
-    }
-    // The parent context should be amended in the two cases where binomial names
-    // are truly accepted: synonyms and species (and below) without parents (resp.
-    // genera and genera and species) to provide parts of the name.
+    // For formatting, it needs to match external databases (i.e. be correct).
+    // For parsing, it needs to match the current file. If relevant parents
+    // (i.e. genus, species) had mistakes that were corrected, the uncorrected
+    // genus and species names need to be used.
+    var parentContext = {
+        genus: parent.genus,
+        subgenus: parent.subgenus,
+        specificEpithet: parent.specificEpithet,
+        incorrect: {
+            genus: parent.incorrect && parent.incorrect.genus,
+            specificEpithet: parent.incorrect && parent.incorrect.specificEpithet
+        }
+    };
+    // Both contexts should be amended in the two cases where binomial names
+    // are fully used: (1) synonyms and (2) multinomial taxa without parents to
+    // provide parts of the name (e.g. bare species without a genus parent, or
+    // even subspecies without a species or genus parent).
     if (isSynonym || !parentContext.genus || (compareRanks('species', rank) < 0 && !parentContext.specificEpithet)) {
         var _a = name.match(BINAME_PATTERN) || [], genus = _a[1], subgenus = _a[2], species = _a[3];
         if (genus) {
-            parentContext.genus = capitalize(genus);
-            if (parentContext.incorrect)
-                parentContext.incorrect.genus = capitalize(genus);
+            parentContext.genus = parentContext.incorrect.genus = capitalize(genus);
         }
         if (subgenus) {
             parentContext.subgenus = capitalize(subgenus);
-            if (parentContext.incorrect)
-                parentContext.incorrect.subgenus = capitalize(subgenus);
         }
         else if (genus) {
-            // If a genus is given but no subgenus, remove it from the parent context
+            // If a genus is given but no subgenus, remove any existing subgenus
+            // from the parent context.
             delete parentContext.subgenus;
-            if (parentContext.incorrect)
-                delete parentContext.incorrect.subgenus;
         }
         if (species) {
-            parentContext.specificEpithet = species;
-            if (parentContext.incorrect)
-                parentContext.incorrect.specificEpithet = species;
+            parentContext.specificEpithet = parentContext.incorrect.specificEpithet = species;
         }
     }
     // In taxa of group, species or lower, the name should just contain the
-    // (inter)specific epithet and the author information & remarks when processing
+    // (infra)specific epithet and the author information & remarks when processing
     // further.
     if (compareRanks('group', rank) <= 0) {
-        var parseContext = parentContext.incorrect || parentContext;
-        if (!parseContext.genus) {
-            parseContext.genus = name.split(' ', 1)[0];
-        }
-        var genusPrefix = new RegExp("^".concat(parentContext.genus, " (\\(.*?\\) )?"), 'i');
-        if (name[0] === parentContext.genus[0]) {
-            name = name.replace(genusPrefix, '');
-        }
+        // Remove genus
+        var genus = parentContext.incorrect.genus || parentContext.genus || '';
+        if (name[0] === genus[0] && name.toLowerCase().startsWith(genus.toLowerCase() + ' ')) {
+            name = name.slice(genus.length + 1);
+        }
+        // Remove subgenus
+        name = name.replace(/^\(.*?\) /, '');
+        // Infraspecific taxa
         if (compareRanks('species', rank) < 0) {
-            var speciesPrefix = parseContext.specificEpithet + ' ';
-            if (name.startsWith(speciesPrefix)) {
-                name = name.slice(speciesPrefix.length);
+            // Remove specific epithet
+            var species = parentContext.incorrect.specificEpithet || parentContext.specificEpithet || '';
+            if (name.startsWith(species + ' ')) {
+                name = name.slice(species.length + 1);
             }
+            // Remove rank abbreviations
             name = name.replace(/^(st|r|ab|f|var|ssp|subsp)\. /, '');
         }
     }
@@ -236,6 +244,11 @@ function parseName(name, rank, parent) {
     if (!nameParts) {
         throw new Error("Taxon \"".concat(name, "\" could not be parsed"));
     }
+    // To encode old names with spaces (e.g. "Orsillus pini canariensis Lindberg, 1953")
+    // underscores are used, which are replaced here.
+    if (nameParts[1].includes('_')) {
+        nameParts[1] = nameParts[1].replace(/_/g, ' ');
+    }
     var _ = nameParts[0], taxon = nameParts[1], _b = nameParts[2], citation = _b === void 0 ? '' : _b, notes = nameParts[3];
     item.scientificNameAuthorship = capitalizeAuthors(citation);
     item.taxonRemarks = notes;
@@ -284,21 +297,21 @@ function parseName(name, rank, parent) {
         item.genericName = parentContext.genus;
         item.infragenericEpithet = parentContext.subgenus;
         item.specificEpithet = parentContext.specificEpithet;
-        item.intraspecificEpithet = taxon.toLowerCase();
+        item.infraspecificEpithet = taxon.toLowerCase();
         // If possible, names below species should have abbreviations for ranks,
         // like "subsp."
         var nameParts_1 = [
             item.genericName,
             item.specificEpithet,
-            item.intraspecificEpithet
+            item.infraspecificEpithet
         ];
         if (item.taxonRank in RANK_LABELS) {
             nameParts_1.splice(2, 0, RANK_LABELS[item.taxonRank]);
         }
         item.scientificName = nameParts_1.join(' ');
-        if (item.intraspecificEpithet !== taxon) {
+        if (item.infraspecificEpithet !== taxon) {
             console.log(item, taxon);
-            throw new Error("Intraspecific epithet should be lowercase: \"".concat(taxon, "\""));
+            throw new Error("Infraspecific epithet should be lowercase: \"".concat(taxon, "\""));
         }
     }
     // Re-add authorship information
@@ -335,15 +348,8 @@ function parseHeader(header) {
     else {
         levels = config.levels;
     }
-    var scope;
-    if (!('scope' in config)) {
-        scope = [];
-    }
-    else if (!Array.isArray(config.scope)) {
-        throw new SyntaxError('"scope" should be an array');
-    }
-    else {
-        scope = config.scope;
+    if ('scope' in config) {
+        throw new SyntaxError('"scope" data should go in "catalog"');
     }
     // No taxon ranks
     if (levels.length === 0) {
@@ -354,9 +360,36 @@ function parseHeader(header) {
     if (invalidTaxonRanks.length) {
         throw new SyntaxError("\"levels\" contains invalid values: ".concat(invalidTaxonRanks.join(', ')));
     }
-    var metadata = { levels: levels, scope: scope };
+    var metadata = { levels: levels };
     if ('catalog' in config && typeof config.catalog === 'object' && config.catalog !== null) {
-        metadata.catalog = config.catalog;
+        var catalog = {};
+        for (var key in config.catalog) {
+            var value = config.catalog[key];
+            if (typeof value === 'number') {
+                catalog[key] = value.toString();
+            }
+            else if (typeof value === 'string') {
+                catalog[key] = value;
+            }
+            else {
+                throw new SyntaxError("\"catalog\" should contain only strings (\"".concat(key, "\")"));
+            }
+        }
+        var work = new work_1.Work(catalog);
+        var errors = work.validate().filter(function (_a) {
+            var error = _a.error;
+            return error !== 'Value(s) required but missing';
+        });
+        if (errors.length > 0) {
+            throw new SyntaxError("\"catalog\" contains errors: ".concat(errors.map(function (_a) {
+                var field = _a.field, error = _a.error;
+                return "[".concat(field, "] ").concat(error);
+            }).join('; ')));
+        }
+        metadata.catalog = {};
+        for (var key in work.fields) {
+            metadata.catalog[key] = work.get(key);
+        }
     }
     return metadata;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larsgw/formica",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "description": "SDK and tools for data from the Library of Identification Resources",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",

package/src/bin/process-resources.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-import { promises as fs } from 'fs'
+import { promises as fs, existsSync as doesFileExist } from 'fs'
 import * as path from 'path'
 import { spawn } from 'child_process'
 import * as util from 'util'
@@ -8,14 +8,20 @@ import * as util from 'util'
 import { csv } from '../index'
 import { prompt, promptForAnswers, numericSort, runCommand } from './util'
 
-const DWC_FIELDS: string[] = [
+export enum ResourceProcessorSource {
+    All = 'all',
+    Unprocessed = 'unprocessed',
+    Modified = 'modified'
+}
+
+const DWC_FIELDS: (keyof AmendedTaxon)[] = [
     'scientificNameID',
     'scientificName',
     'scientificNameAuthorship',
     'genericName',
-    'intragenericEpithet',
+    'infragenericEpithet',
     'specificEpithet',
-    'intraspecificEpithet',
+    'infraspecificEpithet',
 
     'taxonRank',
     'taxonRemarks',
@@ -36,6 +42,7 @@ const DWC_FIELDS: string[] = [
     'genus',
     'subgenus',
     'higherClassification',
+    'verbatimIdentification',
 
     'colTaxonID',
     'gbifTaxonID',
@@ -43,7 +50,7 @@ const DWC_FIELDS: string[] = [
     'gbifAcceptedTaxonID'
 ]
 
-const DISPLAY_FIELDS: string[] = [
+const DISPLAY_FIELDS: (keyof AmendedTaxon)[] = [
     'scientificNameID',
     'taxonRank',
     'scientificName',
@@ -83,11 +90,22 @@ function runGnverifier (names: string): Promise<string> {
     })
 }
 
+async function listFiles (directory: string): Promise<string[]> {
+    const input = await fs.readdir(directory)
+    return input.map(file => path.basename(file, '.txt')).sort(numericSort)
+}
+
+async function listUnprocessedFiles (directory: string, outputDirectory: string): Promise<string[]> {
+    const input = await listFiles(directory)
+    const output = new Set(await fs.readdir(outputDirectory))
+    return input.filter(file => output.has(file + '-1'))
+}
+
 async function listChangedFiles (directory: string): Promise<string[]> {
     const output = await runCommand('git', ['diff', '--name-only', 'HEAD', '--', directory], {
         cwd: directory
     })
-    return output.trimEnd().split('\n').sort(numericSort)
+    return output.trimEnd().split('\n').map(file => path.basename(file, '.txt')).sort(numericSort)
 }
 
 async function getOldFile (file: string): Promise<string> {
@@ -111,49 +129,32 @@ class ResourceProcessor {
         this.FILE_PROBLEMS = path.join(this.DIR_ROOT, 'problems.csv')
     }
 
-    async run (): Promise<void> {
-        const input = await fs.readdir(this.DIR_TXT)
-        const output = await fs.readdir(this.DIR_DWC)
-
-        const ids = input
-            .map(file => path.basename(file, '.txt'))
-            .sort((a, b) => parseInt(a.slice(1)) - parseInt(b.slice(1)))
-
+    async run (source: ResourceProcessorSource, config: ResourceProcessorConfig): Promise<void> {
+        const ids = await this.listWorks(source)
         for (const id of ids) {
-            // Skip existing files
-            if (output.some(file => file.startsWith(id + '-'))) {
-                continue
-            }
-
-            await this.processWork(id)
-        }
-    }
-
-    async runUpdate (): Promise<void> {
-        for (const file of await listChangedFiles(this.DIR_TXT)) {
-            const id = path.basename(file, '.txt')
-            await this.processWork(id, true)
+            await this.processWork(id, config)
         }
     }
 
-    async runMappingsUpdate (): Promise<void> {
-        const input = await fs.readdir(this.DIR_TXT)
-
-        const ids = input
-            .map(file => path.basename(file, '.txt'))
-            .sort((a, b) => parseInt(a.slice(1)) - parseInt(b.slice(1)))
-
-        for (const id of ids) {
-            await this.processWork(id, true)
+    async listWorks (source: ResourceProcessorSource): Promise<string[]> {
+        switch (source) {
+            case ResourceProcessorSource.All:
+                return listFiles(this.DIR_TXT)
+            case ResourceProcessorSource.Unprocessed:
+                return listUnprocessedFiles(this.DIR_TXT, this.DIR_DWC)
+            case ResourceProcessorSource.Modified:
+                return listChangedFiles(this.DIR_TXT)
+            default:
+                return []
         }
     }
 
-    async processWork (id: WorkId, update?: boolean): Promise<void> {
-        const resources = await this.processResources(id, update)
+    async processWork (id: WorkId, config: ResourceProcessorConfig): Promise<void> {
+        const resources = await this.processResources(id, config)
 
         await Promise.all(resources.map(resource => {
             const header = DWC_FIELDS
-            const table = [header]
+            const table: string[][] = [header]
 
             for (const id in resource.taxa) {
                 const taxon = resource.taxa[id] as unknown as Record<string, string | undefined>
@@ -164,12 +165,12 @@ class ResourceProcessor {
         }))
     }
 
-    async processResources (id: WorkId, update?: boolean): Promise<AmendedResource[]> {
-        const resources = await this.processResourceText(id, update)
+    async processResources (id: WorkId, config: ResourceProcessorConfig): Promise<AmendedResource[]> {
+        const resources = await this.processResourceText(id, config)
 
         const amendedResources = []
         for (const resource of resources) {
-            const results = await this.processResourceDwc(resource)
+            const results = await this.processResourceDwc(resource, config)
 
             const skip = await this.shouldBeSkipped(resource.id)
 
@@ -198,7 +199,7 @@ class ResourceProcessor {
                         case 'r':
                         case 'R': {
                             console.log(`${resource.workId}: retrying ${resource.id}`)
-                            return this.processResources(id, update)
+                            return this.processResources(id, config)
                         }
                     }
                 }
@@ -210,14 +211,14 @@ class ResourceProcessor {
         return amendedResources
     }
 
-    async processResourceText (id: WorkId, update?: boolean): Promise<Resource[]> {
+    async processResourceText (id: WorkId, config: ResourceProcessorConfig): Promise<Resource[]> {
         try {
             console.log(`${id}: generating Darwin Core`)
             const filePath = path.join(this.DIR_TXT, id + '.txt')
             const file = await fs.readFile(filePath, 'utf-8')
 
             let old = undefined
-            if (update) {
+            if (config.update) {
                 const dwc = []
                 for (const file of await fs.readdir(this.DIR_DWC)) {
                     if (file.startsWith(id + '-')) {
@@ -243,13 +244,34 @@ class ResourceProcessor {
                 }
             }
 
-            return this.processResourceText(id, update)
+            return this.processResourceText(id, config)
         }
     }
 
-    async processResourceDwc (resource: Resource): Promise<AmendedResource> {
+    async processResourceDwc (resource: Resource, config: ResourceProcessorConfig): Promise<AmendedResource> {
         console.log(`${resource.workId}: matching ${resource.id}`)
 
+        if (!config.updateMappings) {
+            const file = path.join(this.DIR_DWC, resource.file + '.csv')
+            if (doesFileExist(file)) {
+                const [header, ...rows] = csv.parseCsv(await fs.readFile(file, 'utf-8'))
+                for (const row of rows) {
+                    const oldTaxon = row.reduce((taxon, value, index) => {
+                        taxon[header[index]] = value
+                        return taxon
+                    }, {} as Record<string, string>)
+                    const taxon = resource.taxa[oldTaxon.scientificNameID] as AmendedTaxon
+                    if (taxon) {
+                        taxon.colTaxonID = oldTaxon.colTaxonID
+                        taxon.colAcceptedTaxonID = oldTaxon.colAcceptedTaxonID
+                        taxon.gbifTaxonID = oldTaxon.gbifTaxonID
+                        taxon.gbifAcceptedTaxonID = oldTaxon.gbifAcceptedTaxonID
+                    }
+                }
+            }
+            return resource as AmendedResource
+        }
+
         const filteredResults: Record<TaxonId, TaxonMatch[]> = {}
         const taxonNames: Record<string, TaxonId[]> = {}
         const names = new Set()
@@ -429,12 +451,14 @@ class ResourceProcessor {
 function main (): void {
     const args = util.parseArgs({
         options: {
-            update: {
-                type: 'boolean',
-                short: 'u'
+            source: {
+                type: 'string',
+                short: 's',
+                default: 'unprocessed'
             },
-            'update-mappings': {
-                type: 'boolean'
+            'keep-mappings': {
+                type: 'boolean',
+                short: 'k'
             }
         },
         allowPositionals: true
@@ -445,16 +469,13 @@ function main (): void {
         process.stdout.write('\n')
     })
 
-    let task
-    if (args.values.update) {
-        task = processor.runUpdate()
-    } else if (args.values['update-mappings']) {
-        task = processor.runMappingsUpdate()
-    } else {
-        task = processor.run()
+    const source = args.values.source as ResourceProcessorSource
+    const config: ResourceProcessorConfig = {
+        update: source !== 'unprocessed',
+        updateMappings: !args.values['keep-mappings']
     }
 
-    task.catch(error => {
+    processor.run(source, config).catch(error => {
         console.error(error)
         process.exit(1)
     })

package/src/catalog/value.ts

@@ -24,11 +24,11 @@ export const FORMATS = {
 
 export const CHECK = {
     MULTILANG (entry: Record<string, Value>) {
-        return entry.language.length > 1
+        return Array.isArray(entry.language) && entry.language.length > 1
     },
 
     ISBN (entry: Record<string, Value>) {
-        if (entry.ISBN.length < 2) { return false }
+        if (!Array.isArray(entry.ISBN) || entry.ISBN.length < 2) { return false }
         const a = entry.ISBN[0].length === 10
         const b = entry.ISBN[0].length === 13
         const c = entry.ISBN[1].length === 10

package/src/csv.ts

@@ -23,7 +23,7 @@ export function formatCsv (table: string[][], delimiter = ',') {
         .map((row: string[]) => {
             return row.map(value => {
                 if (/["\n]/.test(value) || value.includes(delimiter)) {
-                    return `"${value.replace(/"/, '"""')}"`
+                    return `"${value.replace(/"/g, '""')}"`
                 } else {
                     return value
                 }

package/src/module.d.ts

@@ -34,7 +34,7 @@ interface WorkingTaxon {
     genericName?: string,
     infragenericEpithet?: string,
     specificEpithet?: string,
-    intraspecificEpithet?: string,
+    infraspecificEpithet?: string,
 
     taxonRank?: Rank,
     taxonRemarks?: string,
@@ -55,6 +55,7 @@ interface WorkingTaxon {
     genus?: string,
     subgenus?: string,
     higherClassification?: string,
+    verbatimIdentification?: string,
 
     // Non-standard
     scientificNameOnly?: string,
@@ -66,13 +67,13 @@ interface Taxon extends WorkingTaxon {
     scientificName: string,
     taxonRank: Rank,
     collectionCode: ResourceId,
-    taxonomicStatus: string
+    taxonomicStatus: string,
+    verbatimIdentification: string
 }
 
 interface ResourceMetadata {
     levels: Rank[],
-    scope: string[],
-    catalog?: object
+    catalog?: Record<string, Value>
 }
 
 interface Resource {
@@ -123,3 +124,14 @@ interface TaxonMatch {
 }
 
 type GroupedNameMatches = Record<string, Record<string, Record<TaxonId, TaxonMatch>>>
+
+declare enum ResourceProcessorSource {
+    All = 'all',
+    Unprocessed = 'unprocessed',
+    Modified = 'modified'
+}
+
+interface ResourceProcessorConfig {
+    update: boolean,
+    updateMappings: boolean
+}