@larsgw/formica 0.9.2 → 0.10.1

package/src/bin/validate-catalog.ts

@@ -1,122 +0,0 @@
-#!/usr/bin/env node
-
-import { promises as fs } from 'fs'
-import * as path from 'path'
-import { catalog } from '../index'
-
-class TaxaValidator {
-    taxa: catalog.Entity[];
-    errors: WorkError[];
-    parentIndex: Record<string, string>;
-
-    constructor (taxa: catalog.Entities) {
-        this.taxa = Array.from(taxa).filter(taxon => taxon.has('ancestors_gbif'))
-        this.errors = []
-        this.parentIndex = {}
-    }
-
-    validate (): WorkError[] {
-        for (const taxon of this.taxa) {
-            const parents = taxon.get('ancestors_gbif') as string[]
-
-            if (parents.length > 1) {
-                const id = taxon.get('id') as string
-
-                for (let i = 1; i < parents.length; i++) {
-                    this.addToIndex(id, 'ancestors_gbif', parents[i], parents[i - 1])
-                }
-            }
-        }
-
-        for (const taxon of this.taxa) {
-            if (taxon.has('gbif')) {
-                const id = taxon.get('id') as string
-                const parents = taxon.get('ancestors_gbif') as string[]
-                const parent = parents[parents.length - 1] as string
-
-                this.addToIndex(id, 'gbif', taxon.get('gbif') as string, parent)
-            }
-        }
-
-        for (const taxon of this.taxa) {
-            if (taxon.has('children_gbif')) {
-                const id = taxon.get('id') as string
-                const parents = taxon.get('ancestors_gbif') as string[]
-                const parent = parents[parents.length - 1] as string
-
-                const childIds = taxon.get('children_gbif') as string[]
-                for (const childId of childIds) {
-                    this.addToIndex(id, 'gbif', childId, parent)
-                }
-            }
-        }
-
-        return this.errors
-    }
-
-    addToIndex (entity: string, field: string, childId: string, parentId: string) {
-        const actualParentId = this.parentIndex[childId]
-
-        if (!actualParentId) {
-            this.parentIndex[childId] = parentId
-        } else if (actualParentId !== parentId) {
-            const error = `Inconsistent ancestor of ${childId}, expected ${actualParentId} but got ${parentId}`
-            this.errors.push({ entity, field, error })
-        }
-    }
-}
-
-async function validateFile (arg: string): Promise<WorkError[]> {
-    const filePath = path.resolve(arg)
-    const file = await fs.readFile(filePath, 'utf8')
-    const sheet = path.basename(filePath, '.csv')
-    const entities = catalog.loadData(file, sheet)
-    const errors = entities.validate()
-
-    const ids = new Set()
-    for (const entity of entities) {
-        const id = entity.get('id') as string
-
-        if (ids.has(id)) {
-            errors.push({
-                entity: id,
-                field: 'id',
-                error: `ID "${id}" repeated`
-            })
-        } else {
-            ids.add(id)
-        }
-    }
-
-    if (sheet === 'taxa') {
-        const validator = new TaxaValidator(entities)
-        errors.push(...validator.validate())
-    }
-
-    return errors
-}
-
-async function main (args: string[]): Promise<void> {
-    let exitStatus = 0
-
-    const results = await Promise.allSettled(args.map(validateFile))
-    for (let i = 0; i < results.length; i++) {
-        const result = results[i]
-
-        if (result.status === 'rejected') {
-            console.error(`${args[i]}:`)
-            console.error(result.reason)
-            console.error()
-            exitStatus = 1
-        } else if (result.value.length > 0) {
-            console.error(`${args[i]}:`)
-            console.table(result.value)
-            console.error()
-            exitStatus = 1
-        }
-    }
-
-    process.exit(exitStatus)
-}
-
-main(process.argv.slice(2))

package/src/bin/validate-resources-text.ts

@@ -1,25 +0,0 @@
-#!/usr/bin/env node
-
-import { promises as fs } from 'fs'
-import * as path from 'path'
-import { resources } from '../index'
-
-async function main (args: string[]): Promise<void> {
-    let exitStatus = 0
-
-    for (const arg of args) {
-        const filePath = path.resolve(arg)
-        const file = await fs.readFile(filePath, 'utf8')
-        const id = path.basename(filePath, '.txt')
-        try {
-            resources.parseTextFile(file, id)
-        } catch (error) {
-            console.error(filePath + '\n' + error.message.replace(/^/gm, '    ') + '\n')
-            exitStatus = 1
-        }
-    }
-
-    process.exit(exitStatus)
-}
-
-main(process.argv.slice(2))

package/src/catalog/entities.ts

@@ -1,62 +0,0 @@
-import { Entity } from './entity'
-
-export class Entities {
-    entities: Entity[];
-    indexField: string;
-    index: Record<string, Entity>;
-
-    constructor (entities: Entity[], indexField: string) {
-        this.entities = entities
-        this.indexField = indexField
-        this.index = {}
-        for (const entity of this.entities) {
-            const key = entity.get(indexField)
-            if (typeof key !== 'string') {
-                throw new TypeError('Entity missing index field')
-            }
-            this.index[key] = entity
-        }
-    }
-
-    *[Symbol.iterator] () {
-        for (const entity of this.entities) {
-            yield entity
-        }
-    }
-
-    get (key: string): Entity {
-        return this.index[key]
-    }
-
-    has (key: string): boolean {
-        return key in this.index
-    }
-
-    validate (): WorkError[] {
-        const errors: WorkError[] = []
-        for (const entity of this.entities) {
-            for (const { field, error } of entity.validate()) {
-                errors.push({
-                    entity: (entity.get(this.indexField) || '[missing]') as string,
-                    field,
-                    error
-                })
-            }
-        }
-        return errors
-    }
-
-    toTable (): string[][] {
-        if (this.entities.length === 0) {
-            return [[]]
-        }
-        const header = Object.keys(this.entities[0].schema)
-        const table = this.entities.map((entity: Entity) => {
-            return header.map((field: string): string => {
-                const value = entity.get(field) || ''
-                return Array.isArray(value) ? value.join('; ') : value
-            })
-        })
-        return [header, ...table]
-    }
-}

package/src/catalog/entity.ts

@@ -1,116 +0,0 @@
-import { parseValue } from './value'
-
-export class Entity {
-    fields: Record<string, Value>;
-    schema: Schema;
-    derivedFields: Record<string, Value>;
-
-    constructor (values: Record<string, string>, schema: Schema) {
-        this.fields = {}
-        for (const key in values) {
-            const value = parseValue(values[key], !!schema[key] && !!schema[key].multiple)
-            if (value !== null) {
-                this.fields[key] = value
-            }
-        }
-
-        this.schema = schema
-        this.derivedFields = {}
-    }
-
-    deriveFields () {
-    }
-
-    get (key: string): Value | undefined {
-        if (key in this.fields) {
-            return this.fields[key]
-        } else if (key in this.derivedFields) {
-            return this.derivedFields[key]
-        }
-    }
-
-    has (key: string): boolean {
-        return key in this.fields || key in this.derivedFields
-    }
-
-    validate (): FieldError[] {
-        const errors = []
-        for (const field in { ...this.fields, ...this.schema }) {
-            for (const error of this._validateField(field)) {
-                errors.push({ field, error })
-            }
-        }
-        return errors
-    }
-
-    _validateField (field: string) {
-        const errors = []
-
-        if (!(field in this.schema)) {
-            errors.push('Field unknown')
-            return errors
-        }
-
-        if (!(field in this.fields)) {
-            if (this.schema[field].required) {
-                errors.push('Value(s) required but missing')
-            }
-            return errors
-        }
-
-        const valueCountError = this._validateValueCount(field, this.fields[field])
-        if (valueCountError !== undefined) {
-            errors.push(valueCountError)
-        }
-
-        let values = this.fields[field]
-        if (!Array.isArray(values)) {
-            values = [values]
-        }
-
-        for (const value of values) {
-            const result = this._validateSingleValue(field, value)
-            if (result !== undefined) {
-                errors.push(result)
-            }
-        }
-
-        return errors
-    }
-
-    _validateValueCount (field: string, value: Value): string | undefined {
-        const multipleConfig = this.schema[field].multiple
-        const allowMultiple = typeof multipleConfig === 'function'
-            ? multipleConfig.call(null, this.fields)
-            : multipleConfig
-
-        if (allowMultiple === false) {
-            let multiple = false
-
-            if (Array.isArray(value) && value.length > 2) {
-                multiple = true
-            } else if (typeof value === 'string' && value.includes('; ')) {
-                multiple = true
-            }
-
-            if (multiple) {
-                return 'Multiple values but only one expected'
-            }
-        }
-
-        return undefined
-    }
-
-    _validateSingleValue (field: string, value: SingleValue): string | undefined {
-        const config = this.schema[field].format
-        if (config === undefined) {
-            return undefined
-        } else if (Array.isArray(config) && (value === null || !config.includes(value))) {
-            return `The value "${value}" is not included: ${config.join(', ')}`
-        } else if (config instanceof RegExp && !config.test(value)) {
-            return `The value "${value}" does not conform to pattern: ${config.source}`
-        } else if (typeof config === 'function' && !config.call(null, value)) {
-            return `The value "${value}" does not conform to pattern: ${config.name}`
-        }
-    }
-}

package/src/catalog/index.ts

@@ -1,33 +0,0 @@
-import { Entities } from './entities'
-import { Entity } from './entity'
-import { TYPE_INFO } from './tables/index'
-import { parseCsv } from '../csv'
-
-function getTypeInfo (type: string): [typeof Entity, string] {
-    switch (type) {
-        case 'authors': return TYPE_INFO.authors
-        case 'places': return TYPE_INFO.places
-        case 'publishers': return TYPE_INFO.publishers
-        case 'taxa': return TYPE_INFO.taxa
-        case 'catalog': return TYPE_INFO.catalog
-        default: throw new TypeError(`Unknown type "${type}"`)
-    }
-}
-
-export { Entities, Entity, getTypeInfo }
-
-export function loadData (file: string, type: string): Entities {
-    const [subClass, indexField] = getTypeInfo(type)
-    const entities = []
-    const [header, ...rows] = parseCsv(file)
-
-    for (const row of rows) {
-        const data: Record<string, string> = {}
-        for (let index = 0; index < header.length && index < row.length; index++) {
-            data[header[index]] = row[index]
-        }
-        entities.push(new subClass(data, {}))
-    }
-
-    return new Entities(entities, indexField)
-}

package/src/catalog/tables/author.ts

@@ -1,15 +0,0 @@
-import { FORMATS } from '../value'
-import { Entity } from '../entity'
-
-export class Author extends Entity {
-    constructor (values: Record<string, string>) {
-        super(values, {
-            name: { required: true, multiple: true },
-            id: { required: true, multiple: false, format: FORMATS.AUTHOR_ID },
-            qid: { required: false, multiple: false, format: FORMATS.QID },
-            display_name: { required: true, multiple: false },
-            full_names: { required: false, multiple: true },
-            duplicate_of: { required: false, multiple: false, format: FORMATS.AUTHOR_ID }
-        })
-    }
-}

package/src/catalog/tables/index.ts

@@ -1,14 +0,0 @@
-import { Entity } from '../entity'
-import { Author } from './author'
-import { Publisher } from './publisher'
-import { Place } from './place'
-import { Taxon } from './taxon'
-import { Work } from './work'
-
-export const TYPE_INFO: Record<string, [typeof Entity, string]> = {
-    authors: [Author, 'id'],
-    publishers: [Publisher, 'id'],
-    places: [Place, 'id'],
-    taxa: [Taxon, 'id'],
-    catalog: [Work, 'id']
-}

package/src/catalog/tables/place.ts

@@ -1,14 +0,0 @@
-import { FORMATS } from '../value'
-import { Entity } from '../entity'
-
-export class Place extends Entity {
-    constructor (values: Record<string, string>) {
-        super(values, {
-            name: { required: true, multiple: false },
-            id: { required: true, multiple: false, format: FORMATS.PLACE_ID },
-            qid: { required: false, multiple: false, format: FORMATS.QID },
-            display_name: { required: true, multiple: false },
-            duplicate_of: { required: false, multiple: false, format: FORMATS.PLACE_ID }
-        })
-    }
-}

package/src/catalog/tables/publisher.ts

@@ -1,15 +0,0 @@
-import { FORMATS } from '../value'
-import { Entity } from '../entity'
-
-export class Publisher extends Entity {
-    constructor (values: Record<string, string>) {
-        super(values, {
-            name: { required: true, multiple: true },
-            id: { required: true, multiple: false, format: FORMATS.PUBLISHER_ID },
-            qid: { required: false, multiple: false, format: FORMATS.QID },
-            display_name: { required: true, multiple: false },
-            full_names: { required: false, multiple: true },
-            duplicate_of: { required: false, multiple: false, format: FORMATS.PUBLISHER_ID }
-        })
-    }
-}

package/src/catalog/tables/taxon.ts

@@ -1,17 +0,0 @@
-import { FORMATS } from '../value'
-import { Entity } from '../entity'
-
-export class Taxon extends Entity {
-    constructor (values: Record<string, string>) {
-        super(values, {
-            name: { required: true, multiple: false },
-            id: { required: true, multiple: false, format: FORMATS.TAXON_ID },
-            qid: { required: false, multiple: false, format: FORMATS.QID },
-            rank: { required: false, multiple: false },
-            gbif: { required: false, multiple: false, format: FORMATS.INTEGER },
-            children_gbif: { required: false, multiple: true, format: FORMATS.INTEGER },
-            ancestors_gbif: { required: false, multiple: true, format: FORMATS.INTEGER },
-            duplicate_of: { required: false, multiple: false, format: FORMATS.TAXON_ID }
-        })
-    }
-}

package/src/catalog/tables/work.ts

@@ -1,65 +0,0 @@
-import { FORMATS, CHECK } from '../value'
-import { Entity } from '../entity'
-
-export class Work extends Entity {
-    constructor (values: Record<string, string>) {
-        super(values, {
-            id: { required: true, multiple: false, format: FORMATS.WORK_ID },
-            title: { required: true, multiple: CHECK.MULTILANG },
-            author: { required: false, multiple: true },
-            url: { required: false, multiple: true, format: FORMATS.URL },
-            fulltext_url: { required: false, multiple: true, format: FORMATS.URL },
-            archive_url: { required: false, multiple: true, format: FORMATS.URL },
-            entry_type: { required: true, multiple: false, format: FORMATS.ENTRY_TYPE },
-            date: { required: false, multiple: false, format: FORMATS.EDTF_0 },
-            publisher: { required: false, multiple: true },
-            series: { required: false, multiple: false },
-            ISSN: { required: false, multiple: false, format: FORMATS.ISSN_L },
-            ISBN: { required: false, multiple: CHECK.ISBN, format: FORMATS.ISBN },
-            DOI: { required: false, multiple: false, format: FORMATS.DOI },
-            QID: { required: false, multiple: false, format: FORMATS.QID },
-            volume: { required: false, multiple: false },
-            issue: { required: false, multiple: false },
-            pages: { required: false, multiple: false },
-            edition: { required: false, multiple: false },
-            language: { required: true, multiple: true, format: FORMATS.LANGUAGE },
-            license: { required: false, multiple: true, format: FORMATS.LICENSE },
-            key_type: { required: true, multiple: true, format: FORMATS.KEY_TYPE },
-            taxon: { required: true, multiple: true },
-            taxon_scope: { required: false, multiple: true },
-            scope: { required: false, multiple: true },
-            region: { required: true, multiple: true },
-            complete: { required: false, multiple: false, format: FORMATS.COMPLETE },
-            key_characteristics: { required: false, multiple: true },
-            target_taxa: { required: false, multiple: true },
-            listed_in: { required: false, multiple: true, format: FORMATS.WORK_ID },
-            part_of: { required: false, multiple: true, format: FORMATS.WORK_ID },
-            version_of: { required: false, multiple: true, format: FORMATS.WORK_ID },
-            duplicate_of: { required: false, multiple: false, format: FORMATS.WORK_ID }
-        })
-    }
-
-    deriveFields () {
-        if (typeof this.fields.date === 'string') {
-            const year = parseInt(this.fields.date.split('-')[0])
-            this.derivedFields.year = year.toString()
-            this.derivedFields.decade = (year - (year % 10)).toString()
-        }
-
-        if (typeof this.fields.license === 'string' && !this.fields.license.endsWith('?>')) {
-            this.derivedFields.access = 'Open license'
-        } else {
-            const info = this.fields.url
-            const content = this.fields.fulltext_url
-            const archive = this.fields.archive_url
-
-            if (typeof content === 'string') {
-                this.derivedFields.access = 'Full text available, no license'
-            } else if (typeof archive === 'string' && (!(typeof info === 'string') || !archive.endsWith(info))) {
-                this.derivedFields.access = 'Archived full text available, no license'
-            } else {
-                this.derivedFields.access = 'No full text available'
-            }
-        }
-    }
-}

package/src/catalog/value.ts

@@ -1,51 +0,0 @@
-import spdxLicenseList from 'spdx-license-list'
-import ietfTagListFactory from 'ietf-language-tag-regex'
-
-const ietfTagList = ietfTagListFactory()
-
-export const FORMATS = {
-    ENTRY_TYPE: ['print', 'online', 'cd', 'application'],
-    KEY_TYPE: ['key', 'matrix', 'reference', 'gallery', 'checklist', 'supplement', 'collection', 'algorithm'],
-    COMPLETE: ['TRUE', 'FALSE'],
-
-    WORK_ID: /^B[1-9]\d*$/,
-    AUTHOR_ID: /^A[1-9]\d*$/,
-    PLACE_ID: /^G[1-9]\d*$/,
-    PUBLISHER_ID: /^P[1-9]\d*$/,
-    TAXON_ID: /^T[1-9]\d*$/,
-    EDTF_0: /^(\d{4}(-\d{2}(-\d{2}(T\d{2}:\d{2}:\d{2}(Z|[-+]\d{2}(:\d{2})?))?)?)?|\d{4}(-\d{2}(-\d{2})?)?\/(\d{4}(-\d{2}(-\d{2})?)?|\.\.))$/,
-    ISSN_L: /^[0-9]{4}-[0-9]{3}[0-9X]$/,
-    ISBN: /^(\d{13}|\d{9}[0-9X])$/,
-    DOI: /^10\./,
-    QID: /^Q[1-9][0-9]*$/,
-    URL: /^(ftp|http|https):\/\/((?:[a-z0-9][a-z0-9-_]*?[a-z0-9]?\.)+(?:xn--)?[a-z0-9]+)(:\d*)?((?:\/(?:%\d\d|[!$&'()*+,\-.0-9";=@A-Z_a-z~])*)*)(\?(?:%\d\d|[!$&'()*+,\-./0-9:;=?@A-Z_a-z~])*)?(#(?:%\d\d|[!$&'()*+,\-./0-9:;=?@A-Z_a-z~])*)?/i,
-    INTEGER: /^(0|[1-9]\d*)$/,
-
-    LICENSE (value: SingleValue) { return /^<(public domain|.+\?)>$/.test(value) || !!spdxLicenseList[value] },
-    LANGUAGE (value: SingleValue) { return ietfTagList.test(value) }
-}
-
-export const CHECK = {
-    MULTILANG (entry: Record<string, Value>) {
-        return Array.isArray(entry.language) && entry.language.length > 1
-    },
-
-    ISBN (entry: Record<string, Value>) {
-        if (!Array.isArray(entry.ISBN) || entry.ISBN.length < 2) { return false }
-        const a = entry.ISBN[0].length === 10
-        const b = entry.ISBN[0].length === 13
-        const c = entry.ISBN[1].length === 10
-        const d = entry.ISBN[1].length === 13
-        return (a && d) || (b && c)
-    }
-}
-
-export function parseValue (value: string, multiple: boolean): Value | null {
-    if (value === '') {
-        return null
-    } else if (multiple) {
-        return value.split('; ')
-    } else {
-        return value
-    }
-}

package/src/csv.ts

@@ -1,33 +0,0 @@
-export function parseCsv (file: string): string[][] {
-    const values = file
-        .trim()
-        .match(/("([^"]|"")*?"|[^,\n]+|(?!$))(,|\n|$)/g)
-
-    if (values === null) {
-        throw new TypeError('Failed to parse csv')
-    }
-
-    return values.reduce((rows, value) => {
-        const last: string[] = rows[rows.length - 1]
-        if (value.endsWith('\n')) {
-            rows.push([])
-        }
-        value = value.replace(/[,\n]$/, '')
-        last.push(value.startsWith('"') ? value.replace(/""/g, '"').slice(1, -1) : value)
-        return rows
-    }, [[]])
-}
-
-export function formatCsv (table: string[][], delimiter = ',') {
-    return table
-        .map((row: string[]) => {
-            return row.map(value => {
-                if (/["\n]/.test(value) || value.includes(delimiter)) {
-                    return `"${value.replace(/"/g, '""')}"`
-                } else {
-                    return value
-                }
-            }).join(delimiter)
-        })
-        .join('\n') + '\n'
-}

package/src/index.ts

@@ -1,4 +0,0 @@
-export * as catalog from './catalog/index'
-export * as resources from './resources/index'
-export * as taxonNames from './taxon-names/index'
-export * as csv from './csv'