@larsgw/formica 0.1.0

package/src/catalog/entity.ts

@@ -0,0 +1,113 @@
+import { parseValue } from './value'
+
+export class Entity {
+    fields: Record<string, Value>;
+    schema: Schema;
+    derivedFields: Record<string, Value>;
+
+    constructor (values: Record<string, string>, schema: Schema) {
+        this.fields = {}
+        for (const key in values) {
+            const value = parseValue(values[key], !!schema[key].multiple)
+            if (value !== null) {
+                this.fields[key] = value
+            }
+        }
+
+        this.schema = schema
+        this.derivedFields = {}
+    }
+
+    /* eslint-disable @typescript-eslint/no-empty-function */
+    deriveFields () {
+    }
+    /* eslint-enable @typescript-eslint/no-empty-function */
+
+    get (key: string): Value | undefined {
+        if (key in this.fields) {
+            return this.fields[key]
+        } else if (key in this.derivedFields) {
+            return this.derivedFields[key]
+        }
+    }
+
+    has (key: string): boolean {
+        return key in this.fields || key in this.derivedFields
+    }
+
+    validate (): FieldError[] {
+        const errors = []
+        for (const field in this.fields) {
+            for (const error of this._validateField(field)) {
+                errors.push({ field, error })
+            }
+        }
+        return errors
+    }
+
+    _validateField (field: string) {
+        const errors = []
+
+        if (!(field in this.fields)) {
+            if (this.schema[field].required) {
+                errors.push('Value(s) required but missing')
+            }
+            return errors
+        }
+
+        const valueCountError = this._validateValueCount(field, this.fields[field])
+        if (valueCountError !== undefined) {
+            errors.push(valueCountError)
+        }
+
+        let values = this.fields[field]
+        if (!Array.isArray(values)) {
+            values = [values]
+        }
+
+        for (const value of values) {
+            const result = this._validateSingleValue(field, value)
+            if (result !== undefined) {
+                errors.push(result)
+            }
+        }
+
+        return errors
+    }
+
+    _validateValueCount (field: string, value: Value): string | undefined {
+        const multipleConfig = this.schema[field].multiple
+        const allowMultiple = typeof multipleConfig === 'function'
+            ? multipleConfig.call(null, this.fields)
+            : multipleConfig
+
+        if (allowMultiple === false) {
+            let multiple = false
+
+            if (Array.isArray(value) && value.length > 2) {
+                multiple = true
+            } else if (typeof value === 'string' && value.includes('; ')) {
+                multiple = true
+            }
+
+            if (multiple) {
+                return 'Multiple values but only one expected'
+            }
+        }
+
+        return undefined
+    }
+
+    _validateSingleValue (field: string, value: SingleValue): string | undefined {
+        const config = this.schema[field].format
+        if (config === undefined) {
+            return undefined
+        } else if (Array.isArray(config) && (value === null || !config.includes(value))) {
+            return `The value "${value}" is not included: ${config.join(', ')}`
+        } else if (config instanceof RegExp && !config.test(value)) {
+            return `The value "${value}" does not conform to pattern: ${config.source}`
+        } else if (typeof config === 'function' && !config.call(null, value)) {
+            return `The value "${value}" does not conform to pattern: ${config.name}`
+        }
+    }
+}

package/src/catalog/index.ts

@@ -0,0 +1,32 @@
+import { Entities } from './entities'
+import { Entity } from './entity'
+import { TYPE_INFO } from './tables/index'
+import { parseCsv } from '../csv'
+
+function getTypeInfo (type: string): [typeof Entity, string] {
+    switch (type) {
+        case 'authors': return TYPE_INFO.authors
+        case 'places': return TYPE_INFO.places
+        case 'publishers': return TYPE_INFO.publishers
+        case 'catalog': return TYPE_INFO.catalog
+        default: throw new TypeError(`Unknown type "${type}"`)
+    }
+}
+
+export { Entities, Entity }
+
+export function loadData (file: string, type: string): Entities {
+    const [subClass, indexField] = getTypeInfo(type)
+    const entities = []
+    const [header, ...rows] = parseCsv(file)
+
+    for (const row of rows) {
+        const data: Record<string, string> = {}
+        for (let index = 0; index < header.length && index < row.length; index++) {
+            data[header[index]] = row[index]
+        }
+        entities.push(new subClass(data, {}))
+    }
+
+    return new Entities(entities, indexField)
+}

package/src/catalog/tables/author.ts

@@ -0,0 +1,13 @@
+import { FORMATS } from '../value'
+import { Entity } from '../entity'
+
+export class Author extends Entity {
+    constructor (values: Record<string, string>) {
+        super(values, {
+            name: { required: true, multiple: false },
+            qid: { required: false, multiple: false, format: FORMATS.QID },
+            main_full_name: { required: false, multiple: false },
+            full_names: { required: false, multiple: true }
+        })
+    }
+}

package/src/catalog/tables/index.ts

@@ -0,0 +1,12 @@
+import { Entity } from '../entity'
+import { Author } from './author'
+import { Publisher } from './publisher'
+import { Place } from './place'
+import { Work } from './work'
+
+export const TYPE_INFO: Record<string, [typeof Entity, string]> = {
+    authors: [Author, 'name'],
+    publishers: [Publisher, 'name'],
+    places: [Place, 'name'],
+    catalog: [Work, 'id']
+}

package/src/catalog/tables/place.ts

@@ -0,0 +1,12 @@
+import { FORMATS } from '../value'
+import { Entity } from '../entity'
+
+export class Place extends Entity {
+    constructor (values: Record<string, string>) {
+        super(values, {
+            name: { required: true, multiple: false },
+            qid: { required: false, multiple: false, format: FORMATS.QID },
+            display_name: { required: false, multiple: false }
+        })
+    }
+}

package/src/catalog/tables/publisher.ts

@@ -0,0 +1,13 @@
+import { FORMATS } from '../value'
+import { Entity } from '../entity'
+
+export class Publisher extends Entity {
+    constructor (values: Record<string, string>) {
+        super(values, {
+            name: { required: true, multiple: false },
+            qid: { required: false, multiple: false, format: FORMATS.QID },
+            full_name: { required: false, multiple: false },
+            long_name: { required: false, multiple: true }
+        })
+    }
+}

package/src/catalog/tables/work.ts

@@ -0,0 +1,62 @@
+import { FORMATS, CHECK } from '../value'
+import { Entity } from '../entity'
+
+export class Work extends Entity {
+    constructor (values: Record<string, string>) {
+        super(values, {
+            id: { required: true, multiple: false, format: FORMATS.ID },
+            title: { required: true, multiple: CHECK.MULTILANG },
+            author: { required: false, multiple: true },
+            url: { required: false, multiple: true, format: FORMATS.URL },
+            fulltext_url: { required: false, multiple: true, format: FORMATS.URL },
+            archive_url: { required: false, multiple: true, format: FORMATS.URL },
+            entry_type: { required: true, multiple: false, format: FORMATS.ENTRY_TYPE },
+            date: { required: false, multiple: false, format: FORMATS.EDTF_0 },
+            publisher: { required: false, multiple: true },
+            series: { required: false, multiple: false },
+            ISSN: { required: false, multiple: false, format: FORMATS.ISSN_L },
+            ISBN: { required: false, multiple: CHECK.ISBN, format: FORMATS.ISBN },
+            DOI: { required: false, multiple: false, format: FORMATS.DOI },
+            QID: { required: false, multiple: false, format: FORMATS.QID },
+            volume: { required: false, multiple: false },
+            issue: { required: false, multiple: false },
+            pages: { required: false, multiple: false },
+            edition: { required: false, multiple: false },
+            language: { required: true, multiple: true, format: FORMATS.LANGUAGE },
+            license: { required: false, multiple: true, format: FORMATS.LICENSE },
+            key_type: { required: true, multiple: true, format: FORMATS.KEY_TYPE },
+            taxon: { required: true, multiple: true },
+            scope: { required: false, multiple: true },
+            region: { required: true, multiple: true },
+            complete: { required: false, multiple: false, format: FORMATS.COMPLETE },
+            target_taxa: { required: false, multiple: true },
+            listed_in: { required: false, multiple: true, format: FORMATS.ID },
+            part_of: { required: false, multiple: true, format: FORMATS.ID },
+            version_of: { required: false, multiple: true, format: FORMATS.ID }
+        })
+    }
+
+    deriveFields () {
+        if (typeof this.fields.date === 'string') {
+            const year = parseInt(this.fields.date.split('-')[0])
+            this.derivedFields.year = year.toString()
+            this.derivedFields.decade = (year - (year % 10)).toString()
+        }
+
+        if (typeof this.fields.license === 'string' && !this.fields.license.endsWith('?>')) {
+            this.derivedFields.access = 'Open license'
+        } else {
+            const info = this.fields.url
+            const content = this.fields.fulltext_url
+            const archive = this.fields.archive_url
+
+            if (typeof content === 'string') {
+                this.derivedFields.access = 'Full text available, no license'
+            } else if (typeof archive === 'string' && (!(typeof info === 'string') || !archive.endsWith(info))) {
+                this.derivedFields.access = 'Archived full text available, no license'
+            } else {
+                this.derivedFields.access = 'No full text available'
+            }
+        }
+    }
+}

package/src/catalog/value.ts

@@ -0,0 +1,48 @@
+import spdxLicenseList = require('spdx-license-list')
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import ietfTagListFactory = require('ietf-language-tag-regex')
+
+const ietfTagList = ietfTagListFactory()
+
+export const FORMATS = {
+    ENTRY_TYPE: ['print', 'online', 'cd'],
+    KEY_TYPE: ['key', 'matrix', 'reference', 'gallery', 'checklist', 'supplement', 'collection'],
+    COMPLETE: ['TRUE', 'FALSE'],
+
+    ID: /^B\d+$/,
+    EDTF_0: /^(\d{4}(-\d{2}(-\d{2}(T\d{2}:\d{2}:\d{2}(Z|[-+]\d{2}(:\d{2})?))?)?)?|\d{4}(-\d{2}(-\d{2})?)?\/(\d{4}(-\d{2}(-\d{2})?)?|\.\.))$/,
+    ISSN_L: /^[0-9]{4}-[0-9]{3}[0-9X]$/,
+    ISBN: /^(\d{13}|\d{9}[0-9X])$/,
+    DOI: /^10\./,
+    QID: /^Q[1-9][0-9]*$/,
+    URL: /^(ftp|http|https):\/\/((?:[a-z0-9][a-z0-9-_]*?[a-z0-9]?\.)+(?:xn--)?[a-z0-9]+)(:\d*)?((?:\/(?:%\d\d|[!$&'()*+,\-.0-9";=@A-Z_a-z~])*)*)(\?(?:%\d\d|[!$&'()*+,\-./0-9:;=?@A-Z_a-z~])*)?(#(?:%\d\d|[!$&'()*+,\-./0-9:;=?@A-Z_a-z~])*)?/i,
+
+    LICENSE (value: SingleValue) { return /^<(public domain|.+\?)>$/.test(value) || !!spdxLicenseList[value] },
+    LANGUAGE (value: SingleValue) { return ietfTagList.test(value) }
+}
+
+export const CHECK = {
+    MULTILANG (entry: Record<string, Value>) {
+        return entry.language.length > 1
+    },
+
+    ISBN (entry: Record<string, Value>) {
+        if (entry.ISBN.length < 2) { return false }
+        const a = entry.ISBN[0].length === 10
+        const b = entry.ISBN[0].length === 13
+        const c = entry.ISBN[1].length === 10
+        const d = entry.ISBN[1].length === 13
+        return (a && d) || (b && c)
+    }
+}
+
+export function parseValue (value: string, multiple: boolean): Value | null {
+    if (value === '') {
+        return null
+    } else if (multiple) {
+        return value.split('; ')
+    } else {
+        return value
+    }
+}

package/src/csv.ts

@@ -0,0 +1,33 @@
+export function parseCsv (file: string): string[][] {
+    const values = file
+        .trim()
+        .match(/("([^"]|"")*?"|[^,\n]+|(?!$))(,|\n|$)/g)
+
+    if (values === null) {
+        throw new TypeError('Failed to parse csv')
+    }
+
+    return values.reduce((rows, value) => {
+        const last: string[] = rows[rows.length - 1]
+        if (value.endsWith('\n')) {
+            rows.push([])
+        }
+        value = value.replace(/[,\n]$/, '')
+        last.push(value.startsWith('"') ? value.replace(/""/g, '"').slice(1, -1) : value)
+        return rows
+    }, [[]])
+}
+
+export function formatCsv (table: string[][], delimiter = ',') {
+    return table
+        .map((row: string[]) => {
+            return row.map(value => {
+                if (/["\n]/.test(value) || value.includes(delimiter)) {
+                    return `"${value.replace(/"/, '"""')}"`
+                } else {
+                    return value
+                }
+            }).join(delimiter)
+        })
+        .join('\n') + '\n'
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export * as catalog from './catalog/index'
+export * as resources from './resources/index'
+export * as csv from './csv'

package/src/module.d.ts

@@ -0,0 +1,105 @@
+type Schema = Record<string, FieldSpecification>
+type FieldSpecification = {
+  required: boolean,
+  multiple: boolean | FieldSpecificationCallbackMultiple
+  format?: string[] | RegExp | FieldSpecificationCallbackFormat
+}
+
+type Value = string[] | string
+type SingleValue = string
+type FieldSpecificationCallbackMultiple = (entry: Record<string, Value>) => boolean;
+type FieldSpecificationCallbackFormat = (value: SingleValue) => boolean;
+
+interface FieldError {
+  field: string,
+  error: string
+}
+
+interface WorkError extends FieldError {
+  entity: WorkId
+}
+
+type Rank = string
+type DwcRank = Rank
+type TaxonStatus = string
+
+type TaxonId = string
+type ResourceId = string
+type WorkId = string
+
+interface WorkingTaxon {
+  scientificNameID?: TaxonId,
+  scientificName?: string,
+  scientificNameAuthorship?: string,
+  genericName?: string,
+  infragenericEpithet?: string,
+  specificEpithet?: string,
+  intraspecificEpithet?: string,
+
+  taxonRank?: Rank,
+  taxonRemarks?: string,
+  collectionCode?: ResourceId,
+
+  taxonomicStatus?: TaxonStatus,
+  acceptedNameUsageID?: TaxonId,
+  acceptedNameUsage?: string,
+
+  parentNameUsageID?: TaxonId,
+  parentNameUsage?: string,
+  kingdom?: string,
+  phylum?: string,
+  class?: string,
+  order?: string,
+  family?: string,
+  subfamily?: string,
+  genus?: string,
+  subgenus?: string,
+  higherClassification?: string,
+
+  // Non-standard
+  scientificNameOnly?: string,
+  incorrect?: WorkingTaxon
+}
+
+interface Taxon extends WorkingTaxon {
+  scientificNameID: TaxonId,
+  scientificName: string,
+  taxonRank: Rank,
+  collectionCode: ResourceId,
+  taxonomicStatus: string
+}
+
+interface ResourceMetadata {
+  levels: Rank[],
+  scope: string[],
+  catalog?: object
+}
+
+interface Resource {
+  id: string,
+  file: string,
+  workId: string,
+  metadata: ResourceMetadata,
+  taxa: Record<TaxonId, Taxon>
+}
+
+interface ResourceHistory {
+    txt: string,
+    dwc: Array<string[][]>
+}
+
+type ResourceDiff = ResourceDiffPart[]
+
+interface ResourceDiffPart {
+    text: string,
+    type: ResourceDiffType
+}
+
+type ResourceDiffTokenizer = (text: string) => string[]
+
+declare enum ResourceDiffType {
+    Added = '+',
+    Deleted = '-',
+    Modified = '~',
+    Unchanged = '='
+}

package/src/resources/diff-resource.ts

@@ -0,0 +1,155 @@
+export enum ResourceDiffType {
+    Added = '+',
+    Deleted = '-',
+    Modified = '~',
+    Unchanged = '='
+}
+
+function LCS (X: string[], Y: string[]): ResourceDiffPart[] {
+    const m = X.length
+    const n = Y.length
+
+    // Build matrix
+    const C = Array(m + 1).fill(0).map(() => Array(n + 1).fill(0))
+    for (let i = 0; i < m; i++) {
+        for (let j = 0; j < n; j++) {
+            if (X[i] === Y[j]) {
+                C[i + 1][j + 1] = C[i][j] + 1
+            } else {
+                C[i + 1][j + 1] = Math.max(C[i][j + 1], C[i + 1][j])
+            }
+        }
+    }
+
+    // Backtrace
+    const diff = []
+    let i = m
+    let j = n
+    while (i + j !== 0) {
+        if (X[i - 1] === Y[j - 1]) {
+            diff.unshift({
+                text: X[i - 1],
+                type: ResourceDiffType.Unchanged
+            })
+            i--, j--
+        } else if (i !== 0 && (j === 0 || C[i - 1][j] > C[i][j - 1])) {
+            diff.unshift({
+                text: X[i - 1],
+                type: ResourceDiffType.Added
+            })
+            i--
+        } else {
+            diff.unshift({
+                text: Y[j - 1],
+                type: ResourceDiffType.Deleted
+            })
+            j--
+        }
+    }
+
+    return diff
+}
+
+function gitTokenize (text: string): string[] {
+    if (text.length === 0) {
+        return []
+    }
+    return text.match(/\S+|\n|[\r\t\f\v \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+/g) as string[]
+}
+
+export function createDiff (a: string, b: string, tokenize: ResourceDiffTokenizer = gitTokenize): ResourceDiffPart[] {
+    const X = tokenize(a.trimEnd())
+    const Y = tokenize(b.trimEnd())
+
+    // Remove common prefix
+    const prefix = []
+    while (X.length && X[0] === Y[0]) {
+        prefix.push({
+            text: X[0],
+            type: ResourceDiffType.Unchanged
+        })
+        X.shift()
+        Y.shift()
+    }
+
+    // Remove common suffix
+    const suffix = []
+    while (X.length && X[X.length - 1] === Y[Y.length - 1]) {
+        suffix.unshift({
+            text: X[X.length - 1],
+            type: ResourceDiffType.Unchanged
+        })
+        X.pop()
+        Y.pop()
+    }
+
+    // Generate word from remains, combine with prefix and suffix, add trailing
+    // newline
+    const changes = [
+        ...prefix,
+        ...LCS(X, Y),
+        ...suffix,
+        { text: '\n', type: ResourceDiffType.Unchanged }
+    ]
+
+    // Convert word diff to line diff
+    const lines: ResourceDiff = []
+    let line = null
+    let deletedNewlines = 0
+    let nextLineNew = false
+
+    for (const change of changes) {
+        // Start of line
+        if (line === null) {
+            deletedNewlines = 0
+            line = { text: '', type: change.type }
+        }
+
+        // End of line (could be same token)
+        if (change.text === '\n') {
+            if (change.type === ResourceDiffType.Deleted) {
+                // Keep track of deleted newlines. These indicate the merging of
+                // lines. To keep the taxon identifiers consistent, it is
+                // important that deleted taxa are tracked.
+                deletedNewlines++
+                continue
+            }
+
+            // If there is a new newline, there is a new... line. If the current
+            // line is not seen as new, the next one should be marked as new.
+            if (nextLineNew) {
+                line.type = ResourceDiffType.Added
+                nextLineNew = false
+            }
+            if (change.type === ResourceDiffType.Added && line.type !== ResourceDiffType.Added) {
+                nextLineNew = true
+            }
+
+            // Line ended
+            lines.push(line)
+            line = null
+
+            // Add placeholders for deleted lines
+            while (deletedNewlines--) {
+                lines.push({
+                    text: '',
+                    type: ResourceDiffType.Deleted
+                })
+            }
+
+            continue
+        }
+
+        // Added/Deleted/Unchanged on the same line -> line is modified but did
+        // already exist.
+        if (change.type !== line.type) {
+            line.type = ResourceDiffType.Modified
+        }
+
+        if (change.type !== ResourceDiffType.Deleted) {
+            line.text += change.text
+        }
+    }
+
+    return lines
+}

package/src/resources/index.ts

@@ -0,0 +1,4 @@
+export {
+    parseFile as parseTextFile,
+    parseFileHeader as parseTextFileHeader
+} from './parse-text'