@larsgw/formica 0.8.4 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +1 -1
- package/CHANGELOG.md +34 -0
- package/lib/bin/generate-linked-data.js +209 -328
- package/lib/bin/process-resources-index.js +59 -130
- package/lib/bin/process-resources.js +303 -524
- package/lib/bin/util.js +21 -56
- package/lib/bin/validate-catalog.js +26 -70
- package/lib/bin/validate-resources-text.js +17 -61
- package/lib/catalog/entities.js +28 -84
- package/lib/catalog/entity.js +36 -50
- package/lib/catalog/index.js +11 -12
- package/lib/catalog/tables/author.js +7 -24
- package/lib/catalog/tables/index.js +5 -5
- package/lib/catalog/tables/place.js +7 -24
- package/lib/catalog/tables/publisher.js +7 -24
- package/lib/catalog/tables/taxon.js +7 -24
- package/lib/catalog/tables/work.js +13 -30
- package/lib/catalog/value.js +11 -11
- package/lib/csv.js +7 -8
- package/lib/resources/diff-resource.js +52 -55
- package/lib/resources/parse-name.d.ts +6 -0
- package/lib/resources/parse-name.js +354 -0
- package/lib/resources/parse-text.js +185 -475
- package/lib/resources/resource.js +7 -25
- package/lib/taxon-names/index.js +15 -20
- package/package.json +2 -1
- package/src/bin/generate-linked-data.ts +8 -5
- package/src/bin/process-resources.ts +1 -1
- package/src/bin/validate-resources-text.ts +1 -1
- package/src/module.d.ts +4 -2
- package/src/resources/diff-resource.ts +19 -15
- package/src/resources/parse-name.ts +379 -0
- package/src/resources/parse-text.ts +150 -440
- package/test/resources.js +161 -17
- package/tsconfig.json +4 -1
|
@@ -1,38 +1,7 @@
|
|
|
1
1
|
import * as yaml from 'js-yaml'
|
|
2
2
|
import { WorkResource } from './resource'
|
|
3
3
|
import { createDiff, ResourceDiffType } from './diff-resource'
|
|
4
|
-
|
|
5
|
-
const RANKS: Rank[] = [
|
|
6
|
-
'phylum',
|
|
7
|
-
'subphylum',
|
|
8
|
-
'class',
|
|
9
|
-
'infraclass',
|
|
10
|
-
'superorder',
|
|
11
|
-
'order',
|
|
12
|
-
'suborder',
|
|
13
|
-
'infraorder',
|
|
14
|
-
'superfamily',
|
|
15
|
-
'family',
|
|
16
|
-
'subfamily',
|
|
17
|
-
'tribe',
|
|
18
|
-
'subtribe',
|
|
19
|
-
'genus',
|
|
20
|
-
'subgenus',
|
|
21
|
-
'section', // not ICZN
|
|
22
|
-
'subsection', // not ICZN
|
|
23
|
-
'series', // not ICZN
|
|
24
|
-
'group',
|
|
25
|
-
'subgroup', // ...
|
|
26
|
-
'aggregate', // not ICZN
|
|
27
|
-
'complex', // not ICZN
|
|
28
|
-
'species',
|
|
29
|
-
'subspecies',
|
|
30
|
-
'variety',
|
|
31
|
-
'form',
|
|
32
|
-
'aberration', // not ICZN
|
|
33
|
-
'race', // not ICZN
|
|
34
|
-
'stirps' // not ICZN
|
|
35
|
-
]
|
|
4
|
+
import { parseName, RANKS, RecoverableSyntaxError } from './parse-name'
|
|
36
5
|
|
|
37
6
|
const MAIN_RANKS: Rank[] = [
|
|
38
7
|
'kingdom',
|
|
@@ -62,313 +31,15 @@ const FLAGS: ResourceFlag[] = [
|
|
|
62
31
|
'MISSING_AUTHORSHIP'
|
|
63
32
|
]
|
|
64
33
|
|
|
65
|
-
const
|
|
66
|
-
|
|
67
|
-
'+': 'heterotypic synonym',
|
|
68
|
-
'=': 'synonym'
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
const RANK_LABELS: Record<Rank, string> = {
|
|
72
|
-
'subspecies': 'subsp.',
|
|
73
|
-
'variety': 'var.',
|
|
74
|
-
'form': 'f.',
|
|
75
|
-
'aberration': 'ab.',
|
|
76
|
-
'race': 'r.',
|
|
77
|
-
'stirps': 'st.'
|
|
78
|
-
}
|
|
34
|
+
const RESOURCE_DELIMITER = '\n\n===\n\n'
|
|
35
|
+
const INDENT = 2
|
|
79
36
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
'r': 'race',
|
|
83
|
-
'ab': 'aberration',
|
|
84
|
-
'f': 'form',
|
|
85
|
-
'var': 'variety',
|
|
86
|
-
'ssp': 'subspecies',
|
|
87
|
-
'subsp': 'subspecies'
|
|
37
|
+
function makeParseError (message: string, line: number, column: number = 1): SyntaxError {
|
|
38
|
+
return new SyntaxError(`[${line}:${column}] ${message}`)
|
|
88
39
|
}
|
|
89
40
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
/**
|
|
93
|
-
* 1. Any number of
|
|
94
|
-
* - capitalized words
|
|
95
|
-
* - "&"
|
|
96
|
-
* - " in "
|
|
97
|
-
* - " ex "
|
|
98
|
-
* - lowercase name particles
|
|
99
|
-
* 2. Followed by a capitalized word
|
|
100
|
-
* 3. Optionally, followed by "et al."
|
|
101
|
-
*/
|
|
102
|
-
const LOWERCASE_NAME_PARTICLES = ['y', 'der', 'den', 'de', 'van', 'von'].join('|')
|
|
103
|
-
const SIMPLE_AUTHOR_PATTERN = '(?:(?:\\p{Lu}\\S*|&|in|ex|' + LOWERCASE_NAME_PARTICLES + ')\\s*)*\\p{Lu}\\S+(?:\\s+et\\s+al\\.)?'
|
|
104
|
-
|
|
105
|
-
const NAME_PATTERN = new RegExp(
|
|
106
|
-
'^' +
|
|
107
|
-
// $1 main name part
|
|
108
|
-
'(\\S+)' +
|
|
109
|
-
// $2 optional author citation
|
|
110
|
-
'(?: ' +
|
|
111
|
-
// but not auct(t)., etc.
|
|
112
|
-
'(?!auctt?\\.|(?:syn|comb|sp|spec|nom|gen|subgen)\\. n(?:ov)?\\.|s(?:ens[.u]|\\.)|in part|partim)' +
|
|
113
|
-
'(' +
|
|
114
|
-
// $2.1 anything in parentheses, followed by optional revising author(s)
|
|
115
|
-
'\\(.+?\\)(?:\\s+' + SIMPLE_AUTHOR_PATTERN + ')?' +
|
|
116
|
-
'|' +
|
|
117
|
-
// $2.2 anything followed by a year
|
|
118
|
-
'.+?\\d{4}\\)?' +
|
|
119
|
-
'|' +
|
|
120
|
-
// $2.3 author(s)
|
|
121
|
-
SIMPLE_AUTHOR_PATTERN +
|
|
122
|
-
'))?' +
|
|
123
|
-
// $3 optional notes
|
|
124
|
-
'(?:,? (.+))?' +
|
|
125
|
-
'$',
|
|
126
|
-
'u'
|
|
127
|
-
)
|
|
128
|
-
|
|
129
|
-
/**
|
|
130
|
-
* Structure
|
|
131
|
-
* $1 genus+subgenus (+ trailing space): (?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?
|
|
132
|
-
* $1.1 genus: ((?:x )?[A-Z]\S+)
|
|
133
|
-
* $1.2 subgenus: (?:\(([A-Z]\S+?)\) )?
|
|
134
|
-
* $2 species: (x [a-z-]+|[a-z-][^\s.]+(?: x [a-z-]+)?|[A-Z][a-z]+_[a-z-]+ x [A-Z][a-z]+_[a-z-]+)
|
|
135
|
-
* $2a: x [a-z-]+
|
|
136
|
-
* $2b hybrid: [a-z-][^\s.]+(?: x [a-z-]+)?
|
|
137
|
-
* $2c intergeneric hybrid: [A-Z][a-z]+_[a-z-]+ x [A-Z][a-z]+_[a-z-]+
|
|
138
|
-
*/
|
|
139
|
-
const BINAME_PATTERN = /^(?:((?:x )?[A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?(x [a-z-]+|[a-z-][^\s.]+(?: x [a-z-]+)?|[A-Z][a-z]+_[a-z-]+ x [A-Z][a-z]+_[a-z-]+)(?= |$)/
|
|
140
|
-
|
|
141
|
-
function compareRanks (a: Rank, b: Rank): number {
|
|
142
|
-
return RANKS.indexOf(a) - RANKS.indexOf(b)
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
function capitalize (name: string): string {
|
|
146
|
-
return name[0].toUpperCase() + name.slice(1).toLowerCase()
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
function capitalizeGenericName (name: string): string {
|
|
150
|
-
if (name[0] === HYBRID_SIGN) {
|
|
151
|
-
return HYBRID_SIGN + capitalize(name.slice(1))
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
return capitalize(name)
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
function isUpperCase (name: string): boolean {
|
|
158
|
-
return name === name.toUpperCase()
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
function getSynonymRank (name: string, rank: Rank): Rank {
|
|
162
|
-
const rest = name.replace(BINAME_PATTERN, '')
|
|
163
|
-
const rankPrefix = rest.match(/^(?: |^)(st|r|ab|f|var|ssp|subsp)\. /)
|
|
164
|
-
if (rankPrefix) {
|
|
165
|
-
return RANK_LABELS_REVERSE[rankPrefix[1]] as string
|
|
166
|
-
} else if (!BINAME_PATTERN.test(name)) {
|
|
167
|
-
return rank
|
|
168
|
-
} else if (/^ (?!sensu)[a-z0-9-]+($| )/.test(rest)) {
|
|
169
|
-
return 'subspecies'
|
|
170
|
-
} else {
|
|
171
|
-
return 'species'
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
function capitalizeAuthors (authors: string): string {
|
|
176
|
-
return authors
|
|
177
|
-
.replace(
|
|
178
|
-
/[^\x00-\x40\x5B-\x60\x7B-\x7F]+/g, // eslint-disable-line no-control-regex
|
|
179
|
-
name => isUpperCase(name) ? capitalize(name) : name
|
|
180
|
-
)
|
|
181
|
-
.replace(/ Y /g, ' y ')
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxon {
|
|
185
|
-
const item = {} as WorkingTaxon
|
|
186
|
-
|
|
187
|
-
// Synonyms have the accepted name usage as 'parent'.
|
|
188
|
-
const isSynonym = /^[+=>] /.test(name)
|
|
189
|
-
if (isSynonym) {
|
|
190
|
-
item.taxonomicStatus = TAXONOMIC_STATUS[name[0]]
|
|
191
|
-
name = name.replace(/^[+=>] (\? ?)?/, '')
|
|
192
|
-
rank = getSynonymRank(name, parent.taxonRank as Rank)
|
|
193
|
-
} else {
|
|
194
|
-
item.taxonomicStatus = 'accepted'
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
// Clusters
|
|
198
|
-
if (/^\[(_|\d+)\] /.test(name)) {
|
|
199
|
-
name = name.replace(/^\[(_|\d+)\] /, '')
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
// Set verbatim identification after subsequent syntax is removed.
|
|
203
|
-
item.verbatimIdentification = name.replace(/(?<=^| )x(?=$| )/g, HYBRID_SIGN).replace(/_/g, ' ')
|
|
204
|
-
|
|
205
|
-
// Parent context is used for parsing and formatting binomial names.
|
|
206
|
-
// For formatting, it needs to match external databases (i.e. be correct).
|
|
207
|
-
// For parsing, it needs to match the current file. If relevant parents
|
|
208
|
-
// (i.e. genus, species) had mistakes that were corrected, the uncorrected
|
|
209
|
-
// genus and species names need to be used.
|
|
210
|
-
const parentContext = {
|
|
211
|
-
genus: parent.genus,
|
|
212
|
-
subgenus: parent.subgenus,
|
|
213
|
-
specificEpithet: parent.specificEpithet,
|
|
214
|
-
incorrect: {
|
|
215
|
-
genus: parent.incorrect && parent.incorrect.genus,
|
|
216
|
-
specificEpithet: parent.incorrect && parent.incorrect.specificEpithet
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
// Both contexts should be amended in the two cases where binomial names
|
|
221
|
-
// are fully used: (1) synonyms and (2) multinomial taxa without parents to
|
|
222
|
-
// provide parts of the name (e.g. bare species without a genus parent, or
|
|
223
|
-
// even subspecies without a species or genus parent).
|
|
224
|
-
if (isSynonym || !parentContext.genus || (compareRanks('species', rank) < 0 && !parentContext.specificEpithet)) {
|
|
225
|
-
const [, genus, subgenus, species] = name.match(BINAME_PATTERN) || []
|
|
226
|
-
if (genus) {
|
|
227
|
-
parentContext.incorrect.genus = genus
|
|
228
|
-
parentContext.genus = capitalizeGenericName(genus.replace(/(^| )x /, HYBRID_SIGN))
|
|
229
|
-
}
|
|
230
|
-
if (subgenus) {
|
|
231
|
-
parentContext.subgenus = capitalize(subgenus)
|
|
232
|
-
} else if (genus) {
|
|
233
|
-
// If a genus is given but no subgenus, remove any existing subgenus
|
|
234
|
-
// from the parent context.
|
|
235
|
-
delete parentContext.subgenus
|
|
236
|
-
}
|
|
237
|
-
if (species && compareRanks('species', rank) < 0) {
|
|
238
|
-
parentContext.incorrect.specificEpithet = species
|
|
239
|
-
parentContext.specificEpithet = species.replace(/(^| )x /, HYBRID_SIGN)
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
// In taxa of group, species or lower, the name should just contain the
|
|
244
|
-
// (infra)specific epithet and the author information & remarks when processing
|
|
245
|
-
// further.
|
|
246
|
-
if (compareRanks('group', rank) <= 0) {
|
|
247
|
-
// Remove genus
|
|
248
|
-
const genus = parentContext.incorrect.genus || parentContext.genus || ''
|
|
249
|
-
if (name[0] === genus[0] && name.toLowerCase().startsWith(genus.toLowerCase() + ' ')) {
|
|
250
|
-
name = name.slice(genus.length + 1)
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
// Remove subgenus
|
|
254
|
-
name = name.replace(/^\(.*?\) /, '')
|
|
255
|
-
|
|
256
|
-
// Infraspecific taxa
|
|
257
|
-
if (compareRanks('species', rank) < 0) {
|
|
258
|
-
// Remove specific epithet
|
|
259
|
-
const species = parentContext.incorrect.specificEpithet || parentContext.specificEpithet || ''
|
|
260
|
-
if (name.startsWith(species + ' ')) {
|
|
261
|
-
name = name.slice(species.length + 1)
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
// Remove rank abbreviations
|
|
265
|
-
name = name.replace(/^(st|r|ab|f|var|ssp|subsp)\. /, '')
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
// Hybrids
|
|
270
|
-
if (rank === 'genus' && name.startsWith('x ')) {
|
|
271
|
-
name = HYBRID_SIGN + name.slice(2)
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
if (rank === 'species' && /(^| )x /.test(name)) {
|
|
275
|
-
name = name.replace(/(^| )x /, HYBRID_SIGN)
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
// Divide the name into the main scientific name (only the epithet for taxa
|
|
279
|
-
// lower than genus), the authorship information, and optionally remarks
|
|
280
|
-
const nameParts = name.match(NAME_PATTERN)
|
|
281
|
-
if (!nameParts) {
|
|
282
|
-
throw new Error(`Taxon "${name}" could not be parsed`)
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
// To encode old names with spaces (e.g. "Orsillus pini canariensis Lindberg, 1953")
|
|
286
|
-
// underscores are used, which are replaced here. This is also used for undescribed
|
|
287
|
-
// species (e.g. "Leiobunum species A") and intergeneric hybrids (e.g. "×Festulpia
|
|
288
|
-
// Festuca rubra × Vulpia bromoides")
|
|
289
|
-
if (nameParts[1].includes('_')) {
|
|
290
|
-
nameParts[1] = nameParts[1].replace(/_/g, ' ')
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
const [_, taxon, citation = '', notes] = nameParts
|
|
294
|
-
item.scientificNameAuthorship = capitalizeAuthors(citation)
|
|
295
|
-
item.taxonRemarks = notes
|
|
296
|
-
item.taxonRank = rank
|
|
297
|
-
|
|
298
|
-
// @ts-expect-error TS1501: This regular expression flag is only available when targeting 'es6' or later.
|
|
299
|
-
if (/[^\p{L}0-9\u{00D7}\- ]/u.test(taxon)) {
|
|
300
|
-
throw new Error(`Taxon name contains unexpected characters: "${taxon}"`)
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
// Validate names and recompose binomial and trinomial names
|
|
304
|
-
if (rank === 'genus') {
|
|
305
|
-
item.scientificName = capitalizeGenericName(taxon)
|
|
306
|
-
if (taxon[0].toUpperCase() !== taxon[0] || (taxon[0] === HYBRID_SIGN && taxon[1].toUpperCase() !== taxon[1])) {
|
|
307
|
-
throw new Error(`Generic epithet should be capitalized: "${taxon}"`)
|
|
308
|
-
}
|
|
309
|
-
} else if (compareRanks('group', rank) > 0) {
|
|
310
|
-
item.scientificName = capitalize(taxon)
|
|
311
|
-
if (taxon[0].toUpperCase() !== taxon[0]) {
|
|
312
|
-
throw new Error(`Taxon name (${rank}) should be capitalized: "${taxon}"`)
|
|
313
|
-
}
|
|
314
|
-
} else if (rank === 'group') {
|
|
315
|
-
item.genericName = parentContext.genus
|
|
316
|
-
item.infragenericEpithet = parentContext.subgenus
|
|
317
|
-
const specificEpithet = taxon.toLowerCase().replace(/(-group)?$/, '')
|
|
318
|
-
item.scientificName = `${item.genericName} ${specificEpithet}-group`
|
|
319
|
-
if (taxon.toLowerCase() !== taxon) {
|
|
320
|
-
console.log(item, taxon)
|
|
321
|
-
throw new Error(`Group name should be lowercase: "${taxon}"`)
|
|
322
|
-
}
|
|
323
|
-
} else if (rank === 'subgroup') {
|
|
324
|
-
item.genericName = parentContext.genus
|
|
325
|
-
item.infragenericEpithet = parentContext.subgenus
|
|
326
|
-
const specificEpithet = taxon.toLowerCase().replace(/(-subgroup)?$/, '')
|
|
327
|
-
item.scientificName = `${item.genericName} ${specificEpithet}-subgroup`
|
|
328
|
-
if (taxon.toLowerCase() !== taxon) {
|
|
329
|
-
console.log(item, taxon)
|
|
330
|
-
throw new Error(`Subgroup name should be lowercase: "${taxon}"`)
|
|
331
|
-
}
|
|
332
|
-
} else if (rank === 'species') {
|
|
333
|
-
item.genericName = parentContext.genus
|
|
334
|
-
item.infragenericEpithet = parentContext.subgenus
|
|
335
|
-
if (taxon.toLowerCase() !== taxon && !/^[A-Z][a-z]+ [a-z]+\xD7[A-Z][a-z]+ [a-z]+$/.test(taxon)) {
|
|
336
|
-
console.log(item, taxon)
|
|
337
|
-
throw new Error(`Specific epithet should be lowercase: "${taxon}"`)
|
|
338
|
-
}
|
|
339
|
-
item.specificEpithet = taxon
|
|
340
|
-
item.scientificName = `${item.genericName} ${item.specificEpithet}`
|
|
341
|
-
} else if (compareRanks('species', rank) < 0) {
|
|
342
|
-
item.genericName = parentContext.genus
|
|
343
|
-
item.infragenericEpithet = parentContext.subgenus
|
|
344
|
-
item.specificEpithet = parentContext.specificEpithet
|
|
345
|
-
item.infraspecificEpithet = taxon.toLowerCase()
|
|
346
|
-
|
|
347
|
-
// If possible, names below species should have abbreviations for ranks,
|
|
348
|
-
// like "subsp."
|
|
349
|
-
const nameParts = [
|
|
350
|
-
item.genericName,
|
|
351
|
-
item.specificEpithet,
|
|
352
|
-
item.infraspecificEpithet
|
|
353
|
-
]
|
|
354
|
-
if (item.taxonRank in RANK_LABELS) {
|
|
355
|
-
nameParts.splice(2, 0, RANK_LABELS[item.taxonRank])
|
|
356
|
-
}
|
|
357
|
-
item.scientificName = nameParts.join(' ')
|
|
358
|
-
|
|
359
|
-
if (item.infraspecificEpithet !== taxon) {
|
|
360
|
-
console.log(item, taxon)
|
|
361
|
-
throw new Error(`Infraspecific epithet should be lowercase: "${taxon}"`)
|
|
362
|
-
}
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
// Re-add authorship information
|
|
366
|
-
item.scientificNameOnly = item.scientificName
|
|
367
|
-
if (item.scientificNameAuthorship) {
|
|
368
|
-
item.scientificName += ` ${item.scientificNameAuthorship}`
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
return item
|
|
41
|
+
function mergeParserErrors (errors: SyntaxError[]): SyntaxError {
|
|
42
|
+
return new SyntaxError(errors.map(error => error.message).join('\n'))
|
|
372
43
|
}
|
|
373
44
|
|
|
374
45
|
function parseHeader (header: string): ResourceMetadata {
|
|
@@ -447,113 +118,108 @@ function parseHeader (header: string): ResourceMetadata {
|
|
|
447
118
|
return metadata
|
|
448
119
|
}
|
|
449
120
|
|
|
450
|
-
function
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
const longerIndentMatch = content.match(longerIndent)
|
|
454
|
-
if (longerIndentMatch !== null) {
|
|
455
|
-
const offset = longerIndentMatch.index
|
|
456
|
-
const line = (content.slice(0, offset).match(/\n/g) || []).length + 1
|
|
457
|
-
throw new SyntaxError(`Too much indentation at ${line}:0
|
|
458
|
-
${content.slice(offset).split('\n', 1)}
|
|
459
|
-
^`)
|
|
460
|
-
}
|
|
121
|
+
function parseResource (resource: FilePart): [ResourceMetadata, FilePart] {
|
|
122
|
+
const [header, _, ...rest] = resource.content.split(/(\n---\n+)/)
|
|
123
|
+
let config
|
|
461
124
|
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
const missingLeafTaxa = new RegExp(`^((?: ){0,${leafTaxonParentIndent}})(?![+=> ] ).*\\n(\\1( )+[+=>].*\\n)*(?!\\1 )`, 'm')
|
|
467
|
-
const missingLeafTaxaMatch = content.match(missingLeafTaxa)
|
|
468
|
-
if (missingLeafTaxaMatch !== null) {
|
|
469
|
-
const offset = missingLeafTaxaMatch.index
|
|
470
|
-
const line = (content.slice(0, offset).match(/\n/g) || []).length + 1
|
|
471
|
-
throw new SyntaxError(`Missing leaf taxon at ${line}:0
|
|
472
|
-
${content.slice(offset).split('\n', 1)}
|
|
473
|
-
^`)
|
|
474
|
-
}
|
|
125
|
+
try {
|
|
126
|
+
config = parseHeader(header)
|
|
127
|
+
} catch (error) {
|
|
128
|
+
throw makeParseError(error.message, resource.offsetLine + 1)
|
|
475
129
|
}
|
|
476
|
-
}
|
|
477
130
|
|
|
478
|
-
function parseResource (resource: string): [ResourceMetadata, string] {
|
|
479
|
-
const [header, _, ...rest] = resource.split(/(\n---\n+)/)
|
|
480
|
-
const config = parseHeader(header)
|
|
481
131
|
const content = rest.join('')
|
|
132
|
+
const offsetLine = resource.offsetLine + (header + _).split('\n').length - 1
|
|
482
133
|
|
|
483
|
-
return [config, content]
|
|
134
|
+
return [config, { content, offsetLine }]
|
|
484
135
|
}
|
|
485
136
|
|
|
486
|
-
function
|
|
487
|
-
|
|
488
|
-
|
|
137
|
+
function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds: number[], offsetLine: number): Resource {
|
|
138
|
+
const leafTaxonIndex = resource.metadata.levels.reduce((last, rank, i) => MAIN_RANKS.includes(rank) ? i : last, 0)
|
|
139
|
+
const data = resource.taxa as Record<TaxonId, WorkingTaxon>
|
|
140
|
+
const errors = []
|
|
489
141
|
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
}
|
|
142
|
+
let id = 0
|
|
143
|
+
let newId = Math.max(...oldIds)
|
|
144
|
+
let lineNumber = offsetLine
|
|
494
145
|
|
|
495
|
-
|
|
146
|
+
const parents: Array<TaxonId | null> = []
|
|
147
|
+
const previous = { id: '', indent: 0, group: { isLeaf: false, indent: 0 }, errors: <SyntaxError[]>[] }
|
|
496
148
|
|
|
497
|
-
|
|
498
|
-
|
|
149
|
+
for (const line of content) {
|
|
150
|
+
const hasOriginalId = line.type !== ResourceDiffType.Added && !/^\s*(\[indet\]|> )/.test(line.original ?? line.text as string)
|
|
151
|
+
if (hasOriginalId) {
|
|
152
|
+
id++
|
|
153
|
+
}
|
|
499
154
|
|
|
500
|
-
|
|
501
|
-
|
|
155
|
+
if (line.type === ResourceDiffType.Deleted) {
|
|
156
|
+
continue
|
|
157
|
+
} else {
|
|
158
|
+
lineNumber++
|
|
159
|
+
}
|
|
502
160
|
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
let parents: Array<TaxonId | null> = []
|
|
506
|
-
let groupIndent = 0
|
|
507
|
-
let previousId = ''
|
|
508
|
-
let newIdOffset = Math.max(...oldIds)
|
|
161
|
+
const [indentation, name] = (line.text as string).match(/^(\s*)(.*)/)!.slice(1)
|
|
162
|
+
const lineIndent = indentation.length
|
|
509
163
|
|
|
510
|
-
|
|
511
|
-
if (
|
|
512
|
-
|
|
513
|
-
if (!isIndetLine(line.original as string)) {
|
|
514
|
-
id++
|
|
515
|
-
}
|
|
164
|
+
// Validate line
|
|
165
|
+
if (lineIndent % INDENT === 1) {
|
|
166
|
+
errors.push(makeParseError('Too much or little indentation', lineNumber))
|
|
516
167
|
continue
|
|
168
|
+
} else if (lineIndent / INDENT >= resource.metadata.levels.length && !/^[+=>] /.test(name)) {
|
|
169
|
+
errors.push(makeParseError('Too much indentation', lineNumber))
|
|
170
|
+
continue
|
|
171
|
+
} else if (lineIndent <= previous.group.indent && (data[previous.id] && !previous.group.isLeaf)) {
|
|
172
|
+
errors.push(makeParseError('Missing leaf taxon', lineNumber - 1))
|
|
517
173
|
}
|
|
518
174
|
|
|
519
|
-
|
|
520
|
-
if (lineIndent >
|
|
175
|
+
// Update parentage
|
|
176
|
+
if (lineIndent > previous.indent) {
|
|
521
177
|
// Do not count synonyms as parents (unless this is correcting a typo in the synonym)
|
|
522
|
-
if (data[
|
|
523
|
-
parents.push(
|
|
178
|
+
if (data[previous.id] && data[previous.id].taxonomicStatus === 'accepted' || name.startsWith('> ')) {
|
|
179
|
+
parents.push(previous.id)
|
|
524
180
|
} else {
|
|
525
181
|
parents.push(null)
|
|
526
182
|
}
|
|
183
|
+
|
|
527
184
|
// Handle skips in indentation levels,
|
|
528
185
|
// e.g. if a certain genus has only species
|
|
529
186
|
// whereas other genera in the same key also
|
|
530
187
|
// have subgenera
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
for (let i = 0; i < gap; i++) { parents.push(null) }
|
|
188
|
+
for (let i = previous.indent + INDENT; i < lineIndent; i += INDENT) {
|
|
189
|
+
parents.push(null)
|
|
534
190
|
}
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
parents = parents.slice(0, lineIndent / 2)
|
|
538
|
-
groupIndent = lineIndent
|
|
191
|
+
} else if (lineIndent < previous.indent) {
|
|
192
|
+
parents.splice(lineIndent / INDENT)
|
|
539
193
|
}
|
|
540
194
|
|
|
195
|
+
previous.indent = lineIndent
|
|
196
|
+
|
|
541
197
|
// Do not process "indet" lines further, as they only serve to indicate
|
|
542
198
|
// that subtaxa are explicitely omitted
|
|
543
|
-
if (
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
}
|
|
199
|
+
if (name.startsWith('[indet]')) {
|
|
200
|
+
errors.push(...previous.errors)
|
|
201
|
+
previous.errors.length = 0
|
|
202
|
+
previous.group.isLeaf = lineIndent / INDENT >= leafTaxonIndex
|
|
548
203
|
continue
|
|
549
204
|
}
|
|
550
205
|
|
|
551
|
-
const parentId = parents.reduce((grandparent, parent) => parent
|
|
206
|
+
const parentId = parents.reduce((grandparent, parent) => parent ?? grandparent, null)
|
|
552
207
|
const parent = parentId === null ? {} as WorkingTaxon : data[parentId]
|
|
553
|
-
|
|
554
|
-
const
|
|
555
|
-
|
|
556
|
-
|
|
208
|
+
let item
|
|
209
|
+
const itemErrors = []
|
|
210
|
+
|
|
211
|
+
try {
|
|
212
|
+
const rank = resource.metadata.levels[parents.length]
|
|
213
|
+
item = parseName(name, rank, parent)
|
|
214
|
+
} catch (error) {
|
|
215
|
+
if (error instanceof RecoverableSyntaxError) {
|
|
216
|
+
itemErrors.push(makeParseError(error.message, lineNumber))
|
|
217
|
+
item = error.result
|
|
218
|
+
} else {
|
|
219
|
+
errors.push(makeParseError(error.message, lineNumber))
|
|
220
|
+
continue
|
|
221
|
+
}
|
|
222
|
+
}
|
|
557
223
|
|
|
558
224
|
// Add higher classification info
|
|
559
225
|
const itemAsObject = item as { [index: string]: unknown }
|
|
@@ -575,59 +241,93 @@ function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds
|
|
|
575
241
|
item.subgenus = item.infragenericEpithet
|
|
576
242
|
}
|
|
577
243
|
|
|
578
|
-
|
|
579
|
-
item.higherClassification = parent.higherClassification
|
|
580
|
-
} else if (parent.higherClassification) {
|
|
581
|
-
item.higherClassification = parent.higherClassification + ` | ${parent.scientificNameOnly}`
|
|
582
|
-
} else if (parentId) {
|
|
583
|
-
item.higherClassification = parent.scientificNameOnly
|
|
584
|
-
}
|
|
585
|
-
|
|
586
|
-
// Amend "parent" with corrections
|
|
244
|
+
// Amend "parent" with corrections, exit
|
|
587
245
|
if (item.taxonomicStatus === 'incorrect') {
|
|
246
|
+
if (parent.incorrect) {
|
|
247
|
+
errors.push(makeParseError('Cannot apply a correction to a previous correction', lineNumber))
|
|
248
|
+
continue
|
|
249
|
+
} else if (parentId === null) {
|
|
250
|
+
errors.push(makeParseError('Cannot apply a correction to nothing', lineNumber))
|
|
251
|
+
continue
|
|
252
|
+
}
|
|
253
|
+
|
|
588
254
|
parent.incorrect = { ...parent }
|
|
589
255
|
for (const key in item) {
|
|
590
256
|
if (key !== 'taxonomicStatus' && key !== 'verbatimIdentification') {
|
|
591
257
|
parentAsObject[key] = itemAsObject[key]
|
|
592
258
|
}
|
|
593
259
|
}
|
|
260
|
+
|
|
261
|
+
// If "parent" is corrected, its errors can be dropped
|
|
262
|
+
previous.errors.length = 0
|
|
263
|
+
// ...but errors associated with the corrected name are added immediately
|
|
264
|
+
errors.push(...itemErrors)
|
|
265
|
+
|
|
594
266
|
continue
|
|
595
267
|
}
|
|
596
268
|
|
|
597
|
-
//
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
item.
|
|
601
|
-
} else if (
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
id++
|
|
606
|
-
item.scientificNameID = idBase + (oldIds[id - 1] || id).toString()
|
|
269
|
+
// Add more classification info
|
|
270
|
+
const isSynonym = item.taxonomicStatus !== 'accepted'
|
|
271
|
+
if (isSynonym) {
|
|
272
|
+
item.higherClassification = parent.higherClassification
|
|
273
|
+
} else if (parent.higherClassification) {
|
|
274
|
+
item.higherClassification = parent.higherClassification + ` | ${parent.scientificNameOnly}`
|
|
275
|
+
} else if (parentId) {
|
|
276
|
+
item.higherClassification = parent.scientificNameOnly
|
|
607
277
|
}
|
|
608
|
-
|
|
278
|
+
|
|
279
|
+
// Set identifiers
|
|
280
|
+
item.scientificNameID = `${resource.id}:${hasOriginalId ? (oldIds[id - 1] ?? id) : ++newId}`
|
|
609
281
|
|
|
610
282
|
item.parentNameUsageID = isSynonym ? undefined : parent.scientificNameID
|
|
611
283
|
item.parentNameUsage = isSynonym ? undefined : parent.scientificName
|
|
612
284
|
item.acceptedNameUsageID = isSynonym ? parent.scientificNameID : undefined
|
|
613
285
|
item.acceptedNameUsage = isSynonym ? parent.scientificName : undefined
|
|
614
|
-
item.collectionCode =
|
|
286
|
+
item.collectionCode = resource.id
|
|
615
287
|
|
|
616
288
|
data[item.scientificNameID] = item
|
|
289
|
+
|
|
290
|
+
// Update loop state
|
|
291
|
+
errors.push(...previous.errors)
|
|
292
|
+
previous.errors = itemErrors
|
|
293
|
+
previous.id = item.scientificNameID
|
|
294
|
+
if (item.taxonomicStatus === 'accepted') {
|
|
295
|
+
previous.group.indent = previous.indent
|
|
296
|
+
previous.group.isLeaf = lineIndent / INDENT >= leafTaxonIndex
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
errors.push(...previous.errors)
|
|
301
|
+
if (errors.length) {
|
|
302
|
+
throw mergeParserErrors(errors)
|
|
617
303
|
}
|
|
618
304
|
|
|
619
305
|
return resource
|
|
620
306
|
}
|
|
621
307
|
|
|
622
|
-
|
|
623
|
-
|
|
308
|
+
interface FilePart {
|
|
309
|
+
content: string
|
|
310
|
+
offsetLine: number
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
function splitResources (file: string): FilePart[] {
|
|
314
|
+
const resources: FilePart[] = []
|
|
315
|
+
let offsetLine = 0
|
|
316
|
+
for (const content of file.split(RESOURCE_DELIMITER)) {
|
|
317
|
+
resources.push({ content, offsetLine })
|
|
318
|
+
offsetLine += (content + RESOURCE_DELIMITER).split('\n').length - 1
|
|
319
|
+
}
|
|
320
|
+
return resources
|
|
624
321
|
}
|
|
625
322
|
|
|
626
323
|
export function parseFile (file: string, id: WorkId, old?: ResourceHistory): Resource[] {
|
|
627
324
|
const oldResources = old ? splitResources(old.txt) : []
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
325
|
+
const newResources = splitResources(file)
|
|
326
|
+
const resources: Resource[] = []
|
|
327
|
+
const errors = []
|
|
328
|
+
|
|
329
|
+
for (let index = 0; index < newResources.length; index++) {
|
|
330
|
+
const [config, content] = parseResource(newResources[index])
|
|
631
331
|
const template: Resource = {
|
|
632
332
|
id: `${id}:${index + 1}`,
|
|
633
333
|
file: `${id}-${index + 1}`,
|
|
@@ -638,11 +338,11 @@ export function parseFile (file: string, id: WorkId, old?: ResourceHistory): Res
|
|
|
638
338
|
|
|
639
339
|
let diff
|
|
640
340
|
if (oldResources[index]) {
|
|
641
|
-
diff = createDiff(content, parseResource(oldResources[index])[1])
|
|
341
|
+
diff = createDiff(content.content, parseResource(oldResources[index])[1].content)
|
|
642
342
|
// Ignore empty lines
|
|
643
343
|
diff = diff.filter(line => line.text !== '')
|
|
644
344
|
} else {
|
|
645
|
-
diff = createDiff(content, content)
|
|
345
|
+
diff = createDiff(content.content, content.content)
|
|
646
346
|
}
|
|
647
347
|
|
|
648
348
|
const oldIds = []
|
|
@@ -652,8 +352,18 @@ export function parseFile (file: string, id: WorkId, old?: ResourceHistory): Res
|
|
|
652
352
|
}
|
|
653
353
|
}
|
|
654
354
|
|
|
655
|
-
|
|
656
|
-
|
|
355
|
+
try {
|
|
356
|
+
resources.push(parseResourceContent(diff, template, oldIds, content.offsetLine))
|
|
357
|
+
} catch (error) {
|
|
358
|
+
errors.push(error)
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
if (errors.length) {
|
|
363
|
+
throw mergeParserErrors(errors)
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
return resources
|
|
657
367
|
}
|
|
658
368
|
|
|
659
369
|
export function parseFileHeader (file: string): ResourceMetadata[] {
|