@larsgw/formica 0.1.0

package/lib/resources/parse-text.js

@@ -0,0 +1,499 @@
+"use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseFileHeader = exports.parseFile = void 0;
+var yaml = require("js-yaml");
+var diff_resource_1 = require("./diff-resource");
+var RANKS = [
+    'class',
+    'infraclass',
+    'superorder',
+    'order',
+    'suborder',
+    'infraorder',
+    'superfamily',
+    'family',
+    'subfamily',
+    'tribe',
+    'subtribe',
+    'genus',
+    'subgenus',
+    'section',
+    'subsection',
+    'series',
+    'group',
+    'subgroup',
+    'aggregate',
+    'complex',
+    'species',
+    'subspecies',
+    'variety',
+    'form',
+    'aberration',
+    'race',
+    'stirps' // not ICZN
+];
+var DWC_RANKS = [
+    'kingdom',
+    'phylum',
+    'class',
+    'order',
+    'family',
+    'subfamily',
+    'genus',
+    'subgenus'
+];
+var TAXONOMIC_STATUS = {
+    '>': 'incorrect',
+    '+': 'heterotypic synonym',
+    '=': 'synonym'
+};
+var INDET_SUFFIXES = new Set([
+    'sp.',
+    'spec.',
+    'indet.',
+    'sp. indet.',
+    'spec. indet.'
+]);
+var RANK_LABELS = {
+    'subspecies': 'subsp.',
+    'variety': 'var.',
+    'form': 'f.',
+    'aberration': 'ab.',
+    'race': 'r.',
+    'stirps': 'st.'
+};
+var RANK_LABELS_REVERSE = {
+    'st': 'stirps',
+    'r': 'race',
+    'ab': 'aberration',
+    'f': 'form',
+    'var': 'variety',
+    'ssp': 'subspecies',
+    'subsp': 'subspecies'
+};
+var NAME_PATTERN = new RegExp('^' +
+    // $1 main name part
+    '(\\S+)' +
+    // $2 optional author citation
+    '(?: ' +
+    // but not auct(t)., etc.
+    '(?!auctt?\\.|(?:syn|comb|sp|spec)\\. n(?:ov)?\\.|s(?:ens[.u]|\\.))' +
+    '(' +
+    // $2.1 anything in parentheses
+    '\\(.+?\\)' +
+    '|' +
+    // $2.2 anything followed by a year
+    '.+?\\d{4}\\)?' +
+    '|' +
+    // $2.3 name(, name)* & name
+    '.+(?:, .+)* & \\S+' +
+    '|' +
+    // $2.4 name y name
+    '\\S+ [yY] \\S+' +
+    '|' +
+    // $2.5 name( in name)
+    '\\p{Lu}\\S*(?: in \\S+)?' +
+    '))?' +
+    // $3 optional notes
+    '(?:,? (.+))?' +
+    '$', 'u');
+/**
+ * Structure
+ *   $1 genus+subgenus (+ trailing space): (?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?
+ *     $1.1 genus: ([A-Z]\S+)
+ *     $1.2 subgenus: (?:\(([A-Z]\S+?)\) )?
+ *   $2 species: ([a-z]\S+)
+ */
+var BINAME_PATTERN = /^(?:([A-Z]\S+) (?:\(([A-Z]\S+?)\) )?)?([a-z]\S+) ?/;
+function compareRanks(a, b) {
+    return RANKS.indexOf(a) - RANKS.indexOf(b);
+}
+function capitalize(name) {
+    return name[0].toUpperCase() + name.slice(1).toLowerCase();
+}
+function isUpperCase(name) {
+    return name === name.toUpperCase();
+}
+function getSynonymRank(name, rank) {
+    var BINAME_PATTERN = /^([A-Z]\S+ (\([A-Z]\S+\) )?)?(x )?[a-z0-9-]+(?= |$)/;
+    if (!BINAME_PATTERN.test(name)) {
+        return rank;
+    }
+    var rest = name.replace(BINAME_PATTERN, '');
+    var rankPrefix = rest.match(/^ (st|r|ab|f|var|ssp|subsp)\. /);
+    if (rankPrefix) {
+        return RANK_LABELS_REVERSE[rankPrefix[1]];
+    }
+    else if (/^ (?!sensu)[a-z0-9-]+($| )/.test(rest)) {
+        return 'subspecies';
+    }
+    else {
+        return 'species';
+    }
+}
+function capitalizeAuthors(authors) {
+    return authors
+        .replace(/[^\x00-\x40\x5B-\x60\x7B-\x7F]+/g, // eslint-disable-line no-control-regex
+    function (// eslint-disable-line no-control-regex
+    name) { return isUpperCase(name) ? capitalize(name) : name; })
+        .replace(/ Y /g, ' y ');
+}
+function parseName(name, rank, parent) {
+    var item = {};
+    // Synonyms have the accepted name usage as 'parent'.
+    var isSynonym = /^[+=>] /.test(name);
+    if (isSynonym) {
+        item.taxonomicStatus = TAXONOMIC_STATUS[name[0]];
+        name = name.replace(/^[+=>] (\? ?)?/, '');
+        rank = getSynonymRank(name, parent.taxonRank);
+    }
+    else {
+        item.taxonomicStatus = 'accepted';
+    }
+    // Clusters
+    if (/^\[(_|\d+)\] /.test(name)) {
+        name = name.replace(/^\[(_|\d+)\] /, '');
+    }
+    // Parent context is used for parsing and formatting binomial names.
+    var parentContext = __assign({}, parent);
+    if (parent.incorrect) {
+        parentContext.incorrect = __assign({}, parent.incorrect);
+    }
+    // The parent context should be amended in the two cases where binomial names
+    // are truly accepted: synonyms and species (and below) without parents (resp.
+    // genera and genera and species) to provide parts of the name.
+    if (isSynonym || !parentContext.genus || (compareRanks('species', rank) < 0 && !parentContext.specificEpithet)) {
+        var _a = name.match(BINAME_PATTERN) || [], genus = _a[1], subgenus = _a[2], species = _a[3];
+        if (genus) {
+            parentContext.genus = capitalize(genus);
+            if (parentContext.incorrect)
+                parentContext.incorrect.genus = capitalize(genus);
+        }
+        if (subgenus) {
+            parentContext.subgenus = capitalize(subgenus);
+            if (parentContext.incorrect)
+                parentContext.incorrect.subgenus = capitalize(subgenus);
+        }
+        else if (genus) {
+            // If a genus is given but no subgenus, remove it from the parent context
+            delete parentContext.subgenus;
+            if (parentContext.incorrect)
+                delete parentContext.incorrect.subgenus;
+        }
+        if (species) {
+            parentContext.specificEpithet = species;
+            if (parentContext.incorrect)
+                parentContext.incorrect.specificEpithet = species;
+        }
+    }
+    // In taxa of group, species or lower, the name should just contain the
+    // (inter)specific epithet and the author information & remarks when processing
+    // further.
+    if (compareRanks('group', rank) <= 0) {
+        var parseContext = parentContext.incorrect || parentContext;
+        if (!parseContext.genus) {
+            parseContext.genus = name.split(' ', 1)[0];
+        }
+        var genusPrefix = new RegExp("^".concat(parentContext.genus, " (\\(.*?\\) )?"), 'i');
+        if (name[0] === parentContext.genus[0]) {
+            name = name.replace(genusPrefix, '');
+        }
+        if (compareRanks('species', rank) < 0) {
+            var speciesPrefix = parseContext.specificEpithet + ' ';
+            if (name.startsWith(speciesPrefix)) {
+                name = name.slice(speciesPrefix.length).replace(/^(st|r|ab|f|var|ssp|subsp)\. /, '');
+            }
+        }
+    }
+    // Hybrids
+    if (rank === 'species' && /^x /.test(name)) {
+        name = '\u00D7' + name.slice(2);
+    }
+    // Divide the name into the main scientific name (only the epithet for taxa
+    // lower than genus), the authorship information, and optionally remarks
+    var nameParts = name.match(NAME_PATTERN);
+    if (!nameParts) {
+        throw new Error("Taxon \"".concat(name, "\" could not be parsed"));
+    }
+    var _ = nameParts[0], taxon = nameParts[1], _b = nameParts[2], citation = _b === void 0 ? '' : _b, notes = nameParts[3];
+    item.scientificNameAuthorship = capitalizeAuthors(citation);
+    item.taxonRemarks = notes;
+    item.taxonRank = rank;
+    if (/[^\p{L}0-9\u{00D7}\- ]/u.test(taxon) && !INDET_SUFFIXES.has(taxon)) {
+        throw new Error("Taxon name contains unexpected characters: \"".concat(taxon, "\""));
+    }
+    // Validate names and recompose binomial and trinomial names
+    if (compareRanks('group', rank) > 0) {
+        item.scientificName = capitalize(taxon);
+        if (item.scientificName[0] !== taxon[0]) {
+            throw new Error("Taxon name (".concat(rank, ") should be capitalized: \"").concat(taxon, "\""));
+        }
+    }
+    else if (rank === 'group') {
+        item.genericName = parentContext.genus;
+        item.infragenericEpithet = parentContext.subgenus;
+        var specificEpithet = taxon.toLowerCase().replace(/(-group)?$/, '');
+        item.scientificName = "".concat(item.genericName, " ").concat(specificEpithet, "-group");
+        if (taxon.toLowerCase() !== taxon) {
+            console.log(item, taxon);
+            throw new Error("Group name should be lowercase: \"".concat(taxon, "\""));
+        }
+    }
+    else if (rank === 'subgroup') {
+        item.genericName = parentContext.genus;
+        item.infragenericEpithet = parentContext.subgenus;
+        var specificEpithet = taxon.toLowerCase().replace(/(-subgroup)?$/, '');
+        item.scientificName = "".concat(item.genericName, " ").concat(specificEpithet, "-subgroup");
+        if (taxon.toLowerCase() !== taxon) {
+            console.log(item, taxon);
+            throw new Error("Subgroup name should be lowercase: \"".concat(taxon, "\""));
+        }
+    }
+    else if (rank === 'species') {
+        item.genericName = parentContext.genus;
+        item.infragenericEpithet = parentContext.subgenus;
+        item.specificEpithet = taxon.toLowerCase();
+        item.scientificName = "".concat(item.genericName, " ").concat(item.specificEpithet);
+        if (item.specificEpithet !== taxon) {
+            console.log(item, taxon);
+            throw new Error("Specific epithet should be lowercase: \"".concat(taxon, "\""));
+        }
+    }
+    else if (compareRanks('species', rank) < 0) {
+        item.genericName = parentContext.genus;
+        item.infragenericEpithet = parentContext.subgenus;
+        item.specificEpithet = parentContext.specificEpithet;
+        item.intraspecificEpithet = taxon.toLowerCase();
+        // If possible, names below species should have abbreviations for ranks,
+        // like "subsp."
+        var nameParts_1 = [
+            item.genericName,
+            item.specificEpithet,
+            item.intraspecificEpithet
+        ];
+        if (item.taxonRank in RANK_LABELS) {
+            nameParts_1.splice(2, 0, RANK_LABELS[item.taxonRank]);
+        }
+        item.scientificName = nameParts_1.join(' ');
+        if (item.intraspecificEpithet !== taxon) {
+            console.log(item, taxon);
+            throw new Error("Intraspecific epithet should be lowercase: \"".concat(taxon, "\""));
+        }
+    }
+    // Re-add authorship information
+    item.scientificNameOnly = item.scientificName;
+    if (item.scientificNameAuthorship) {
+        item.scientificName += " ".concat(item.scientificNameAuthorship);
+    }
+    // Amend "parent" with corrections
+    if (item.taxonomicStatus === 'incorrect') {
+        var itemAsObject = item;
+        var parentAsObject = parent;
+        parent.incorrect = __assign({}, parent);
+        for (var key in item) {
+            if (key !== 'taxonomicStatus') {
+                parentAsObject[key] = itemAsObject[key];
+            }
+        }
+    }
+    return item;
+}
+function parseHeader(header) {
+    var config = yaml.load(header);
+    if (typeof config !== 'object' || Array.isArray(config) || config === null) {
+        throw new SyntaxError('yaml header should be an object');
+    }
+    // Invalid configuration
+    var levels;
+    if (!('levels' in config)) {
+        levels = [];
+    }
+    else if (!Array.isArray(config.levels)) {
+        throw new SyntaxError('"levels" should be an array');
+    }
+    else {
+        levels = config.levels;
+    }
+    var scope;
+    if (!('scope' in config)) {
+        scope = [];
+    }
+    else if (!Array.isArray(config.scope)) {
+        throw new SyntaxError('"scope" should be an array');
+    }
+    else {
+        scope = config.scope;
+    }
+    // No taxon ranks
+    if (levels.length === 0) {
+        throw new SyntaxError('Resource contains no taxa');
+    }
+    // Invalid taxon ranks
+    var invalidTaxonRanks = levels.filter(function (rank) { return !RANKS.includes(rank); });
+    if (invalidTaxonRanks.length) {
+        throw new SyntaxError("\"levels\" contains invalid values: ".concat(invalidTaxonRanks.join(', ')));
+    }
+    var metadata = { levels: levels, scope: scope };
+    if ('catalog' in config && typeof config.catalog === 'object' && config.catalog !== null) {
+        metadata.catalog = config.catalog;
+    }
+    return metadata;
+}
+function parseResource(resource) {
+    var _a = resource.split(/(\n---\n+)/), header = _a[0], _ = _a[1], rest = _a.slice(2);
+    var config = parseHeader(header);
+    var content = rest.join('');
+    // Check for too much indentation
+    var longerIndent = new RegExp("^(  ){".concat(config.levels.length - 1, "}(?!  [+=>] ) "), 'm');
+    var longerIndentMatch = content.match(longerIndent);
+    if (longerIndentMatch !== null) {
+        var offset = longerIndentMatch.index;
+        var line = (content.slice(0, offset).match(/\n/g) || []).length + 1;
+        throw new SyntaxError("Too much indentation at ".concat(line, ":0\n").concat(content.slice(offset).split('\n', 1), "\n^"));
+    }
+    return [config, content];
+}
+function parseResourceContent(content, resource, oldIds) {
+    var idBase = "".concat(resource.id, ":");
+    var data = resource.taxa;
+    var id = 0;
+    var parents = [];
+    var groupIndent = 0;
+    var previousId = '';
+    var newIdOffset = Math.max.apply(Math, oldIds);
+    var _loop_1 = function (line, type) {
+        if (type === diff_resource_1.ResourceDiffType.Deleted) {
+            id++;
+            return "continue";
+        }
+        var lineIndent = line.match(/^ */)[0].length;
+        if (lineIndent > groupIndent) {
+            // Do not count synonyms as parents
+            if (data[previousId] && data[previousId].taxonomicStatus === 'accepted') {
+                parents.push(previousId);
+            }
+            else {
+                parents.push(null);
+            }
+            // Handle skips in indentation levels,
+            // e.g. if a certain genus has only species
+            // whereas other genera in the same key also
+            // have subgenera
+            if ((lineIndent - groupIndent) > 2) {
+                var gap = (lineIndent - groupIndent - 2) / 2;
+                for (var i = 0; i < gap; i++) {
+                    parents.push(null);
+                }
+            }
+            groupIndent = lineIndent;
+        }
+        else if (lineIndent < groupIndent) {
+            parents = parents.slice(0, lineIndent / 2);
+            groupIndent = lineIndent;
+        }
+        var parentId = parents.reduce(function (grandparent, parent) { return parent || grandparent; }, null);
+        var parent_1 = parentId === null ? {} : data[parentId];
+        var name_1 = line.slice(groupIndent);
+        var rank = resource.metadata.levels[groupIndent / 2];
+        var item = parseName(name_1, rank, parent_1);
+        var isSynonym = item.taxonomicStatus !== 'accepted';
+        var isIndet = Array.from(INDET_SUFFIXES).some(function (suffix) { return name_1.endsWith(' ' + suffix); });
+        if (item.taxonomicStatus === 'incorrect' || isIndet) {
+            return "continue";
+        }
+        if (type === diff_resource_1.ResourceDiffType.Added) {
+            newIdOffset++;
+            item.scientificNameID = idBase + newIdOffset.toString();
+        }
+        else {
+            id++;
+            item.scientificNameID = idBase + (oldIds[id - 1] || id).toString();
+        }
+        previousId = item.scientificNameID;
+        item.parentNameUsageID = isSynonym ? undefined : parent_1.scientificNameID;
+        item.parentNameUsage = isSynonym ? undefined : parent_1.scientificName;
+        item.acceptedNameUsageID = isSynonym ? parent_1.scientificNameID : undefined;
+        item.acceptedNameUsage = isSynonym ? parent_1.scientificName : undefined;
+        item.collectionCode = idBase.slice(0, -1);
+        for (var _b = 0, DWC_RANKS_1 = DWC_RANKS; _b < DWC_RANKS_1.length; _b++) {
+            var rank_1 = DWC_RANKS_1[_b];
+            var itemAsObject = item;
+            var parentAsObject = parent_1;
+            itemAsObject[rank_1] = undefined;
+            if (parentAsObject[rank_1]) {
+                itemAsObject[rank_1] = parentAsObject[rank_1];
+            }
+            if (item.taxonRank === rank_1) {
+                itemAsObject[rank_1] = item.scientificNameOnly;
+            }
+        }
+        if (item.genericName && !item.genus) {
+            item.genus = item.genericName;
+        }
+        if (item.infragenericEpithet && !item.subgenus) {
+            item.subgenus = item.infragenericEpithet;
+        }
+        if (isSynonym) {
+            item.higherClassification = parent_1.higherClassification;
+        }
+        else if (parent_1.higherClassification) {
+            item.higherClassification = parent_1.higherClassification + " | ".concat(parent_1.scientificNameOnly);
+        }
+        else if (parentId) {
+            item.higherClassification = parent_1.scientificNameOnly;
+        }
+        data[item.scientificNameID] = item;
+    };
+    for (var _i = 0, content_1 = content; _i < content_1.length; _i++) {
+        var _a = content_1[_i], line = _a.text, type = _a.type;
+        _loop_1(line, type);
+    }
+    return resource;
+}
+function splitResources(file) {
+    return file.split('\n\n===\n\n');
+}
+function parseFile(file, id, old) {
+    var oldResources = old ? splitResources(old.txt) : [];
+    return splitResources(file).map(function (resource, index) {
+        var _a = parseResource(resource), config = _a[0], content = _a[1];
+        var template = {
+            id: "".concat(id, ":").concat(index + 1),
+            file: "".concat(id, "-").concat(index + 1),
+            workId: id,
+            metadata: config,
+            taxa: {}
+        };
+        var diff;
+        if (oldResources[index]) {
+            diff = (0, diff_resource_1.createDiff)(content, parseResource(oldResources[index])[1]);
+        }
+        else {
+            diff = (0, diff_resource_1.createDiff)(content, content);
+        }
+        var oldIds = [];
+        if (old) {
+            for (var _i = 0, _b = old.dwc[index].slice(1); _i < _b.length; _i++) {
+                var row = _b[_i];
+                oldIds.push(parseInt(row[0].split(':')[2]));
+            }
+        }
+        return parseResourceContent(diff, template, oldIds);
+    });
+}
+exports.parseFile = parseFile;
+function parseFileHeader(file) {
+    return splitResources(file).map(function (resource) { return parseResource(resource)[0]; });
+}
+exports.parseFileHeader = parseFileHeader;

package/lib/types.d.ts

@@ -0,0 +1,62 @@
+type Value = string[] | string;
+type SingleValue = string;
+interface FieldError {
+    field: string;
+    error: string;
+}
+interface WorkError extends FieldError {
+    entity: WorkId;
+}
+type Rank = string;
+type DwcRank = Rank;
+type TaxonStatus = string;
+type TaxonId = string;
+type ResourceId = string;
+type WorkId = string;
+interface WorkingTaxon {
+    scientificNameID?: TaxonId;
+    scientificName?: string;
+    scientificNameAuthorship?: string;
+    genericName?: string;
+    infragenericEpithet?: string;
+    specificEpithet?: string;
+    intraspecificEpithet?: string;
+    taxonRank?: Rank;
+    taxonRemarks?: string;
+    collectionCode?: ResourceId;
+    taxonomicStatus?: TaxonStatus;
+    acceptedNameUsageID?: TaxonId;
+    acceptedNameUsage?: string;
+    parentNameUsageID?: TaxonId;
+    parentNameUsage?: string;
+    kingdom?: string;
+    phylum?: string;
+    class?: string;
+    order?: string;
+    family?: string;
+    subfamily?: string;
+    genus?: string;
+    subgenus?: string;
+    higherClassification?: string;
+    scientificNameOnly?: string;
+    incorrect?: WorkingTaxon;
+}
+interface Taxon extends WorkingTaxon {
+    scientificNameID: TaxonId;
+    scientificName: string;
+    taxonRank: Rank;
+    collectionCode: ResourceId;
+    taxonomicStatus: string;
+}
+interface ResourceMetadata {
+    levels: Rank[];
+    scope: string[];
+    catalog?: object;
+}
+interface Resource {
+    id: string;
+    file: string;
+    workId: string;
+    metadata: ResourceMetadata;
+    taxa: Record<TaxonId, Taxon>;
+}

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@larsgw/formica",
+  "version": "0.1.0",
+  "description": "SDK and tools for data from the Library of Identification Resources",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "bin": {
+    "loir-validate-catalog": "./lib/bin/validate-catalog.js",
+    "loir-validate-resources": "./lib/bin/validate-resources-text.js",
+    "loir-resources-process": "./lib/bin/process-resources.js",
+    "loir-resources-index": "./lib/bin/process-resources-index.js"
+  },
+  "scripts": {
+    "lint": "eslint src",
+    "build": "tsc -d",
+    "preversion": "npm run lint",
+    "prepublishOnly": "npm run build"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/identification-resources/formica.git"
+  },
+  "author": "Lars Willighagen (https://larsgw.github.io/)",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/identification-resources/formica/issues"
+  },
+  "homepage": "https://github.com/identification-resources/formica#readme",
+  "dependencies": {
+    "ietf-language-tag-regex": "^0.0.5",
+    "js-yaml": "^4.1.0",
+    "spdx-license-list": "^6.6.0"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^4.0.5",
+    "@types/node": "^18.14.1",
+    "@typescript-eslint/eslint-plugin": "^5.54.0",
+    "@typescript-eslint/parser": "^5.54.0",
+    "eslint": "^8.35.0",
+    "typescript": "^4.9.5"
+  }
+}

package/src/bin/process-resources-index.ts

@@ -0,0 +1,73 @@
+import { promises as fs, existsSync as fileExists } from 'fs'
+import * as path from 'path'
+
+import { csv, resources } from '../index'
+import { numericSort } from './util'
+
+interface AmendedResourceMetadata extends ResourceMetadata {
+    id: ResourceId,
+    taxonCount: number
+}
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+function sortObject (object: Record<string, any>): Record<string, any> {
+    const sorted: Record<string, any> = {}
+    for (const key of Object.keys(object).sort(numericSort)) {
+        sorted[key] = object[key]
+    }
+    return sorted
+}
+/* eslint-enable @typescript-eslint/no-explicit-any */
+
+async function main (args: string[]): Promise<void> {
+    const REPO_ROOT = path.resolve(args[0])
+
+    const files = await fs.readdir(path.join(REPO_ROOT, 'txt'))
+
+    const gbifIndex: Record<string, TaxonId[]> = {}
+    const resourceIndex: Record<TaxonId, AmendedResourceMetadata> = {}
+
+    await Promise.all(files.map(async function (fileName) {
+        if (!fileName.endsWith('.txt')) { return }
+        const id = fileName.slice(0, -4)
+        const file = await fs.readFile(path.join(REPO_ROOT, 'txt', fileName), 'utf-8')
+
+        return Promise.all(resources.parseTextFileHeader(file).map(async function (resource, index) {
+            const amendedResource = {
+                ...resource,
+                id: `${id}:${index + 1}`,
+                taxonCount: 0
+            }
+
+            const dwcFile = path.join(REPO_ROOT, 'dwc', `${id}-${index + 1}.csv`)
+            if (!fileExists(dwcFile)) {
+                return
+            }
+
+            const [_header, ...dwc] = csv.parseCsv(await fs.readFile(dwcFile, 'utf-8'))
+            for (const taxon of dwc) {
+                const gbifId = taxon[25]
+                if (gbifId) {
+                    if (!(gbifId in gbifIndex)) {
+                        gbifIndex[gbifId] = []
+                    }
+                    gbifIndex[gbifId].push(taxon[0])
+                    gbifIndex[gbifId].sort(numericSort)
+                }
+                amendedResource.taxonCount += 1
+            }
+
+            resourceIndex[amendedResource.id] = amendedResource
+        }))
+    }))
+
+    await Promise.all([
+        fs.writeFile(path.join(REPO_ROOT, 'gbif.index.json'), JSON.stringify(sortObject(gbifIndex), null, 2)),
+        fs.writeFile(path.join(REPO_ROOT, 'index.json'), JSON.stringify(sortObject(resourceIndex), null, 2))
+    ])
+}
+
+main(process.argv.slice(2)).catch(error => {
+    console.error(error)
+    process.exit(1)
+})