@ca-plant-list/ca-plant-list 0.4.21 → 0.4.22

package/lib/errorlog.js

@@ -17,7 +17,7 @@ class ErrorLog {
     }
 
     /**
-     * @param  {...string} args
+     * @param  {...any} args
      */
     log(...args) {
         if (this.#echo) {

package/lib/externalsites.js

@@ -6,6 +6,20 @@
 }} InatObsOptions */
 
 export class ExternalSites {
+    /**
+     * @param {import("./taxon.js").Taxon} taxon
+     * @returns {URL|undefined}
+     */
+    static getCalscapeLink(taxon) {
+        const calscapeCN = taxon.getCalscapeCommonName();
+        if (!calscapeCN) {
+            return;
+        }
+        return new URL(
+            `https://www.calscape.org/${taxon.getCalscapeName().replaceAll(" ", "-")}-()`,
+        );
+    }
+
     /**
      * @param {import("./taxon.js").Taxon} taxon
      * @param {import("./config.js").Config} config
@@ -34,6 +48,21 @@ export class ExternalSites {
         return url;
     }
 
+    /**
+     * @param {import("./taxon.js").Taxon} taxon
+     * @returns {URL|undefined}
+     */
+    static getFNARefLink(taxon) {
+        const name = taxon.getFNAName();
+        if (!name) {
+            return;
+        }
+        const url = new URL(
+            "http://floranorthamerica.org/" + name.replaceAll(" ", "_"),
+        );
+        return url;
+    }
+
     /**
      * @param {InatObsOptions} options
      */

package/lib/files.js

@@ -13,15 +13,13 @@ class Files {
 
     /**
      * @param {string} fileName
-     * @param {*} inStream
-     * @access private
+     * @param {import("node:stream").Stream} inStream
      */
     static #createFileFromStream(fileName, inStream) {
         /**
-         *
          * @param {string} fileName
-         * @param {*} inStream
-         * @param {*} resolve
+         * @param {import("node:stream").Stream} inStream
+         * @param {function(boolean):void} resolve
          */
         function implementation(fileName, inStream, resolve) {
             const outStream = fs.createWriteStream(fileName);

package/lib/htmltaxon.js

@@ -1,5 +1,6 @@
 import { Config } from "./config.js";
 import { DateUtils } from "./dateutils.js";
+import { ExternalSites } from "./externalsites.js";
 import { HTML } from "./html.js";
 import { Markdown } from "./markdown.js";
 import { RarePlants } from "./rareplants.js";
@@ -53,6 +54,22 @@ const DEFAULT_TAXA_COLUMNS = [
     TAXA_LIST_COLS.COMMON_NAME,
 ];
 
+/** @type {Object<string,{label:string,href:function(import("./taxon.js").Taxon):URL|undefined}>} */
+const REFLINKS = {
+    calscape: {
+        label: "Calscape",
+        href: (taxon) => ExternalSites.getCalscapeLink(taxon),
+    },
+    fna: {
+        label: "Flora of North America",
+        href: (taxon) => ExternalSites.getFNARefLink(taxon),
+    },
+    cch: {
+        label: "CCH2",
+        href: (taxon) => ExternalSites.getCCH2RefLink(taxon),
+    },
+};
+
 class HTMLTaxon {
     /**
      * @param {string[]} links
@@ -68,20 +85,13 @@ class HTMLTaxon {
     }
 
     /**
+     * @param {string[]} links
      * @param {import("./taxon.js").Taxon} taxon
-     * @returns {string|undefined}
+     * @param {import("./index.js").RefSourceCode} sourceCode
      */
-    static getCalscapeLink(taxon) {
-        const calscapeCN = taxon.getCalscapeCommonName();
-        if (!calscapeCN) {
-            return;
-        }
-        return HTML.getLink(
-            `https://www.calscape.org/${taxon.getCalscapeName().replaceAll(" ", "-")}-()`,
-            "Calscape",
-            {},
-            true,
-        );
+    static addRefLink(links, taxon, sourceCode) {
+        const source = REFLINKS[sourceCode];
+        this.addLink(links, source.href(taxon), source.label);
     }
 
     /**

package/lib/index.d.ts

@@ -4,6 +4,10 @@ import { Command } from "commander";
 
 export type NativeStatusCode = "N" | "NC" | "U" | "X";
 
+type PhotoRights = "CC0" | "CC BY" | "CC BY-NC" | "C" | null;
+
+type RefSourceCode = "calscape" | "cch" | "fna";
+
 export type TaxonData = {
     bloom_end: string;
     bloom_start: string;
@@ -14,6 +18,7 @@ export type TaxonData = {
     "common name": string;
     CRPR: string;
     FESA: string;
+    fna: string;
     flower_color: string;
     GRank: string;
     "inat id": string;
@@ -63,7 +68,6 @@ export class Exceptions {
 
 export class ExternalSites {
     static getCCH2ObsLink(taxon: Taxon, config: Config): URL | undefined;
-    static getCCH2RefLink(taxon: Taxon): URL | undefined;
 }
 
 export class Family {
@@ -78,7 +82,7 @@ export class Files {
     ): Promise<Headers>;
     static mkdir(dir: string): void;
     static rmDir(dir: string): void;
-    static write(fileName: string, data: string, overwrite: boolean): void;
+    static write(fileName: string, data: string, overwrite?: boolean): void;
 }
 
 export class Genera {}
@@ -89,6 +93,7 @@ export class Genus {
 
 export class HTML {
     static arrayToLI(items: string[]): string;
+    static escapeText(text: string): string;
     static getLink(
         href: string | undefined,
         linkText: string,
@@ -113,6 +118,11 @@ export class HTMLTaxon {
         href: URL | string | undefined,
         label: string,
     ): void;
+    static addRefLink(
+        links: string[],
+        taxon: Taxon,
+        sourceCode: RefSourceCode,
+    ): void;
     static getFooterHTML(taxon: Taxon): string;
     static getListSectionHTML(
         list: string[],
@@ -123,11 +133,10 @@ export class HTMLTaxon {
 }
 
 export class Jekyll {
+    static hasInclude(baseDir: string, path: string): boolean;
     static include(fileName: string): string;
 }
 
-type PhotoRights = "CC0" | "CC BY" | "CC BY-NC" | "C" | null;
-
 export class Photo {
     getAttribution(): string;
     getExt(): string;
@@ -141,32 +150,33 @@ export class Program {
     static getProgram(): Command;
 }
 
-export class Taxa {
+export class Taxa<T> {
     constructor(
         inclusionList: Record<string, TaxonData> | true,
         errorLog: ErrorLog,
         showFlowerErrors: boolean,
-        taxonFactory?: (td: TaxonData, g: Genera) => Taxon,
+        taxonFactory?: (td: TaxonData, g: Genera) => T,
         extraTaxa?: TaxonData[],
         extraSynonyms?: Record<string, string>[],
     );
-    getTaxon(name: string): Taxon;
-    getTaxonList(): Taxon[];
+    getTaxon(name: string): T;
+    getTaxonList(): T[];
 }
 
 export class Taxon {
+    constructor(data: TaxonData, genera: Genera);
     getBaseFileName(): string;
     getCalfloraID(): string;
     getCalfloraTaxonLink(): string;
-    getCESA(): string | undefined;
-    getCNDDBRank(): string | undefined;
+    getCESA(): string;
+    getCNDDBRank(): string;
     getCommonNames(): string[];
     getFamily(): Family;
     getFileName(): string;
-    getFESA(): string | undefined;
+    getFESA(): string;
     getGenus(): Genus;
     getGenusName(): string;
-    getGlobalRank(): string | undefined;
+    getGlobalRank(): string;
     getINatID(): string;
     getINatTaxonLink(): string;
     getJepsonID(): string;

package/lib/taxon.js

@@ -15,6 +15,7 @@ class Taxon {
     /**@type {string|undefined} */
     #iNatSyn;
     #cch2id;
+    #fnaName;
     #calscapeCN;
     #lifeCycle;
     #flowerColors;
@@ -53,6 +54,7 @@ class Taxon {
         this.#calRecNum = data["calrecnum"];
         this.#iNatID = data["inat id"];
         this.#cch2id = data.cch2_id;
+        this.#fnaName = data.fna ?? "";
         this.#calscapeCN =
             data.calscape_cn === "" ? undefined : data.calscape_cn;
         this.#lifeCycle = data.life_cycle;
@@ -171,12 +173,18 @@ class Taxon {
         return this.#cch2id;
     }
 
+    /**
+     * @returns {string}
+     */
     getCESA() {
-        return this.#cesa;
+        return this.#cesa ?? "";
     }
 
+    /**
+     * @returns {string}
+     */
     getCNDDBRank() {
-        return this.#rankCNDDB;
+        return this.#rankCNDDB ?? "";
     }
 
     getCommonNames() {
@@ -187,14 +195,27 @@ class Taxon {
         return this.getGenus().getFamily();
     }
 
+    /**
+     * @returns {string}
+     */
     getFESA() {
-        return this.#fesa;
+        return this.#fesa ?? "";
     }
 
     getFileName(ext = "html") {
         return this.getBaseFileName() + "." + ext;
     }
 
+    /**
+     * @returns {string}
+     */
+    getFNAName() {
+        if (this.#fnaName === "true") {
+            return this.getName();
+        }
+        return this.#fnaName;
+    }
+
     getFlowerColors() {
         return this.#flowerColors;
     }
@@ -207,8 +228,11 @@ class Taxon {
         return this.#genus;
     }
 
+    /**
+     * @returns {string}
+     */
     getGlobalRank() {
-        return this.#rankGlobal;
+        return this.#rankGlobal ?? "";
     }
 
     /**
@@ -287,6 +311,9 @@ class Taxon {
         return this.#rpiID;
     }
 
+    /**
+     * @returns {string}
+     */
     getRPIRank() {
         if (!this.#rankRPI) {
             return this.#rankRPI;
@@ -341,6 +368,8 @@ class Taxon {
                 return "Native";
             case "NC":
                 return config.getLabel("status-NC", "Introduced");
+            case "U":
+                return "Nativity Uncertain";
             case "X":
                 return "Introduced";
         }

package/lib/tools/cch2.js

@@ -1,6 +1,9 @@
 import path from "node:path";
 import { CSV } from "../csv.js";
 import { TaxaCSV } from "./taxacsv.js";
+import { Files } from "../files.js";
+import puppeteer from "puppeteer";
+import { renameSync } from "node:fs";
 
 /**
  * @typedef {{id:string}} CCHTaxon
@@ -11,37 +14,94 @@ export class CCH2 {
     /**
      * @param {string} toolsDataDir
      * @param {string} dataDir
+     * @param {import("../exceptions.js").Exceptions} exceptions
      * @param {import("../taxa.js").Taxa} taxa
      * @param {import("../errorlog.js").ErrorLog} errorLog
      * @param {boolean} update
      */
-    static async analyze(toolsDataDir, dataDir, taxa, errorLog, update) {
+    static async analyze(
+        toolsDataDir,
+        dataDir,
+        exceptions,
+        taxa,
+        errorLog,
+        update,
+    ) {
         const toolsDataPath = path.join(toolsDataDir, "cch2");
 
         const cchTaxa = await getCCHTaxa(toolsDataPath, taxa);
 
         const idsToUpdate = new Map();
         for (const taxon of taxa.getTaxonList()) {
-            const cchTaxon = cchTaxa.get(taxon.getName());
+            const name = taxon.getName();
+            const cchTaxon = cchTaxa.get(name);
             if (!cchTaxon) {
-                errorLog.log(taxon.getName(), "not found in CCH data");
+                if (!exceptions.hasException(name, "cch", "notincch")) {
+                    errorLog.log(name, "not found in CCH data");
+                }
                 continue;
             }
             if (cchTaxon.id !== taxon.getCCH2ID()) {
                 errorLog.log(
-                    taxon.getName(),
+                    name,
                     "id in CCH data does not match id in taxa.csv",
                     cchTaxon.id,
                     taxon.getCCH2ID(),
                 );
-                idsToUpdate.set(taxon.getName(), cchTaxon.id);
+                idsToUpdate.set(name, cchTaxon.id);
             }
         }
 
+        this.#checkExceptions(exceptions, taxa, errorLog, cchTaxa);
+
         if (update) {
             updateTaxaCSV(dataDir, idsToUpdate);
         }
     }
+
+    /**
+     * @param {import("../exceptions.js").Exceptions} exceptions
+     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../errorlog.js").ErrorLog} errorLog
+     * @param {CCHTaxa} cchTaxa
+     */
+    static #checkExceptions(exceptions, taxa, errorLog, cchTaxa) {
+        // Check the CCH exceptions and make sure they still apply.
+        for (const [name, v] of exceptions.getExceptions()) {
+            const exceptions = v.cch;
+            if (!exceptions) {
+                continue;
+            }
+
+            // Make sure the taxon is still in our list.
+            const taxon = taxa.getTaxon(name);
+            if (!taxon) {
+                // Don't process global exceptions if taxon is not in local list.
+                if (taxa.isSubset() && !v.local) {
+                    continue;
+                }
+                errorLog.log(name, "has CCH exceptions but is not in taxa.tsv");
+                continue;
+            }
+
+            for (const [k] of Object.entries(exceptions)) {
+                const jepsonData = cchTaxa.get(name);
+                switch (k) {
+                    case "notincch":
+                        // Make sure it is really not in CCH data.
+                        if (jepsonData) {
+                            errorLog.log(
+                                name,
+                                "has CCH notincch exception but is in CCH data",
+                            );
+                        }
+                        break;
+                    default:
+                        errorLog.log(name, "unrecognized CCH exception", k);
+                }
+            }
+        }
+    }
 }
 
 /**
@@ -58,17 +118,22 @@ async function getCCHTaxa(toolsDataPath, taxa) {
             // Ignore ranks above species.
             return;
         }
-        if (record.acceptance !== "1") {
-            return;
-        }
         if (!taxa.getTaxon(record.scientificName)) {
             // If we're not tracking the taxon, ignore it.
             return;
         }
+        if (record.acceptance !== "1" && data.has(record.scientificName)) {
+            // Only add the synonym if there is no main entry.
+            return;
+        }
         data.set(record.scientificName, { id: record.acceptedTaxonID });
     }
 
     const fileName = path.join(toolsDataPath, "taxa.csv");
+    if (!Files.exists(fileName)) {
+        await retrieveDataFile(toolsDataPath);
+    }
+
     const data = new Map();
 
     await CSV.parseFileStream(fileName, callback);
@@ -76,6 +141,41 @@ async function getCCHTaxa(toolsDataPath, taxa) {
     return data;
 }
 
+/**
+ * @param {string} toolsDataPath
+ */
+async function retrieveDataFile(toolsDataPath) {
+    const url =
+        "https://www.cch2.org/portal/taxa/taxonomy/taxonomydynamicdisplay.php";
+    console.log(`retrieving file from ${url}`);
+
+    const browser = await puppeteer.launch({ headless: true });
+
+    const page = await browser.newPage();
+
+    await page.goto(url);
+    await page.locator("#taxontarget").fill("Tracheophyta");
+
+    // See https://stackoverflow.com/questions/53471235/how-to-wait-for-all-downloads-to-complete-with-puppeteer
+    const session = await browser.target().createCDPSession();
+    await session.send("Browser.setDownloadBehavior", {
+        behavior: "allowAndName",
+        downloadPath: path.resolve(toolsDataPath),
+        eventsEnabled: true,
+    });
+
+    await page.locator('button[value="exportTaxonTree"]').click();
+
+    const filename = await waitUntilDownload(session);
+    // Download file name is the guid; rename it to taxa.csv.
+    renameSync(
+        path.join(toolsDataPath, filename),
+        path.join(toolsDataPath, "taxa.csv"),
+    );
+
+    await browser.close();
+}
+
 /**
  * @param {string} dataDir
  * @param {Map<string,string>} idsToUpdate
@@ -93,3 +193,21 @@ function updateTaxaCSV(dataDir, idsToUpdate) {
 
     taxa.write();
 }
+
+/**
+ * @param {import("puppeteer").CDPSession} session
+ * @returns {Promise<string>}
+ * @see https://stackoverflow.com/questions/53471235/how-to-wait-for-all-downloads-to-complete-with-puppeteer
+ * @see https://scrapeops.io/puppeteer-web-scraping-playbook/nodejs-puppeteer-downloading-a-file/#setting-a-custom-download-behaviour
+ */
+async function waitUntilDownload(session) {
+    return new Promise((resolve, reject) => {
+        session.on("Browser.downloadProgress", (e) => {
+            if (e.state === "completed") {
+                resolve(e.guid);
+            } else if (e.state === "canceled") {
+                reject();
+            }
+        });
+    });
+}

package/lib/tools/fna.js

@@ -0,0 +1,163 @@
+import path from "node:path";
+import { Files } from "../files.js";
+import { scrape } from "@htmltools/scrape";
+import { TaxaCSV } from "./taxacsv.js";
+
+/**
+ * @typedef {{name:string}} FNATaxon
+ * @typedef {Map<string,FNATaxon>} FNATaxa
+ */
+
+export class FNA {
+    /**
+     * @param {string} toolsDataDir
+     * @param {string} dataDir
+     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../errorlog.js").ErrorLog} errorLog
+     * @param {boolean} update
+     */
+    static async analyze(toolsDataDir, dataDir, taxa, errorLog, update) {
+        const toolsDataPath = path.join(toolsDataDir, "fna");
+        const fnaTaxa = await getFNATaxa(toolsDataPath);
+
+        const namesToUpdate = new Map();
+
+        for (const taxon of taxa.getTaxonList()) {
+            const fnaName = getFNAName(taxon, fnaTaxa) ?? "";
+            const taxonFNAName = taxon.getFNAName();
+            if (fnaName !== taxonFNAName) {
+                const name = taxon.getName();
+                errorLog.log(
+                    name,
+                    "FNA name does not match name from FNA data",
+                    taxonFNAName,
+                    fnaName,
+                );
+                namesToUpdate.set(name, fnaName === name ? "true" : fnaName);
+            }
+        }
+
+        if (update) {
+            updateNames(dataDir, namesToUpdate);
+        }
+    }
+}
+
+/**
+ * @param {import("../taxon.js").Taxon} taxon
+ * @param {FNATaxa} fnaTaxa
+ * @returns {string|undefined}
+ */
+function getFNAName(taxon, fnaTaxa) {
+    /**
+     * @param {string} input
+     * @returns {string|undefined}
+     */
+    function getName(input) {
+        if (fnaTaxa.has(input)) {
+            return input;
+        }
+
+        // See if we can swap var./subsp. to find it.
+        const parts = input.split(" ");
+        if (parts.length === 4) {
+            parts[2] = parts[2] === "var." ? "subsp." : "var.";
+            input = parts.join(" ");
+            if (fnaTaxa.has(input)) {
+                return input;
+            }
+        }
+
+        // If it's the nominate subsp/var, see if we can find the species.
+        if (parts.length === 4 && parts[1] === parts[3]) {
+            input = parts[0] + " " + parts[1];
+            if (fnaTaxa.has(input)) {
+                return input;
+            }
+        }
+    }
+    const name = getName(taxon.getName());
+    if (name !== undefined) {
+        return name;
+    }
+
+    // See if any synonyms match.
+    for (const synonym of taxon.getSynonyms()) {
+        const name = getName(synonym);
+        if (name !== undefined) {
+            return name;
+        }
+    }
+}
+
+/**
+ * @param {string} toolsDataPath
+ * @returns {Promise<FNATaxa>}
+ */
+async function getFNATaxa(toolsDataPath) {
+    /** @type {FNATaxa} */
+    const fnaTaxa = new Map();
+
+    Files.mkdir(toolsDataPath);
+
+    // Get list of volumes.
+    const volumePage = path.join(toolsDataPath, "volumes.html");
+    if (!Files.exists(volumePage)) {
+        await Files.fetch(
+            "http://floranorthamerica.org/Special:SearchByProperty/:Volume/",
+            volumePage,
+        );
+    }
+
+    const volDoc = scrape.parseFile(volumePage);
+    const links = scrape.getSubtrees(volDoc, (e) => {
+        const href = scrape.getAttr(e, "href");
+        return href !== undefined && href.startsWith("/Volume_");
+    });
+    const vols = links.map((e) => scrape.getTextContent(e));
+
+    // For each volume, retrieve the JSON.
+    const baseURL =
+        "http://floranorthamerica.org/Special:Ask/limit=5000/unescape=true/format=json";
+    for (const vol of vols) {
+        const fileName = path.join(toolsDataPath, `${vol}.json`);
+        if (!Files.exists(fileName)) {
+            const url = baseURL + `/-5B-5BVolume::${vol}-5D-5D`;
+            await Files.fetch(url, fileName);
+        }
+
+        const text = Files.read(fileName);
+        /** @type {{results:Object<string,{}>}} */
+        const json = JSON.parse(text);
+        const results = json.results;
+
+        // If there are more than 5000 results, this will need to be updated to retrieve chunks.
+        if (Object.entries(results).length >= 5000) {
+            throw new Error(`${vol} has more than 5000 results`);
+        }
+
+        for (const [k] of Object.entries(results)) {
+            fnaTaxa.set(k, { name: k });
+        }
+    }
+
+    return fnaTaxa;
+}
+
+/**
+ * @param {string} dataDir
+ * @param {Map<string,string>} namesToUpdate
+ */
+function updateNames(dataDir, namesToUpdate) {
+    const taxa = new TaxaCSV(dataDir);
+
+    for (const taxonData of taxa.getTaxa()) {
+        const newName = namesToUpdate.get(taxonData.taxon_name);
+        if (!newName) {
+            continue;
+        }
+        taxonData.fna = newName;
+    }
+
+    taxa.write();
+}