@ca-plant-list/ca-plant-list 0.4.21 → 0.4.23

package/lib/{taxon.js → taxonomy/taxon.js}

@@ -1,20 +1,17 @@
-import { HTML } from "./html.js";
-import { RarePlants } from "./rareplants.js";
+import { Taxonomy } from "./taxonomy.js";
 
-class Taxon {
+class Taxon extends Taxonomy {
     #genera;
     #name;
     #genus;
     #commonNames;
     #status;
-    #jepsonID;
     #calRecNum;
-    /**@type {string|undefined} */
-    #cfSyn;
     #iNatID;
     /**@type {string|undefined} */
     #iNatSyn;
     #cch2id;
+    #fnaName;
     #calscapeCN;
     #lifeCycle;
     #flowerColors;
@@ -28,14 +25,15 @@ class Taxon {
     #rankGlobal;
     /** @type {string[]} */
     #synonyms = [];
-    /** @type {import("./photo.js").Photo[]}*/
+    /** @type {import("../photo.js").Photo[]}*/
     #photos = [];
 
     /**
-     * @param {import("./index.js").TaxonData} data
-     * @param {import("./genera.js").Genera} genera
+     * @param {import("../index.js").TaxonData} data
+     * @param {import("../genera.js").Genera} genera
      */
     constructor(data, genera) {
+        super(data);
         this.#genera = genera;
         const name = data["taxon_name"];
         const commonNames = data["common name"];
@@ -49,10 +47,10 @@ class Taxon {
             ? commonNames.split(",").map((t) => t.trim())
             : [];
         this.#status = data["status"];
-        this.#jepsonID = data["jepson id"];
         this.#calRecNum = data["calrecnum"];
         this.#iNatID = data["inat id"];
         this.#cch2id = data.cch2_id;
+        this.#fnaName = data.fna ?? "";
         this.#calscapeCN =
             data.calscape_cn === "" ? undefined : data.calscape_cn;
         this.#lifeCycle = data.life_cycle;
@@ -80,10 +78,6 @@ class Taxon {
     addSynonym(syn, type) {
         this.#synonyms.push(syn);
         switch (type) {
-            case "CF":
-                // Synonym is in Calflora format.
-                this.#cfSyn = syn;
-                break;
             case "INAT":
                 // Synonyms should be in Jepson format, but store iNatName in iNat format (no var or subsp, space after x).
                 this.#iNatSyn = syn;
@@ -92,14 +86,14 @@ class Taxon {
     }
 
     /**
-     * @param {import("./photo.js").Photo} photo
+     * @param {import("../photo.js").Photo} photo
      */
     addPhoto(photo) {
         this.#photos = this.#photos.concat([photo]);
     }
 
     /**
-     * @returns {import("./photo.js").Photo[]}
+     * @returns {import("../photo.js").Photo[]}
      */
     getPhotos() {
         return this.#photos;
@@ -125,9 +119,6 @@ class Taxon {
     }
 
     getCalfloraName() {
-        if (this.#cfSyn) {
-            return this.#cfSyn;
-        }
         return this.getName().replace(" subsp.", " ssp.").replace("×", "X");
     }
 
@@ -135,20 +126,6 @@ class Taxon {
         return this.#calRecNum;
     }
 
-    getCalfloraTaxonLink() {
-        const calfloraID = this.getCalfloraID();
-        if (!calfloraID) {
-            return;
-        }
-        const link = HTML.getLink(
-            "https://www.calflora.org/app/taxon?crn=" + calfloraID,
-            "Calflora",
-            {},
-            true,
-        );
-        return this.#cfSyn ? link + " (" + this.#cfSyn + ")" : link;
-    }
-
     getCalscapeCommonName() {
         return this.#calscapeCN;
     }
@@ -171,12 +148,18 @@ class Taxon {
         return this.#cch2id;
     }
 
+    /**
+     * @returns {string}
+     */
     getCESA() {
-        return this.#cesa;
+        return this.#cesa ?? "";
     }
 
+    /**
+     * @returns {string}
+     */
     getCNDDBRank() {
-        return this.#rankCNDDB;
+        return this.#rankCNDDB ?? "";
     }
 
     getCommonNames() {
@@ -187,14 +170,27 @@ class Taxon {
         return this.getGenus().getFamily();
     }
 
+    /**
+     * @returns {string}
+     */
     getFESA() {
-        return this.#fesa;
+        return this.#fesa ?? "";
     }
 
     getFileName(ext = "html") {
         return this.getBaseFileName() + "." + ext;
     }
 
+    /**
+     * @returns {string}
+     */
+    getFNAName() {
+        if (this.#fnaName === "true") {
+            return this.getName();
+        }
+        return this.#fnaName;
+    }
+
     getFlowerColors() {
         return this.#flowerColors;
     }
@@ -207,35 +203,11 @@ class Taxon {
         return this.#genus;
     }
 
-    getGlobalRank() {
-        return this.#rankGlobal;
-    }
-
     /**
-     *
-     * @param {boolean|string|undefined} href
-     * @param {boolean} includeRPI
+     * @returns {string}
      */
-    getHTMLLink(href = true, includeRPI = true) {
-        href = href ? "./" + this.getFileName() : undefined;
-        let className = this.isNative() ? "native" : "non-native";
-        let isRare = false;
-        if (includeRPI && this.isRare()) {
-            isRare = true;
-            className += " rare";
-        }
-        const attributes = { class: className };
-        const link = HTML.wrap(
-            "span",
-            HTML.getLink(href, this.getName()),
-            attributes,
-        );
-        if (isRare) {
-            return HTML.getToolTip(link, this.getRPIRankAndThreatTooltip(), {
-                icon: false,
-            });
-        }
-        return link;
+    getGlobalRank() {
+        return this.#rankGlobal ?? "";
     }
 
     getINatID() {
@@ -254,27 +226,6 @@ class Taxon {
         return this.#iNatSyn;
     }
 
-    getINatTaxonLink() {
-        const iNatID = this.getINatID();
-        if (!iNatID) {
-            return "";
-        }
-        const link = HTML.getLink(
-            "https://www.inaturalist.org/taxa/" + iNatID,
-            "iNaturalist",
-            {},
-            true,
-        );
-        return this.#iNatSyn ? link + " (" + this.#iNatSyn + ")" : link;
-    }
-
-    /**
-     * @returns {string}
-     */
-    getJepsonID() {
-        return this.#jepsonID;
-    }
-
     getLifeCycle() {
         return this.#lifeCycle;
     }
@@ -287,6 +238,9 @@ class Taxon {
         return this.#rpiID;
     }
 
+    /**
+     * @returns {string}
+     */
     getRPIRank() {
         if (!this.#rankRPI) {
             return this.#rankRPI;
@@ -301,38 +255,12 @@ class Taxon {
         return this.#rankRPI;
     }
 
-    /**
-     * @deprecated
-     */
-    getRPIRankAndThreatTooltip() {
-        return RarePlants.getRPIRankAndThreatDescriptions(
-            this.getRPIRankAndThreat(),
-        ).join("<br>");
-    }
-
-    /**
-     * @deprecated
-     */
-    getRPITaxonLink() {
-        const rpiID = this.getRPIID();
-        if (!rpiID) {
-            return "";
-        }
-        const link = HTML.getLink(
-            "https://rareplants.cnps.org/Plants/Details/" + rpiID,
-            "CNPS Rare Plant Inventory",
-            {},
-            true,
-        );
-        return link;
-    }
-
     getStatus() {
         return this.#status;
     }
 
     /**
-     * @param {import("./config.js").Config} config
+     * @param {import("../config.js").Config} config
      * @returns {string}
      */
     getStatusDescription(config) {
@@ -341,6 +269,8 @@ class Taxon {
                 return "Native";
             case "NC":
                 return config.getLabel("status-NC", "Introduced");
+            case "U":
+                return "Nativity Uncertain";
             case "X":
                 return "Introduced";
         }

package/lib/taxonomy/taxonomy.js

@@ -0,0 +1,17 @@
+export class Taxonomy {
+    #data;
+
+    /**
+     * @param {import("../index.js").TaxonomyData} data
+     */
+    constructor(data) {
+        this.#data = data;
+    }
+
+    /**
+     * @returns {string}
+     */
+    getJepsonID() {
+        return this.#data["jepson id"];
+    }
+}

package/lib/tools/calflora.js

@@ -25,7 +25,7 @@ export class Calflora {
     /**
      * @param {string} toolsDataDir
      * @param {string} dataDir
-     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../types.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
      * @param {import("../errorlog.js").ErrorLog} errorLog
      * @param {boolean} update
@@ -175,7 +175,7 @@ export class Calflora {
     }
 
     /**
-     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../types.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
      * @param {import("../errorlog.js").ErrorLog} errorLog
      */

package/lib/tools/calscape.js

@@ -2,13 +2,13 @@ import path from "node:path";
 import xlsx from "exceljs";
 import { Files } from "../files.js";
 import { TaxaCSV } from "./taxacsv.js";
-import { Taxon } from "../taxon.js";
+import { Taxon } from "../taxonomy/taxon.js";
 
 export class Calscape {
     /**
      * @param {string} toolsDataDir
      * @param {string} dataDir
-     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../types.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
      * @param {import("../errorlog.js").ErrorLog} errorLog
      * @param {boolean} update
@@ -58,7 +58,7 @@ export class Calscape {
 }
 
 /**
- * @param {import("../taxa.js").Taxa} taxa
+ * @param {import("../types.js").Taxa} taxa
  * @param {import("../exceptions.js").Exceptions} exceptions
  * @param {import("../errorlog.js").ErrorLog} errorLog
  */

package/lib/tools/cch2.js

@@ -1,6 +1,9 @@
 import path from "node:path";
 import { CSV } from "../csv.js";
 import { TaxaCSV } from "./taxacsv.js";
+import { Files } from "../files.js";
+import puppeteer from "puppeteer";
+import { renameSync } from "node:fs";
 
 /**
  * @typedef {{id:string}} CCHTaxon
@@ -11,42 +14,99 @@ export class CCH2 {
     /**
      * @param {string} toolsDataDir
      * @param {string} dataDir
-     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../exceptions.js").Exceptions} exceptions
+     * @param {import("../types.js").Taxa} taxa
      * @param {import("../errorlog.js").ErrorLog} errorLog
      * @param {boolean} update
      */
-    static async analyze(toolsDataDir, dataDir, taxa, errorLog, update) {
+    static async analyze(
+        toolsDataDir,
+        dataDir,
+        exceptions,
+        taxa,
+        errorLog,
+        update,
+    ) {
         const toolsDataPath = path.join(toolsDataDir, "cch2");
 
         const cchTaxa = await getCCHTaxa(toolsDataPath, taxa);
 
         const idsToUpdate = new Map();
         for (const taxon of taxa.getTaxonList()) {
-            const cchTaxon = cchTaxa.get(taxon.getName());
+            const name = taxon.getName();
+            const cchTaxon = cchTaxa.get(name);
             if (!cchTaxon) {
-                errorLog.log(taxon.getName(), "not found in CCH data");
+                if (!exceptions.hasException(name, "cch", "notincch")) {
+                    errorLog.log(name, "not found in CCH data");
+                }
                 continue;
             }
             if (cchTaxon.id !== taxon.getCCH2ID()) {
                 errorLog.log(
-                    taxon.getName(),
+                    name,
                     "id in CCH data does not match id in taxa.csv",
                     cchTaxon.id,
                     taxon.getCCH2ID(),
                 );
-                idsToUpdate.set(taxon.getName(), cchTaxon.id);
+                idsToUpdate.set(name, cchTaxon.id);
             }
         }
 
+        this.#checkExceptions(exceptions, taxa, errorLog, cchTaxa);
+
         if (update) {
             updateTaxaCSV(dataDir, idsToUpdate);
         }
     }
+
+    /**
+     * @param {import("../exceptions.js").Exceptions} exceptions
+     * @param {import("../types.js").Taxa} taxa
+     * @param {import("../errorlog.js").ErrorLog} errorLog
+     * @param {CCHTaxa} cchTaxa
+     */
+    static #checkExceptions(exceptions, taxa, errorLog, cchTaxa) {
+        // Check the CCH exceptions and make sure they still apply.
+        for (const [name, v] of exceptions.getExceptions()) {
+            const exceptions = v.cch;
+            if (!exceptions) {
+                continue;
+            }
+
+            // Make sure the taxon is still in our list.
+            const taxon = taxa.getTaxon(name);
+            if (!taxon) {
+                // Don't process global exceptions if taxon is not in local list.
+                if (taxa.isSubset() && !v.local) {
+                    continue;
+                }
+                errorLog.log(name, "has CCH exceptions but is not in taxa.tsv");
+                continue;
+            }
+
+            for (const [k] of Object.entries(exceptions)) {
+                const jepsonData = cchTaxa.get(name);
+                switch (k) {
+                    case "notincch":
+                        // Make sure it is really not in CCH data.
+                        if (jepsonData) {
+                            errorLog.log(
+                                name,
+                                "has CCH notincch exception but is in CCH data",
+                            );
+                        }
+                        break;
+                    default:
+                        errorLog.log(name, "unrecognized CCH exception", k);
+                }
+            }
+        }
+    }
 }
 
 /**
  * @param {string} toolsDataPath
- * @param {import("../taxa.js").Taxa} taxa
+ * @param {import("../types.js").Taxa} taxa
  * @returns {Promise<CCHTaxa>}
  */
 async function getCCHTaxa(toolsDataPath, taxa) {
@@ -58,17 +118,22 @@ async function getCCHTaxa(toolsDataPath, taxa) {
             // Ignore ranks above species.
             return;
         }
-        if (record.acceptance !== "1") {
-            return;
-        }
         if (!taxa.getTaxon(record.scientificName)) {
             // If we're not tracking the taxon, ignore it.
             return;
         }
+        if (record.acceptance !== "1" && data.has(record.scientificName)) {
+            // Only add the synonym if there is no main entry.
+            return;
+        }
         data.set(record.scientificName, { id: record.acceptedTaxonID });
     }
 
     const fileName = path.join(toolsDataPath, "taxa.csv");
+    if (!Files.exists(fileName)) {
+        await retrieveDataFile(toolsDataPath);
+    }
+
     const data = new Map();
 
     await CSV.parseFileStream(fileName, callback);
@@ -76,6 +141,41 @@ async function getCCHTaxa(toolsDataPath, taxa) {
     return data;
 }
 
+/**
+ * @param {string} toolsDataPath
+ */
+async function retrieveDataFile(toolsDataPath) {
+    const url =
+        "https://www.cch2.org/portal/taxa/taxonomy/taxonomydynamicdisplay.php";
+    console.log(`retrieving file from ${url}`);
+
+    const browser = await puppeteer.launch({ headless: true });
+
+    const page = await browser.newPage();
+
+    await page.goto(url);
+    await page.locator("#taxontarget").fill("Tracheophyta");
+
+    // See https://stackoverflow.com/questions/53471235/how-to-wait-for-all-downloads-to-complete-with-puppeteer
+    const session = await browser.target().createCDPSession();
+    await session.send("Browser.setDownloadBehavior", {
+        behavior: "allowAndName",
+        downloadPath: path.resolve(toolsDataPath),
+        eventsEnabled: true,
+    });
+
+    await page.locator('button[value="exportTaxonTree"]').click();
+
+    const filename = await waitUntilDownload(session);
+    // Download file name is the guid; rename it to taxa.csv.
+    renameSync(
+        path.join(toolsDataPath, filename),
+        path.join(toolsDataPath, "taxa.csv"),
+    );
+
+    await browser.close();
+}
+
 /**
  * @param {string} dataDir
  * @param {Map<string,string>} idsToUpdate
@@ -93,3 +193,21 @@ function updateTaxaCSV(dataDir, idsToUpdate) {
 
     taxa.write();
 }
+
+/**
+ * @param {import("puppeteer").CDPSession} session
+ * @returns {Promise<string>}
+ * @see https://stackoverflow.com/questions/53471235/how-to-wait-for-all-downloads-to-complete-with-puppeteer
+ * @see https://scrapeops.io/puppeteer-web-scraping-playbook/nodejs-puppeteer-downloading-a-file/#setting-a-custom-download-behaviour
+ */
+async function waitUntilDownload(session) {
+    return new Promise((resolve, reject) => {
+        session.on("Browser.downloadProgress", (e) => {
+            if (e.state === "completed") {
+                resolve(e.guid);
+            } else if (e.state === "canceled") {
+                reject();
+            }
+        });
+    });
+}