@ca-plant-list/ca-plant-list 0.4.20 → 0.4.22

package/lib/tools/cch2.js

@@ -0,0 +1,213 @@
+import path from "node:path";
+import { CSV } from "../csv.js";
+import { TaxaCSV } from "./taxacsv.js";
+import { Files } from "../files.js";
+import puppeteer from "puppeteer";
+import { renameSync } from "node:fs";
+
+/**
+ * @typedef {{id:string}} CCHTaxon
+ * @typedef {Map<string,CCHTaxon>} CCHTaxa
+ */
+
+export class CCH2 {
+    /**
+     * @param {string} toolsDataDir
+     * @param {string} dataDir
+     * @param {import("../exceptions.js").Exceptions} exceptions
+     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../errorlog.js").ErrorLog} errorLog
+     * @param {boolean} update
+     */
+    static async analyze(
+        toolsDataDir,
+        dataDir,
+        exceptions,
+        taxa,
+        errorLog,
+        update,
+    ) {
+        const toolsDataPath = path.join(toolsDataDir, "cch2");
+
+        const cchTaxa = await getCCHTaxa(toolsDataPath, taxa);
+
+        const idsToUpdate = new Map();
+        for (const taxon of taxa.getTaxonList()) {
+            const name = taxon.getName();
+            const cchTaxon = cchTaxa.get(name);
+            if (!cchTaxon) {
+                if (!exceptions.hasException(name, "cch", "notincch")) {
+                    errorLog.log(name, "not found in CCH data");
+                }
+                continue;
+            }
+            if (cchTaxon.id !== taxon.getCCH2ID()) {
+                errorLog.log(
+                    name,
+                    "id in CCH data does not match id in taxa.csv",
+                    cchTaxon.id,
+                    taxon.getCCH2ID(),
+                );
+                idsToUpdate.set(name, cchTaxon.id);
+            }
+        }
+
+        this.#checkExceptions(exceptions, taxa, errorLog, cchTaxa);
+
+        if (update) {
+            updateTaxaCSV(dataDir, idsToUpdate);
+        }
+    }
+
+    /**
+     * @param {import("../exceptions.js").Exceptions} exceptions
+     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../errorlog.js").ErrorLog} errorLog
+     * @param {CCHTaxa} cchTaxa
+     */
+    static #checkExceptions(exceptions, taxa, errorLog, cchTaxa) {
+        // Check the CCH exceptions and make sure they still apply.
+        for (const [name, v] of exceptions.getExceptions()) {
+            const exceptions = v.cch;
+            if (!exceptions) {
+                continue;
+            }
+
+            // Make sure the taxon is still in our list.
+            const taxon = taxa.getTaxon(name);
+            if (!taxon) {
+                // Don't process global exceptions if taxon is not in local list.
+                if (taxa.isSubset() && !v.local) {
+                    continue;
+                }
+                errorLog.log(name, "has CCH exceptions but is not in taxa.tsv");
+                continue;
+            }
+
+            for (const [k] of Object.entries(exceptions)) {
+                const jepsonData = cchTaxa.get(name);
+                switch (k) {
+                    case "notincch":
+                        // Make sure it is really not in CCH data.
+                        if (jepsonData) {
+                            errorLog.log(
+                                name,
+                                "has CCH notincch exception but is in CCH data",
+                            );
+                        }
+                        break;
+                    default:
+                        errorLog.log(name, "unrecognized CCH exception", k);
+                }
+            }
+        }
+    }
+}
+
+/**
+ * @param {string} toolsDataPath
+ * @param {import("../taxa.js").Taxa} taxa
+ * @returns {Promise<CCHTaxa>}
+ */
+async function getCCHTaxa(toolsDataPath, taxa) {
+    /**
+     * @param {{taxonID:string,scientificName:string,rankID:string,acceptance:"0"|"1",acceptedTaxonID:string}} record
+     */
+    function callback(record) {
+        if (parseInt(record.rankID) < 220) {
+            // Ignore ranks above species.
+            return;
+        }
+        if (!taxa.getTaxon(record.scientificName)) {
+            // If we're not tracking the taxon, ignore it.
+            return;
+        }
+        if (record.acceptance !== "1" && data.has(record.scientificName)) {
+            // Only add the synonym if there is no main entry.
+            return;
+        }
+        data.set(record.scientificName, { id: record.acceptedTaxonID });
+    }
+
+    const fileName = path.join(toolsDataPath, "taxa.csv");
+    if (!Files.exists(fileName)) {
+        await retrieveDataFile(toolsDataPath);
+    }
+
+    const data = new Map();
+
+    await CSV.parseFileStream(fileName, callback);
+
+    return data;
+}
+
+/**
+ * @param {string} toolsDataPath
+ */
+async function retrieveDataFile(toolsDataPath) {
+    const url =
+        "https://www.cch2.org/portal/taxa/taxonomy/taxonomydynamicdisplay.php";
+    console.log(`retrieving file from ${url}`);
+
+    const browser = await puppeteer.launch({ headless: true });
+
+    const page = await browser.newPage();
+
+    await page.goto(url);
+    await page.locator("#taxontarget").fill("Tracheophyta");
+
+    // See https://stackoverflow.com/questions/53471235/how-to-wait-for-all-downloads-to-complete-with-puppeteer
+    const session = await browser.target().createCDPSession();
+    await session.send("Browser.setDownloadBehavior", {
+        behavior: "allowAndName",
+        downloadPath: path.resolve(toolsDataPath),
+        eventsEnabled: true,
+    });
+
+    await page.locator('button[value="exportTaxonTree"]').click();
+
+    const filename = await waitUntilDownload(session);
+    // Download file name is the guid; rename it to taxa.csv.
+    renameSync(
+        path.join(toolsDataPath, filename),
+        path.join(toolsDataPath, "taxa.csv"),
+    );
+
+    await browser.close();
+}
+
+/**
+ * @param {string} dataDir
+ * @param {Map<string,string>} idsToUpdate
+ */
+function updateTaxaCSV(dataDir, idsToUpdate) {
+    const taxa = new TaxaCSV(dataDir);
+
+    for (const taxonData of taxa.getTaxa()) {
+        const id = idsToUpdate.get(taxonData.taxon_name);
+        if (!id) {
+            continue;
+        }
+        taxonData.cch2_id = id;
+    }
+
+    taxa.write();
+}
+
+/**
+ * @param {import("puppeteer").CDPSession} session
+ * @returns {Promise<string>}
+ * @see https://stackoverflow.com/questions/53471235/how-to-wait-for-all-downloads-to-complete-with-puppeteer
+ * @see https://scrapeops.io/puppeteer-web-scraping-playbook/nodejs-puppeteer-downloading-a-file/#setting-a-custom-download-behaviour
+ */
+async function waitUntilDownload(session) {
+    return new Promise((resolve, reject) => {
+        session.on("Browser.downloadProgress", (e) => {
+            if (e.state === "completed") {
+                resolve(e.guid);
+            } else if (e.state === "canceled") {
+                reject();
+            }
+        });
+    });
+}

package/lib/tools/fna.js

@@ -0,0 +1,163 @@
+import path from "node:path";
+import { Files } from "../files.js";
+import { scrape } from "@htmltools/scrape";
+import { TaxaCSV } from "./taxacsv.js";
+
+/**
+ * @typedef {{name:string}} FNATaxon
+ * @typedef {Map<string,FNATaxon>} FNATaxa
+ */
+
+export class FNA {
+    /**
+     * @param {string} toolsDataDir
+     * @param {string} dataDir
+     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../errorlog.js").ErrorLog} errorLog
+     * @param {boolean} update
+     */
+    static async analyze(toolsDataDir, dataDir, taxa, errorLog, update) {
+        const toolsDataPath = path.join(toolsDataDir, "fna");
+        const fnaTaxa = await getFNATaxa(toolsDataPath);
+
+        const namesToUpdate = new Map();
+
+        for (const taxon of taxa.getTaxonList()) {
+            const fnaName = getFNAName(taxon, fnaTaxa) ?? "";
+            const taxonFNAName = taxon.getFNAName();
+            if (fnaName !== taxonFNAName) {
+                const name = taxon.getName();
+                errorLog.log(
+                    name,
+                    "FNA name does not match name from FNA data",
+                    taxonFNAName,
+                    fnaName,
+                );
+                namesToUpdate.set(name, fnaName === name ? "true" : fnaName);
+            }
+        }
+
+        if (update) {
+            updateNames(dataDir, namesToUpdate);
+        }
+    }
+}
+
+/**
+ * @param {import("../taxon.js").Taxon} taxon
+ * @param {FNATaxa} fnaTaxa
+ * @returns {string|undefined}
+ */
+function getFNAName(taxon, fnaTaxa) {
+    /**
+     * @param {string} input
+     * @returns {string|undefined}
+     */
+    function getName(input) {
+        if (fnaTaxa.has(input)) {
+            return input;
+        }
+
+        // See if we can swap var./subsp. to find it.
+        const parts = input.split(" ");
+        if (parts.length === 4) {
+            parts[2] = parts[2] === "var." ? "subsp." : "var.";
+            input = parts.join(" ");
+            if (fnaTaxa.has(input)) {
+                return input;
+            }
+        }
+
+        // If it's the nominate subsp/var, see if we can find the species.
+        if (parts.length === 4 && parts[1] === parts[3]) {
+            input = parts[0] + " " + parts[1];
+            if (fnaTaxa.has(input)) {
+                return input;
+            }
+        }
+    }
+    const name = getName(taxon.getName());
+    if (name !== undefined) {
+        return name;
+    }
+
+    // See if any synonyms match.
+    for (const synonym of taxon.getSynonyms()) {
+        const name = getName(synonym);
+        if (name !== undefined) {
+            return name;
+        }
+    }
+}
+
+/**
+ * @param {string} toolsDataPath
+ * @returns {Promise<FNATaxa>}
+ */
+async function getFNATaxa(toolsDataPath) {
+    /** @type {FNATaxa} */
+    const fnaTaxa = new Map();
+
+    Files.mkdir(toolsDataPath);
+
+    // Get list of volumes.
+    const volumePage = path.join(toolsDataPath, "volumes.html");
+    if (!Files.exists(volumePage)) {
+        await Files.fetch(
+            "http://floranorthamerica.org/Special:SearchByProperty/:Volume/",
+            volumePage,
+        );
+    }
+
+    const volDoc = scrape.parseFile(volumePage);
+    const links = scrape.getSubtrees(volDoc, (e) => {
+        const href = scrape.getAttr(e, "href");
+        return href !== undefined && href.startsWith("/Volume_");
+    });
+    const vols = links.map((e) => scrape.getTextContent(e));
+
+    // For each volume, retrieve the JSON.
+    const baseURL =
+        "http://floranorthamerica.org/Special:Ask/limit=5000/unescape=true/format=json";
+    for (const vol of vols) {
+        const fileName = path.join(toolsDataPath, `${vol}.json`);
+        if (!Files.exists(fileName)) {
+            const url = baseURL + `/-5B-5BVolume::${vol}-5D-5D`;
+            await Files.fetch(url, fileName);
+        }
+
+        const text = Files.read(fileName);
+        /** @type {{results:Object<string,{}>}} */
+        const json = JSON.parse(text);
+        const results = json.results;
+
+        // If there are more than 5000 results, this will need to be updated to retrieve chunks.
+        if (Object.entries(results).length >= 5000) {
+            throw new Error(`${vol} has more than 5000 results`);
+        }
+
+        for (const [k] of Object.entries(results)) {
+            fnaTaxa.set(k, { name: k });
+        }
+    }
+
+    return fnaTaxa;
+}
+
+/**
+ * @param {string} dataDir
+ * @param {Map<string,string>} namesToUpdate
+ */
+function updateNames(dataDir, namesToUpdate) {
+    const taxa = new TaxaCSV(dataDir);
+
+    for (const taxonData of taxa.getTaxa()) {
+        const newName = namesToUpdate.get(taxonData.taxon_name);
+        if (!newName) {
+            continue;
+        }
+        taxonData.fna = newName;
+    }
+
+    taxa.write();
+}

package/lib/tools/jepsoneflora.js

@@ -102,8 +102,25 @@ export class JepsonEFlora {
                 idsToUpdate.set(name, jepsInfo.id);
             }
 
-            const efStatus = this.#getStatusCode(jepsInfo);
+            let efStatus = this.#getStatusCode(jepsInfo);
             const taxonStatus = taxon.getStatus();
+            // Override if exception is specified.
+            const nativeException = exceptions.getValue(
+                name,
+                "jepson",
+                "native",
+            );
+            if (typeof nativeException === "boolean") {
+                const overrideStatus = nativeException ? "N" : "X";
+                if (overrideStatus === efStatus) {
+                    this.#errorLog.log(
+                        name,
+                        "has unnecessary Jepson native override",
+                    );
+                }
+                efStatus = overrideStatus;
+            }
+
             if (
                 efStatus !== taxonStatus &&
                 !(taxonStatus === "NC" && efStatus === "N")
@@ -154,6 +171,8 @@ export class JepsonEFlora {
             for (const [k] of Object.entries(exceptions)) {
                 const jepsonData = this.#nameInfo.get(name);
                 switch (k) {
+                    case "native":
+                        break;
                     case "notineflora":
                         // Make sure it is really not in eFlora.
                         if (jepsonData) {
@@ -228,7 +247,7 @@ export class JepsonEFlora {
 
     /**
      * @param {JepsonTaxon} jepsInfo
-     * @returns {import("../taxon.js").StatusCode|undefined}
+     * @returns {import("../index.js").NativeStatusCode|undefined}
      */
     #getStatusCode(jepsInfo) {
         switch (jepsInfo.type) {

package/lib/tools/rpi.js

@@ -80,8 +80,8 @@ class RPI {
             const rank = row["CRPR"];
             const rawCESA = row["CESA"];
             const rawFESA = row["FESA"];
-            const cesa = rawCESA === "None" ? undefined : rawCESA;
-            const fesa = rawFESA === "None" ? undefined : rawFESA;
+            const cesa = rawCESA === "None" ? "" : rawCESA;
+            const fesa = rawFESA === "None" ? "" : rawFESA;
             const cnddb = row["SRank"];
             const globalRank = row["GRank"];
             const taxon = taxa.getTaxon(name);

package/lib/tools/taxacsv.js

@@ -1,9 +1,30 @@
 import path from "path";
 import { CSV } from "../csv.js";
 
+const HEADERS = [
+    "taxon_name",
+    "common name",
+    "status",
+    "jepson id",
+    "calrecnum",
+    "inat id",
+    "cch2_id",
+    "fna",
+    "calscape_cn",
+    "life_cycle",
+    "flower_color",
+    "bloom_start",
+    "bloom_end",
+    "RPI ID",
+    "CRPR",
+    "CESA",
+    "FESA",
+    "SRank",
+    "GRank",
+];
+
 export class TaxaCSV {
     #filePath;
-    #headers;
     /** @type {import("../index.js").TaxonData[]} */
     #taxa;
 
@@ -13,9 +34,7 @@ export class TaxaCSV {
     constructor(dataDir) {
         this.#filePath = path.join(dataDir, "taxa.csv");
         const csv = CSV.readFileAndHeaders(this.#filePath);
-        // @ts-ignore
         this.#taxa = csv.data;
-        this.#headers = csv.headers;
     }
 
     /**
@@ -26,6 +45,6 @@ export class TaxaCSV {
     }
 
     write() {
-        CSV.writeFileObject(this.#filePath, this.#taxa, this.#headers);
+        CSV.writeFileObject(this.#filePath, this.#taxa, HEADERS);
     }
 }

package/lib/web/pagetaxon.js

@@ -5,7 +5,7 @@ import { ExternalSites } from "../externalsites.js";
 import { HTML } from "../html.js";
 import { HTMLTaxon } from "../htmltaxon.js";
 
-class PageTaxon extends GenericPage {
+export class PageTaxon extends GenericPage {
     #config;
     #taxon;
 
@@ -34,14 +34,13 @@ class PageTaxon extends GenericPage {
         if (iNatLink) {
             links.push(iNatLink);
         }
-        const calscapeLink = HTMLTaxon.getCalscapeLink(this.#taxon);
-        if (calscapeLink) {
-            links.push(calscapeLink);
-        }
         const rpiLink = this.#taxon.getRPITaxonLink();
         if (rpiLink) {
             links.push(rpiLink);
         }
+        HTMLTaxon.addRefLink(links, this.#taxon, "fna");
+        HTMLTaxon.addRefLink(links, this.#taxon, "cch");
+        HTMLTaxon.addRefLink(links, this.#taxon, "calscape");
         return links;
     }
 
@@ -76,6 +75,11 @@ class PageTaxon extends GenericPage {
                 ),
             );
         }
+        HTMLTaxon.addLink(
+            links,
+            ExternalSites.getCCH2ObsLink(this.#taxon, this.#config),
+            "CCH2",
+        );
 
         return links;
     }
@@ -218,5 +222,3 @@ class PageTaxon extends GenericPage {
         this.writeFile(html);
     }
 }
-
-export { PageTaxon };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@ca-plant-list/ca-plant-list",
-    "version": "0.4.20",
+    "version": "0.4.22",
     "description": "Tools to create Jekyll files for a website listing plants in an area of California.",
     "license": "MIT",
     "repository": {
@@ -58,6 +58,7 @@
         "eslint": "^9.18.0",
         "jest": "^29.7.0",
         "prettier": "^3.4.2",
+        "puppeteer": "^24.1.1",
         "typescript": "^5.7.3"
     }
 }

package/scripts/cpl-tools.js

@@ -13,10 +13,14 @@ import { Config } from "../lib/config.js";
 import { Taxa } from "../lib/taxa.js";
 import { SupplementalText } from "../lib/tools/supplementaltext.js";
 import { JepsonFamilies } from "../lib/tools/jepsonfamilies.js";
+import { CCH2 } from "../lib/tools/cch2.js";
+import { FNA } from "../lib/tools/fna.js";
 
 const TOOLS = {
     CALFLORA: "calflora",
     CALSCAPE: "calscape",
+    CCH2: "cch",
+    FNA: "fna",
     INAT: "inat",
     JEPSON_EFLORA: "jepson-eflora",
     JEPSON_FAM: "jepson-families",
@@ -27,6 +31,8 @@ const TOOLS = {
 const ALL_TOOLS = [
     TOOLS.CALFLORA,
     TOOLS.CALSCAPE,
+    TOOLS.CCH2,
+    TOOLS.FNA,
     TOOLS.INAT,
     TOOLS.JEPSON_EFLORA,
     TOOLS.RPI,
@@ -77,6 +83,26 @@ async function build(program, options) {
                     !!options.update,
                 );
                 break;
+            case TOOLS.CCH2:
+                await CCH2.analyze(
+                    TOOLS_DATA_DIR,
+                    options.datadir,
+                    exceptions,
+                    taxa,
+                    errorLog,
+                    !!options.update,
+                );
+                break;
+            case TOOLS.FNA:
+                await FNA.analyze(
+                    TOOLS_DATA_DIR,
+                    options.datadir,
+                    taxa,
+                    errorLog,
+                    !!options.update,
+                );
+                break;
+
             case TOOLS.INAT:
                 await INat.analyze(
                     TOOLS_DATA_DIR,
@@ -138,9 +164,11 @@ program.addHelpText(
     "after",
     `
 Tools:
-    'all' runs the 'calflora', '${TOOLS.CALSCAPE}', 'inat', 'jepson-eflora', 'rpi', and 'text' tools.
+    'all' runs the 'calflora', '${TOOLS.CALSCAPE}', '${TOOLS.CCH2}, '${TOOLS.FNA}, 'inat', 'jepson-eflora', 'rpi', and 'text' tools.
     '${TOOLS.CALFLORA}' retrieves data from Calflora and compares with local data.
     '${TOOLS.CALSCAPE}' retrieves data from Calscape and compares with local data.
+    '${TOOLS.CCH2}' retrieves data from CCH2 and compares with local data.
+    '${TOOLS.FNA}' retrieves data from Flora of North America and compares with local data.
     '${TOOLS.INAT}' retrieves data from iNaturalist and compares with local data.
     '${TOOLS.JEPSON_EFLORA}' retrieves data from Jepson eFlora indexes and compares with local data.
     '${TOOLS.JEPSON_FAM}' retrieves section, family and genus data from Jepson eFlora and creates data files for use by ca-plant-list.