@ca-plant-list/ca-plant-list 0.4.21 → 0.4.23

package/lib/tools/fna.js

@@ -0,0 +1,163 @@
+import path from "node:path";
+import { Files } from "../files.js";
+import { scrape } from "@htmltools/scrape";
+import { TaxaCSV } from "./taxacsv.js";
+
+/**
+ * @typedef {{name:string}} FNATaxon
+ * @typedef {Map<string,FNATaxon>} FNATaxa
+ */
+
+export class FNA {
+    /**
+     * @param {string} toolsDataDir
+     * @param {string} dataDir
+     * @param {import("../types.js").Taxa} taxa
+     * @param {import("../errorlog.js").ErrorLog} errorLog
+     * @param {boolean} update
+     */
+    static async analyze(toolsDataDir, dataDir, taxa, errorLog, update) {
+        const toolsDataPath = path.join(toolsDataDir, "fna");
+        const fnaTaxa = await getFNATaxa(toolsDataPath);
+
+        const namesToUpdate = new Map();
+
+        for (const taxon of taxa.getTaxonList()) {
+            const fnaName = getFNAName(taxon, fnaTaxa) ?? "";
+            const taxonFNAName = taxon.getFNAName();
+            if (fnaName !== taxonFNAName) {
+                const name = taxon.getName();
+                errorLog.log(
+                    name,
+                    "FNA name does not match name from FNA data",
+                    taxonFNAName,
+                    fnaName,
+                );
+                namesToUpdate.set(name, fnaName === name ? "true" : fnaName);
+            }
+        }
+
+        if (update) {
+            updateNames(dataDir, namesToUpdate);
+        }
+    }
+}
+
+/**
+ * @param {import("../types.js").Taxon} taxon
+ * @param {FNATaxa} fnaTaxa
+ * @returns {string|undefined}
+ */
+function getFNAName(taxon, fnaTaxa) {
+    /**
+     * @param {string} input
+     * @returns {string|undefined}
+     */
+    function getName(input) {
+        if (fnaTaxa.has(input)) {
+            return input;
+        }
+
+        // See if we can swap var./subsp. to find it.
+        const parts = input.split(" ");
+        if (parts.length === 4) {
+            parts[2] = parts[2] === "var." ? "subsp." : "var.";
+            input = parts.join(" ");
+            if (fnaTaxa.has(input)) {
+                return input;
+            }
+        }
+
+        // If it's the nominate subsp/var, see if we can find the species.
+        if (parts.length === 4 && parts[1] === parts[3]) {
+            input = parts[0] + " " + parts[1];
+            if (fnaTaxa.has(input)) {
+                return input;
+            }
+        }
+    }
+    const name = getName(taxon.getName());
+    if (name !== undefined) {
+        return name;
+    }
+
+    // See if any synonyms match.
+    for (const synonym of taxon.getSynonyms()) {
+        const name = getName(synonym);
+        if (name !== undefined) {
+            return name;
+        }
+    }
+}
+
+/**
+ * @param {string} toolsDataPath
+ * @returns {Promise<FNATaxa>}
+ */
+async function getFNATaxa(toolsDataPath) {
+    /** @type {FNATaxa} */
+    const fnaTaxa = new Map();
+
+    Files.mkdir(toolsDataPath);
+
+    // Get list of volumes.
+    const volumePage = path.join(toolsDataPath, "volumes.html");
+    if (!Files.exists(volumePage)) {
+        await Files.fetch(
+            "http://floranorthamerica.org/Special:SearchByProperty/:Volume/",
+            volumePage,
+        );
+    }
+
+    const volDoc = scrape.parseFile(volumePage);
+    const links = scrape.getSubtrees(volDoc, (e) => {
+        const href = scrape.getAttr(e, "href");
+        return href !== undefined && href.startsWith("/Volume_");
+    });
+    const vols = links.map((e) => scrape.getTextContent(e));
+
+    // For each volume, retrieve the JSON.
+    const baseURL =
+        "http://floranorthamerica.org/Special:Ask/limit=5000/unescape=true/format=json";
+    for (const vol of vols) {
+        const fileName = path.join(toolsDataPath, `${vol}.json`);
+        if (!Files.exists(fileName)) {
+            const url = baseURL + `/-5B-5BVolume::${vol}-5D-5D`;
+            await Files.fetch(url, fileName);
+        }
+
+        const text = Files.read(fileName);
+        /** @type {{results:Object<string,{}>}} */
+        const json = JSON.parse(text);
+        const results = json.results;
+
+        // If there are more than 5000 results, this will need to be updated to retrieve chunks.
+        if (Object.entries(results).length >= 5000) {
+            throw new Error(`${vol} has more than 5000 results`);
+        }
+
+        for (const [k] of Object.entries(results)) {
+            fnaTaxa.set(k, { name: k });
+        }
+    }
+
+    return fnaTaxa;
+}
+
+/**
+ * @param {string} dataDir
+ * @param {Map<string,string>} namesToUpdate
+ */
+function updateNames(dataDir, namesToUpdate) {
+    const taxa = new TaxaCSV(dataDir);
+
+    for (const taxonData of taxa.getTaxa()) {
+        const newName = namesToUpdate.get(taxonData.taxon_name);
+        if (!newName) {
+            continue;
+        }
+        taxonData.fna = newName;
+    }
+
+    taxa.write();
+}

package/lib/tools/inat.js

@@ -21,7 +21,7 @@ export class INat {
     /**
      * @param {string} toolsDataDir
      * @param {string} dataDir
-     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../types.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
      * @param {import("../errorlog.js").ErrorLog} errorLog
      * @param {string} csvFileName
@@ -115,7 +115,7 @@ export class INat {
 
     /**
      *
-     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../types.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
      * @param {import("../errorlog.js").ErrorLog} errorLog
      */
@@ -168,7 +168,7 @@ export class INat {
 
     /**
      *
-     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../types.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
      * @param {import("../errorlog.js").ErrorLog} errorLog
      * @param {string} name

package/lib/tools/jepsoneflora.js

@@ -54,7 +54,7 @@ export class JepsonEFlora {
 
     /**
      * @param {string} toolsDataDir
-     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../taxonomy/taxa.js").Taxa} taxa
      * @param {import("../errorlog.js").ErrorLog} errorLog
      */
     constructor(toolsDataDir, taxa, errorLog) {
@@ -102,8 +102,25 @@ export class JepsonEFlora {
                 idsToUpdate.set(name, jepsInfo.id);
             }
 
-            const efStatus = this.#getStatusCode(jepsInfo);
+            let efStatus = this.#getStatusCode(jepsInfo);
             const taxonStatus = taxon.getStatus();
+            // Override if exception is specified.
+            const nativeException = exceptions.getValue(
+                name,
+                "jepson",
+                "native",
+            );
+            if (typeof nativeException === "boolean") {
+                const overrideStatus = nativeException ? "N" : "X";
+                if (overrideStatus === efStatus) {
+                    this.#errorLog.log(
+                        name,
+                        "has unnecessary Jepson native override",
+                    );
+                }
+                efStatus = overrideStatus;
+            }
+
             if (
                 efStatus !== taxonStatus &&
                 !(taxonStatus === "NC" && efStatus === "N")
@@ -154,6 +171,8 @@ export class JepsonEFlora {
             for (const [k] of Object.entries(exceptions)) {
                 const jepsonData = this.#nameInfo.get(name);
                 switch (k) {
+                    case "native":
+                        break;
                     case "notineflora":
                         // Make sure it is really not in eFlora.
                         if (jepsonData) {

package/lib/tools/rpi.js

@@ -12,7 +12,7 @@ class RPI {
 
     /**
      * @param {string} toolsDataDir
-     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../types.js").Taxa} taxa
      * @param {import("../config.js").Config} config
      * @param {import("../exceptions.js").Exceptions} exceptions
      * @param {import("../errorlog.js").ErrorLog} errorLog
@@ -80,8 +80,8 @@ class RPI {
             const rank = row["CRPR"];
             const rawCESA = row["CESA"];
             const rawFESA = row["FESA"];
-            const cesa = rawCESA === "None" ? undefined : rawCESA;
-            const fesa = rawFESA === "None" ? undefined : rawFESA;
+            const cesa = rawCESA === "None" ? "" : rawCESA;
+            const fesa = rawFESA === "None" ? "" : rawFESA;
             const cnddb = row["SRank"];
             const globalRank = row["GRank"];
             const taxon = taxa.getTaxon(name);
@@ -202,7 +202,7 @@ class RPI {
     }
 
     /**
-     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../types.js").Taxa} taxa
      * @param {import("../config.js").Config} config
      * @param {import("../exceptions.js").Exceptions} exceptions
      * @param {import("../errorlog.js").ErrorLog} errorLog
@@ -366,7 +366,7 @@ class RPI {
 
     /**
      * @param {string} toolsDataDir
-     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../types.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
      * @param {import("../errorlog.js").ErrorLog} errorLog
      */

package/lib/tools/supplementaltext.js

@@ -5,7 +5,7 @@ const VALID_EXTENSIONS = new Set(["md", "footer.md"]);
 export class SupplementalText {
     /**
      *
-     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../types.js").Taxa} taxa
      * @param {import("../errorlog.js").ErrorLog} errorLog
      */
     static analyze(taxa, errorLog) {

package/lib/tools/taxacsv.js

@@ -1,9 +1,30 @@
 import path from "path";
 import { CSV } from "../csv.js";
 
+const HEADERS = [
+    "taxon_name",
+    "common name",
+    "status",
+    "jepson id",
+    "calrecnum",
+    "inat id",
+    "cch2_id",
+    "fna",
+    "calscape_cn",
+    "life_cycle",
+    "flower_color",
+    "bloom_start",
+    "bloom_end",
+    "RPI ID",
+    "CRPR",
+    "CESA",
+    "FESA",
+    "SRank",
+    "GRank",
+];
+
 export class TaxaCSV {
     #filePath;
-    #headers;
     /** @type {import("../index.js").TaxonData[]} */
     #taxa;
 
@@ -13,9 +34,7 @@ export class TaxaCSV {
     constructor(dataDir) {
         this.#filePath = path.join(dataDir, "taxa.csv");
         const csv = CSV.readFileAndHeaders(this.#filePath);
-        // @ts-ignore
         this.#taxa = csv.data;
-        this.#headers = csv.headers;
     }
 
     /**
@@ -26,6 +45,6 @@ export class TaxaCSV {
     }
 
     write() {
-        CSV.writeFileObject(this.#filePath, this.#taxa, this.#headers);
+        CSV.writeFileObject(this.#filePath, this.#taxa, HEADERS);
     }
 }

package/lib/types.js

@@ -0,0 +1,10 @@
+/**
+ * Classes
+ * @typedef {import("./config.js").Config} Config
+ * @typedef {import("./taxonomy/families.js").Families} Families
+ * @typedef {import("./taxonomy/families.js").Family} Family
+ * @typedef {import("./taxonomy/taxa.js").Taxa} Taxa
+ * @typedef {import("./index.js").TaxaColDef<import("./types.js").Taxon>} TaxaColDef
+ * @typedef {import("./taxonomy/taxon.js").Taxon} Taxon
+ * @typedef {import("./taxonomy/taxonomy.js").Taxonomy} Taxonomy
+ */

package/lib/utils/inat-tools.js

@@ -41,7 +41,7 @@ import { chunk, sleep } from "../util.js";
 const ALLOWED_LICENSE_CODES = ["cc0", "cc-by", "cc-by-nc"];
 
 /**
- * @param {import("../taxon.js").Taxon[]} taxa
+ * @param {import("../types.js").Taxon[]} taxa
  * @return {Promise<InatApiTaxon[]>}
  */
 async function fetchInatTaxa(taxa) {
@@ -57,7 +57,7 @@ async function fetchInatTaxa(taxa) {
 }
 
 /**
- * @param {import("../taxon.js").Taxon[]} taxaToUpdate
+ * @param {import("../types.js").Taxon[]} taxaToUpdate
  * @returns {Promise<Map<string,InatPhotoInfo[]>>}
  */
 export async function getTaxonPhotos(taxaToUpdate) {

package/lib/web/pageFamily.js

@@ -0,0 +1,146 @@
+import { ExternalSites } from "../externalsites.js";
+import { GenericPage } from "../genericpage.js";
+import { HTML } from "../html.js";
+import { HTMLTaxon } from "../htmltaxon.js";
+import { Sections } from "../taxonomy/families.js";
+
+export class PageFamilyList extends GenericPage {
+    #families;
+
+    /**
+     * @param {string} outputDir
+     * @param {import("../types.js").Family[]} families
+     */
+    constructor(outputDir, families) {
+        super(outputDir, "Families", "list_families");
+        this.#families = families;
+    }
+
+    /**
+     * @param {import("../types.js").TaxaColDef[]} [taxaColumns]
+     */
+    render(taxaColumns) {
+        let html = this.getDefaultIntro();
+
+        const sections = Sections.getSections();
+        const sectionLinks = [];
+        for (const name of Object.keys(sections).sort()) {
+            const taxa = sections[name];
+
+            // Render the section page.
+            new PageSection(this.getOutputDir(), name, taxa).render(
+                taxaColumns,
+            );
+
+            // Render the link.
+            const href = "./" + name + ".html";
+            sectionLinks.push(
+                HTML.getLink(href, name) + " (" + taxa.length + ")",
+            );
+        }
+        html += HTML.wrap("ul", HTML.arrayToLI(sectionLinks), {
+            class: "listmenu",
+        });
+
+        html += "<table>";
+        html += "<thead>";
+        html += HTML.textElement("th", "Family");
+        html += HTML.textElement("th", "Number of Species", { class: "right" });
+        html += "</thead>";
+
+        html += "<tbody>";
+        for (const family of this.#families) {
+            const taxa = family.getTaxa();
+            if (!taxa) {
+                continue;
+            }
+            let cols = HTML.wrap(
+                "td",
+                HTML.getLink("./" + family.getFileName(), family.getName()),
+            );
+            cols += HTML.wrap("td", taxa.length, { class: "right" });
+            html += HTML.wrap("tr", cols);
+        }
+        html += "</tbody>";
+
+        html += "</table>";
+
+        this.writeFile(html);
+    }
+
+    /**
+     * @param {string} outputDir
+     * @param {import("../types.js").TaxaColDef[]} [taxaColumns]
+     */
+    renderPages(outputDir, taxaColumns) {
+        for (const family of this.#families) {
+            if (family.getTaxa()) {
+                new PageFamily(outputDir, family).render(taxaColumns);
+            }
+        }
+    }
+}
+
+class PageFamily extends GenericPage {
+    #family;
+
+    /**
+     * @param {string} outputDir
+     * @param {import("../types.js").Family} family
+     */
+    constructor(outputDir, family) {
+        super(outputDir, family.getName(), family.getBaseFileName());
+        this.#family = family;
+    }
+
+    /**
+     * @param {import("../types.js").TaxaColDef[]} [columns]
+     */
+    render(columns) {
+        let html = this.getDefaultIntro();
+
+        const jepsonLink = ExternalSites.getJepsonRefLink(this.#family);
+        if (jepsonLink) {
+            html += HTML.wrap(
+                "div",
+                HTML.getLink(jepsonLink, "Jepson eFlora", {}, true),
+                {
+                    class: "section",
+                },
+            );
+        }
+
+        const taxa = this.#family.getTaxa();
+        if (!taxa) {
+            throw new Error();
+        }
+        html += HTMLTaxon.getTaxaTable(taxa, columns);
+
+        this.writeFile(html);
+    }
+}
+
+class PageSection extends GenericPage {
+    #taxa;
+
+    /**
+     * @param {string} outputDir
+     * @param {string} name
+     * @param {import("../types.js").Taxon[]} taxa
+     */
+    constructor(outputDir, name, taxa) {
+        super(outputDir, name, name);
+        this.#taxa = taxa;
+    }
+
+    /**
+     * @param {import("../types.js").TaxaColDef[]} [columns]
+     */
+    render(columns) {
+        let html = this.getDefaultIntro();
+
+        html += HTMLTaxon.getTaxaTable(this.#taxa, columns);
+
+        this.writeFile(html);
+    }
+}

package/lib/web/pagetaxon.js

@@ -1,7 +1,5 @@
-import { Jepson } from "../jepson.js";
 import { RarePlants } from "../rareplants.js";
 import { GenericPage } from "../genericpage.js";
-import { ExternalSites } from "../externalsites.js";
 import { HTML } from "../html.js";
 import { HTMLTaxon } from "../htmltaxon.js";
 
@@ -12,7 +10,7 @@ export class PageTaxon extends GenericPage {
     /**
      * @param {string} outputDir
      * @param {import("../config.js").Config} config
-     * @param {import("../taxon.js").Taxon} taxon
+     * @param {import("../types.js").Taxon} taxon
      */
     constructor(outputDir, config, taxon) {
         super(outputDir, taxon.getName(), taxon.getBaseFileName());
@@ -21,72 +19,31 @@ export class PageTaxon extends GenericPage {
     }
 
     #getInfoLinks() {
+        /** @type {string[]} */
         const links = [];
-        const jepsonID = this.#taxon.getJepsonID();
-        if (jepsonID) {
-            links.push(Jepson.getEFloraLink(jepsonID));
-        }
-        const cfLink = this.#taxon.getCalfloraTaxonLink();
-        if (cfLink) {
-            links.push(cfLink);
-        }
-        const iNatLink = this.#taxon.getINatTaxonLink();
-        if (iNatLink) {
-            links.push(iNatLink);
-        }
-        const calscapeLink = HTMLTaxon.getCalscapeLink(this.#taxon);
-        if (calscapeLink) {
-            links.push(calscapeLink);
-        }
-        const rpiLink = this.#taxon.getRPITaxonLink();
-        if (rpiLink) {
-            links.push(rpiLink);
-        }
-        HTMLTaxon.addLink(
+        HTMLTaxon.addRefLink(links, this.#taxon, "jepson");
+        HTMLTaxon.addRefLink(links, this.#taxon, "calflora");
+        HTMLTaxon.addRefLink(
             links,
-            ExternalSites.getCCH2RefLink(this.#taxon),
-            "CCH2",
+            this.#taxon,
+            "inat",
+            this.#taxon.getINatSyn()
+                ? " (" + this.#taxon.getINatSyn() + ")"
+                : "",
         );
+        HTMLTaxon.addRefLink(links, this.#taxon, "rpi");
+        HTMLTaxon.addRefLink(links, this.#taxon, "fna");
+        HTMLTaxon.addRefLink(links, this.#taxon, "cch");
+        HTMLTaxon.addRefLink(links, this.#taxon, "calscape");
         return links;
     }
 
     #getObsLinks() {
+        /** @type {string[]} */
         const links = [];
-        links.push(
-            HTML.getLink(
-                "https://www.calflora.org/entry/observ.html?track=m#srch=t&grezc=5&cols=b&lpcli=t&cc=" +
-                    this.#config.getCountyCodes().join("!") +
-                    "&incobs=f&taxon=" +
-                    this.#taxon.getCalfloraName().replaceAll(" ", "+"),
-                "Calflora",
-                {},
-                true,
-            ),
-        );
-        const iNatID = this.#taxon.getINatID();
-        if (iNatID) {
-            links.push(
-                HTML.getLink(
-                    ExternalSites.getInatObsLink({
-                        project_id: this.#config.getConfigValue(
-                            "inat",
-                            "project_id",
-                        ),
-                        subview: "map",
-                        taxon_id: iNatID,
-                    }),
-                    "iNaturalist",
-                    {},
-                    true,
-                ),
-            );
-        }
-        HTMLTaxon.addLink(
-            links,
-            ExternalSites.getCCH2ObsLink(this.#taxon, this.#config),
-            "CCH2",
-        );
-
+        HTMLTaxon.addObsLink(links, this.#taxon, this.#config, "inat");
+        HTMLTaxon.addObsLink(links, this.#taxon, this.#config, "calflora");
+        HTMLTaxon.addObsLink(links, this.#taxon, this.#config, "cch");
         return links;
     }
 
@@ -103,7 +60,7 @@ export class PageTaxon extends GenericPage {
             }) +
                 HTML.getToolTip(
                     cnpsRank,
-                    this.#taxon.getRPIRankAndThreatTooltip(),
+                    HTMLTaxon.getRPIRankAndThreatTooltip(this.#taxon),
                 ),
         );
         if (this.#taxon.getCESA()) {
@@ -126,7 +83,8 @@ export class PageTaxon extends GenericPage {
             if (taxa.length > 1) {
                 for (const taxon of taxa) {
                     links.push(
-                        taxon.getHTMLLink(
+                        HTMLTaxon.getHTMLLink(
+                            taxon,
                             taxon.getName() !== this.#taxon.getName(),
                         ),
                     );

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@ca-plant-list/ca-plant-list",
-    "version": "0.4.21",
+    "version": "0.4.23",
     "description": "Tools to create Jekyll files for a website listing plants in an area of California.",
     "license": "MIT",
     "repository": {
@@ -58,6 +58,7 @@
         "eslint": "^9.18.0",
         "jest": "^29.7.0",
         "prettier": "^3.4.2",
+        "puppeteer": "^24.1.1",
         "typescript": "^5.7.3"
     }
 }

package/scripts/build-ebook.js

@@ -5,7 +5,7 @@ import { ErrorLog } from "../lib/errorlog.js";
 import { Files } from "../lib/files.js";
 import { PlantBook } from "../lib/ebook/plantbook.js";
 import { Program } from "../lib/program.js";
-import { Taxa } from "../lib/taxa.js";
+import { Taxa } from "../lib/taxonomy/taxa.js";
 
 const program = Program.getProgram();
 program
@@ -33,7 +33,7 @@ async function build(options) {
                 await buildBook(
                     outputBase + suffix,
                     path,
-                    options.showFlowerErrors
+                    options.showFlowerErrors,
                 );
             }
         }
@@ -42,7 +42,7 @@ async function build(options) {
         await buildBook(
             options.outputdir,
             options.datadir,
-            options.showFlowerErrors
+            options.showFlowerErrors,
         );
     }
 }
@@ -58,7 +58,7 @@ async function buildBook(outputDir, dataDir, showFlowerErrors) {
     const taxa = new Taxa(
         Program.getIncludeList(dataDir),
         errorLog,
-        showFlowerErrors
+        showFlowerErrors,
     );
 
     const config = new Config(dataDir);