@ca-plant-list/ca-plant-list 0.4.19 → 0.4.21

package/lib/taxa.js

@@ -12,6 +12,10 @@ import { ErrorLog } from "./errorlog.js";
 import { Program } from "./program.js";
 import { Photo } from "./photo.js";
 
+/**
+ * @typedef {{Current: string;Former: string;Type: string;}} SynonymData
+ */
+
 const FLOWER_COLORS = [
     { name: "white", color: "white" },
     { name: "red", color: "red" },
@@ -36,11 +40,11 @@ class Taxa {
     #isSubset;
 
     /**
-     * @param {Object<string,TaxonData>|true} inclusionList
+     * @param {Object<string,import("./index.js").TaxonData>|true} inclusionList
      * @param {ErrorLog} errorLog
      * @param {boolean} showFlowerErrors
-     * @param {function(TaxonData,Genera):Taxon} taxonFactory
-     * @param {TaxonData[]} [extraTaxa=[]]
+     * @param {function(import("./index.js").TaxonData,Genera):Taxon} taxonFactory
+     * @param {import("./index.js").TaxonData[]} [extraTaxa=[]]
      * @param {SynonymData[]} [extraSynonyms=[]]
      * @param {boolean} includePhotos
      */
@@ -110,9 +114,9 @@ class Taxa {
      */
     #loadPhotosFromFile(dataDir, filename) {
         if (!fs.existsSync(path.join(dataDir, filename))) return;
-        /** @type {InatCsvPhoto[]} */
+        /** @type {import("./utils/inat-tools.js").InatCsvPhoto[]} */
         const csvPhotos = CSV.parseFile(dataDir, filename).map((row) => {
-            /** @type {InatLicenseCode} */
+            /** @type {import("./utils/inat-tools.js").InatLicenseCode} */
             let licenseCode = "cc-by";
             if (row.licenseCode === "cc-by-nc-sa") licenseCode = "cc-by-nc-sa";
             else if (row.licenseCode === "cc-by-nc") licenseCode = "cc-by-nc";
@@ -220,7 +224,7 @@ class Taxa {
 
     /**
      * @param {SynonymData[]} synCSV
-     * @param {Object<string,TaxonData>|boolean} inclusionList
+     * @param {Object<string,import("./index.js").TaxonData>|boolean} inclusionList
      */
     #loadSyns(synCSV, inclusionList) {
         for (const syn of synCSV) {
@@ -241,9 +245,9 @@ class Taxa {
     }
 
     /**
-     * @param {TaxonData[]} taxaCSV
-     * @param {Object<string,TaxonData>|true} inclusionList
-     * @param {function(TaxonData,Genera):Taxon} taxonFactory
+     * @param {import("./index.js").TaxonData[]} taxaCSV
+     * @param {Object<string,import("./index.js").TaxonData>|true} inclusionList
+     * @param {function(import("./index.js").TaxonData,Genera):Taxon} taxonFactory
      * @param {Genera} genera
      * @param {boolean} showFlowerErrors
      */
@@ -251,7 +255,7 @@ class Taxa {
         for (const row of taxaCSV) {
             const name = row["taxon_name"];
 
-            /** @type {TaxonData|{status?:StatusCode}} */
+            /** @type {import("./index.js").TaxonData|{status?:import("./index.js").NativeStatusCode}} */
             let taxon_overrides = {};
             if (inclusionList !== true) {
                 taxon_overrides = inclusionList[name];

package/lib/taxon.js

@@ -1,15 +1,7 @@
 import { HTML } from "./html.js";
 import { RarePlants } from "./rareplants.js";
 
-const TAXA_COLNAMES = {
-    BLOOM_START: "bloom_start",
-    BLOOM_END: "bloom_end",
-    COMMON_NAME: "common name",
-    FLOWER_COLOR: "flower_color",
-};
-
 class Taxon {
-    /** @type {Genera} */
     #genera;
     #name;
     #genus;
@@ -22,6 +14,7 @@ class Taxon {
     #iNatID;
     /**@type {string|undefined} */
     #iNatSyn;
+    #cch2id;
     #calscapeCN;
     #lifeCycle;
     #flowerColors;
@@ -39,7 +32,7 @@ class Taxon {
     #photos = [];
 
     /**
-     * @param {TaxonData} data
+     * @param {import("./index.js").TaxonData} data
      * @param {import("./genera.js").Genera} genera
      */
     constructor(data, genera) {
@@ -59,6 +52,7 @@ class Taxon {
         this.#jepsonID = data["jepson id"];
         this.#calRecNum = data["calrecnum"];
         this.#iNatID = data["inat id"];
+        this.#cch2id = data.cch2_id;
         this.#calscapeCN =
             data.calscape_cn === "" ? undefined : data.calscape_cn;
         this.#lifeCycle = data.life_cycle;
@@ -170,6 +164,13 @@ class Taxon {
         return name.replace(" subsp.", " ssp.");
     }
 
+    /**
+     * @returns {string}
+     */
+    getCCH2ID() {
+        return this.#cch2id;
+    }
+
     getCESA() {
         return this.#cesa;
     }
@@ -300,12 +301,18 @@ class Taxon {
         return this.#rankRPI;
     }
 
+    /**
+     * @deprecated
+     */
     getRPIRankAndThreatTooltip() {
         return RarePlants.getRPIRankAndThreatDescriptions(
             this.getRPIRankAndThreat(),
         ).join("<br>");
     }
 
+    /**
+     * @deprecated
+     */
     getRPITaxonLink() {
         const rpiID = this.getRPIID();
         if (!rpiID) {
@@ -320,15 +327,12 @@ class Taxon {
         return link;
     }
 
-    /**
-     * @returns {StatusCode}
-     */
     getStatus() {
         return this.#status;
     }
 
     /**
-     * @param {Config} config
+     * @param {import("./config.js").Config} config
      * @returns {string}
      */
     getStatusDescription(config) {
@@ -382,4 +386,4 @@ class Taxon {
     }
 }
 
-export { TAXA_COLNAMES, Taxon };
+export { Taxon };

package/lib/tools/calflora.js

@@ -1,6 +1,7 @@
 import * as path from "path";
 import { CSV } from "../csv.js";
 import { Files } from "../files.js";
+import { TaxaCSV } from "./taxacsv.js";
 
 const CALFLORA_URL_ALL =
     "https://www.calflora.org/app/downtext?xun=117493&table=species&format=Tab&cols=0,1,4,5,8,38,41,43&psp=lifeform::grass,Tree,Herb,Fern,Shrub,Vine!!&par=f&active=";
@@ -17,18 +18,26 @@ const CALFLORA_URL_COUNTY =
  * }} CalfloraData
  */
 
-class Calflora {
+export class Calflora {
     /** @type {Object<string,CalfloraData>} */
     static #taxa = {};
 
     /**
-     *
      * @param {string} toolsDataDir
-     * @param {Taxa} taxa
+     * @param {string} dataDir
+     * @param {import("../taxa.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
-     * @param {ErrorLog} errorLog
+     * @param {import("../errorlog.js").ErrorLog} errorLog
+     * @param {boolean} update
      */
-    static async analyze(toolsDataDir, taxa, exceptions, errorLog) {
+    static async analyze(
+        toolsDataDir,
+        dataDir,
+        taxa,
+        exceptions,
+        errorLog,
+        update,
+    ) {
         /**
          * @param {string} url
          * @param {string} targetFile
@@ -78,6 +87,8 @@ class Calflora {
             this.#taxa[row["Taxon"]] = row;
         }
 
+        const idsToUpdate = new Map();
+
         for (const taxon of taxa.getTaxonList()) {
             const name = taxon.getName();
             if (name.includes(" unknown")) {
@@ -152,16 +163,21 @@ class Calflora {
                     cfID,
                     taxon.getCalfloraID(),
                 );
+                idsToUpdate.set(name, cfID);
             }
         }
 
         this.#checkExceptions(taxa, exceptions, errorLog);
+
+        if (update) {
+            this.#updateIds(dataDir, idsToUpdate);
+        }
     }
 
     /**
-     * @param {Taxa} taxa
+     * @param {import("../taxa.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
-     * @param {ErrorLog} errorLog
+     * @param {import("../errorlog.js").ErrorLog} errorLog
      */
     static #checkExceptions(taxa, exceptions, errorLog) {
         // Check the Calflora exceptions and make sure they still apply.
@@ -220,6 +236,22 @@ class Calflora {
             }
         }
     }
-}
 
-export { Calflora };
+    /**
+     * @param {string} dataDir
+     * @param {Map<string,string>} idsToUpdate
+     */
+    static #updateIds(dataDir, idsToUpdate) {
+        const taxa = new TaxaCSV(dataDir);
+
+        for (const taxonData of taxa.getTaxa()) {
+            const id = idsToUpdate.get(taxonData.taxon_name);
+            if (!id) {
+                continue;
+            }
+            taxonData["calrecnum"] = id;
+        }
+
+        taxa.write();
+    }
+}

package/lib/tools/calscape.js

@@ -8,9 +8,9 @@ export class Calscape {
     /**
      * @param {string} toolsDataDir
      * @param {string} dataDir
-     * @param {Taxa} taxa
+     * @param {import("../taxa.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
-     * @param {ErrorLog} errorLog
+     * @param {import("../errorlog.js").ErrorLog} errorLog
      * @param {boolean} update
      */
     static async analyze(
@@ -58,9 +58,9 @@ export class Calscape {
 }
 
 /**
- * @param {Taxa} taxa
+ * @param {import("../taxa.js").Taxa} taxa
  * @param {import("../exceptions.js").Exceptions} exceptions
- * @param {ErrorLog} errorLog
+ * @param {import("../errorlog.js").ErrorLog} errorLog
  */
 function checkExceptions(taxa, exceptions, errorLog) {
     // Check the Calscape exceptions and make sure they still apply.

package/lib/tools/cch2.js

@@ -0,0 +1,95 @@
+import path from "node:path";
+import { CSV } from "../csv.js";
+import { TaxaCSV } from "./taxacsv.js";
+
+/**
+ * @typedef {{id:string}} CCHTaxon
+ * @typedef {Map<string,CCHTaxon>} CCHTaxa
+ */
+
+export class CCH2 {
+    /**
+     * @param {string} toolsDataDir
+     * @param {string} dataDir
+     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../errorlog.js").ErrorLog} errorLog
+     * @param {boolean} update
+     */
+    static async analyze(toolsDataDir, dataDir, taxa, errorLog, update) {
+        const toolsDataPath = path.join(toolsDataDir, "cch2");
+
+        const cchTaxa = await getCCHTaxa(toolsDataPath, taxa);
+
+        const idsToUpdate = new Map();
+        for (const taxon of taxa.getTaxonList()) {
+            const cchTaxon = cchTaxa.get(taxon.getName());
+            if (!cchTaxon) {
+                errorLog.log(taxon.getName(), "not found in CCH data");
+                continue;
+            }
+            if (cchTaxon.id !== taxon.getCCH2ID()) {
+                errorLog.log(
+                    taxon.getName(),
+                    "id in CCH data does not match id in taxa.csv",
+                    cchTaxon.id,
+                    taxon.getCCH2ID(),
+                );
+                idsToUpdate.set(taxon.getName(), cchTaxon.id);
+            }
+        }
+
+        if (update) {
+            updateTaxaCSV(dataDir, idsToUpdate);
+        }
+    }
+}
+
+/**
+ * @param {string} toolsDataPath
+ * @param {import("../taxa.js").Taxa} taxa
+ * @returns {Promise<CCHTaxa>}
+ */
+async function getCCHTaxa(toolsDataPath, taxa) {
+    /**
+     * @param {{taxonID:string,scientificName:string,rankID:string,acceptance:"0"|"1",acceptedTaxonID:string}} record
+     */
+    function callback(record) {
+        if (parseInt(record.rankID) < 220) {
+            // Ignore ranks above species.
+            return;
+        }
+        if (record.acceptance !== "1") {
+            return;
+        }
+        if (!taxa.getTaxon(record.scientificName)) {
+            // If we're not tracking the taxon, ignore it.
+            return;
+        }
+        data.set(record.scientificName, { id: record.acceptedTaxonID });
+    }
+
+    const fileName = path.join(toolsDataPath, "taxa.csv");
+    const data = new Map();
+
+    await CSV.parseFileStream(fileName, callback);
+
+    return data;
+}
+
+/**
+ * @param {string} dataDir
+ * @param {Map<string,string>} idsToUpdate
+ */
+function updateTaxaCSV(dataDir, idsToUpdate) {
+    const taxa = new TaxaCSV(dataDir);
+
+    for (const taxonData of taxa.getTaxa()) {
+        const id = idsToUpdate.get(taxonData.taxon_name);
+        if (!id) {
+            continue;
+        }
+        taxonData.cch2_id = id;
+    }
+
+    taxa.write();
+}

package/lib/tools/inat.js

@@ -21,9 +21,9 @@ export class INat {
     /**
      * @param {string} toolsDataDir
      * @param {string} dataDir
-     * @param {Taxa} taxa
+     * @param {import("../taxa.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
-     * @param {ErrorLog} errorLog
+     * @param {import("../errorlog.js").ErrorLog} errorLog
      * @param {string} csvFileName
      * @param {boolean} update
      */
@@ -115,9 +115,9 @@ export class INat {
 
     /**
      *
-     * @param {Taxa} taxa
+     * @param {import("../taxa.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
-     * @param {ErrorLog} errorLog
+     * @param {import("../errorlog.js").ErrorLog} errorLog
      */
     static #checkExceptions(taxa, exceptions, errorLog) {
         // Check the iNat exceptions and make sure they still apply.
@@ -168,9 +168,9 @@ export class INat {
 
     /**
      *
-     * @param {Taxa} taxa
+     * @param {import("../taxa.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
-     * @param {ErrorLog} errorLog
+     * @param {import("../errorlog.js").ErrorLog} errorLog
      * @param {string} name
      * @param {string} iNatName
      */
@@ -255,7 +255,7 @@ export class INat {
 
     /**
      * @param {{name:string,rank:string}} iNatResult
-     * @param {ErrorLog} errorLog
+     * @param {import("../errorlog.js").ErrorLog} errorLog
      */
     static makeSynonymName(iNatResult, errorLog) {
         const synParts = iNatResult.name.split(" ");

package/lib/tools/jepsoneflora.js

@@ -1,6 +1,7 @@
 import { scrape } from "@htmltools/scrape";
 import { Files } from "../files.js";
 import { SynCSV } from "./syncsv.js";
+import { TaxaCSV } from "./taxacsv.js";
 
 /**
  * @typedef {{
@@ -53,8 +54,8 @@ export class JepsonEFlora {
 
     /**
      * @param {string} toolsDataDir
-     * @param {Taxa} taxa
-     * @param {ErrorLog} errorLog
+     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../errorlog.js").ErrorLog} errorLog
      */
     constructor(toolsDataDir, taxa, errorLog) {
         this.#toolsDataPath = toolsDataDir + "/jepson-eflora";
@@ -63,16 +64,19 @@ export class JepsonEFlora {
     }
 
     /**
+     * @param {string} dataDir
      * @param {import("../exceptions.js").Exceptions} exceptions
      * @param {boolean} update
      */
-    async analyze(exceptions, update) {
+    async analyze(dataDir, exceptions, update) {
         // Create data directory if it's not there.
         Files.mkdir(this.#toolsDataPath);
 
         // Retrieve all Jepson indexes.
         await this.#loadIndexPages();
 
+        const idsToUpdate = new Map();
+
         for (const taxon of this.#taxa.getTaxonList()) {
             const name = taxon.getName();
             if (name.includes(" unknown")) {
@@ -95,6 +99,7 @@ export class JepsonEFlora {
                     taxon.getJepsonID(),
                     jepsInfo.id,
                 );
+                idsToUpdate.set(name, jepsInfo.id);
             }
 
             const efStatus = this.#getStatusCode(jepsInfo);
@@ -116,6 +121,7 @@ export class JepsonEFlora {
         this.#checkExceptions(exceptions);
 
         if (update) {
+            this.#updateIds(dataDir, idsToUpdate);
             this.#updateSynCSV();
         }
     }
@@ -222,7 +228,7 @@ export class JepsonEFlora {
 
     /**
      * @param {JepsonTaxon} jepsInfo
-     * @returns {StatusCode|undefined}
+     * @returns {import("../index.js").NativeStatusCode|undefined}
      */
     #getStatusCode(jepsInfo) {
         switch (jepsInfo.type) {
@@ -396,6 +402,24 @@ export class JepsonEFlora {
         }
     }
 
+    /**
+     * @param {string} dataDir
+     * @param {Map<string,string>} idsToUpdate
+     */
+    #updateIds(dataDir, idsToUpdate) {
+        const taxa = new TaxaCSV(dataDir);
+
+        for (const taxonData of taxa.getTaxa()) {
+            const id = idsToUpdate.get(taxonData.taxon_name);
+            if (!id) {
+                continue;
+            }
+            taxonData["jepson id"] = id;
+        }
+
+        taxa.write();
+    }
+
     #updateSynCSV() {
         const csv = new SynCSV("./data");
         const data = csv.getData();

package/lib/tools/jepsonfamilies.js

@@ -0,0 +1,102 @@
+import path from "node:path";
+import { Files } from "../files.js";
+import { scrape } from "@htmltools/scrape";
+
+export class JepsonFamilies {
+    /**
+     * @param {string} toolsDataDir
+     * @param {string} outputdir
+     */
+    static async build(toolsDataDir, outputdir) {
+        const url = "https://ucjeps.berkeley.edu/eflora/toc.html";
+        const indexFileName = path.basename(url);
+        const toolsDataPath = toolsDataDir + "/jepsonfam";
+        const indexFilePath = toolsDataPath + "/" + indexFileName;
+
+        // Create data directory if it's not there.
+        Files.mkdir(toolsDataPath);
+
+        // Download the data file if it doesn't exist.
+        if (!Files.exists(indexFilePath)) {
+            console.log("retrieving Jepson family index");
+            await Files.fetch(url, indexFilePath);
+        }
+
+        const document = scrape.parseFile(indexFilePath);
+
+        const body = scrape.getSubtree(document, (t) => t.tagName === "body");
+        if (!body) {
+            throw new Error();
+        }
+        const contentDiv = scrape.getSubtree(
+            body,
+            (t) => scrape.getAttr(t, "id") === "content",
+        );
+        if (!contentDiv) {
+            throw new Error();
+        }
+        const rows = scrape.getSubtrees(contentDiv, (t) => t.tagName === "tr");
+
+        this.#parseRows(outputdir, rows);
+    }
+
+    /**
+     * @param {string} toolsDataPath
+     * @param {import("@htmltools/scrape").Element[]} rows
+     */
+    static #parseRows(toolsDataPath, rows) {
+        /** @type {Object<string,{section:string,id:string}>} */
+        const families = {};
+        /** @type {Object<string,{family:string,id:string}>} */
+        const genera = {};
+
+        for (const row of rows) {
+            const cols = scrape.getSubtrees(row, (t) => t.tagName === "td");
+            if (!cols || cols.length < 3) {
+                continue;
+            }
+
+            // Find the section.
+            const section = scrape.getTextContent(cols[0].children[0]);
+
+            // Find the family name and ID.
+            const familyLink = cols[1].children[0];
+            if (familyLink.type !== "element") {
+                throw new Error();
+            }
+            const familyTarget = scrape.getAttr(familyLink, "href");
+            if (!familyTarget) {
+                throw new Error();
+            }
+            const familyID = familyTarget.split("=")[1];
+            const familyName = scrape.getTextContent(familyLink.children[0]);
+            families[familyName] = { section: section, id: familyID };
+
+            // Find all the genera.
+            const genusLinks = scrape.getSubtrees(
+                cols[2],
+                (t) => t.tagName === "a",
+            );
+            for (const genusLink of genusLinks) {
+                const genusTarget = scrape.getAttr(genusLink, "href");
+                if (!genusTarget) {
+                    throw new Error();
+                }
+                const genusID = genusTarget.split("=")[1];
+                const genusName = scrape.getTextContent(genusLink.children[0]);
+                genera[genusName] = { family: familyName, id: genusID };
+            }
+        }
+
+        Files.write(
+            toolsDataPath + "/families.json",
+            JSON.stringify(families, undefined, 4),
+            true,
+        );
+        Files.write(
+            toolsDataPath + "/genera.json",
+            JSON.stringify(genera, undefined, 4),
+            true,
+        );
+    }
+}

package/lib/tools/rpi.js

@@ -13,9 +13,9 @@ class RPI {
     /**
      * @param {string} toolsDataDir
      * @param {import("../taxa.js").Taxa} taxa
-     * @param {Config} config
+     * @param {import("../config.js").Config} config
      * @param {import("../exceptions.js").Exceptions} exceptions
-     * @param {ErrorLog} errorLog
+     * @param {import("../errorlog.js").ErrorLog} errorLog
      */
     static async analyze(toolsDataDir, taxa, config, exceptions, errorLog) {
         /**
@@ -203,9 +203,9 @@ class RPI {
 
     /**
      * @param {import("../taxa.js").Taxa} taxa
-     * @param {Config} config
+     * @param {import("../config.js").Config} config
      * @param {import("../exceptions.js").Exceptions} exceptions
-     * @param {ErrorLog} errorLog
+     * @param {import("../errorlog.js").ErrorLog} errorLog
      */
     static #checkExceptions(taxa, config, exceptions, errorLog) {
         const countyCodes = config.getCountyCodes();
@@ -368,7 +368,7 @@ class RPI {
      * @param {string} toolsDataDir
      * @param {import("../taxa.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
-     * @param {ErrorLog} errorLog
+     * @param {import("../errorlog.js").ErrorLog} errorLog
      */
     static async #scrape(toolsDataDir, taxa, exceptions, errorLog) {
         const toolsDataPath = toolsDataDir + "/rpi";

package/lib/tools/supplementaltext.js

@@ -0,0 +1,43 @@
+import { Files } from "../files.js";
+
+const VALID_EXTENSIONS = new Set(["md", "footer.md"]);
+
+export class SupplementalText {
+    /**
+     *
+     * @param {import("../taxa.js").Taxa} taxa
+     * @param {import("../errorlog.js").ErrorLog} errorLog
+     */
+    static analyze(taxa, errorLog) {
+        /**
+         * @param {string} fileName
+         */
+        function fileNameToTaxonName(fileName) {
+            const parts = fileName.split(".");
+            const ext = parts.slice(1).join(".");
+            const taxonName = parts[0]
+                .replace("-", " ")
+                .replace("-var-", " var. ")
+                .replace("-subsp-", " subsp. ");
+            return { taxonName: taxonName, ext: ext };
+        }
+
+        const dirName = "data/text";
+
+        if (!Files.isDir(dirName)) {
+            return;
+        }
+
+        const entries = Files.getDirEntries(dirName);
+        for (const entry of entries) {
+            const parsed = fileNameToTaxonName(entry);
+            const taxon = taxa.getTaxon(parsed.taxonName);
+            if (!taxon) {
+                errorLog.log(dirName + "/" + entry, "not found in taxa.csv");
+            }
+            if (!VALID_EXTENSIONS.has(parsed.ext)) {
+                errorLog.log(dirName + "/" + entry, "has invalid extension");
+            }
+        }
+    }
+}