@ca-plant-list/ca-plant-list 0.4.11 → 0.4.12

package/lib/csv.js

@@ -109,17 +109,53 @@ class CSV {
         return parseSync(content, options);
     }
 
+    /**
+     * @param {string} fileName
+     * @param {string} [delimiter]
+     * @returns {{headers:string[],data:Object<string,any>[]}}
+     */
+    static readFileAndHeaders(fileName, delimiter) {
+        let headers;
+        /**
+         * @param {string[]} h
+         */
+        function getHeaders(h) {
+            headers = h;
+            return h;
+        }
+
+        const content = fs.readFileSync(fileName);
+        const options = this.#getOptions(fileName, getHeaders, delimiter);
+
+        const data = parseSync(content, options);
+        if (headers === undefined) {
+            throw new Error();
+        }
+        return { headers: headers, data: data };
+    }
+
     /**
      *
      * @param {string} fileName
      * @param {string[][]} data
      * @param {string[]} [headerData]
      */
-    static writeFile(fileName, data, headerData) {
+    static writeFileArray(fileName, data, headerData) {
         const header = headerData ? stringify([headerData]) : "";
         const content = header + stringify(data);
         fs.writeFileSync(fileName, content);
     }
+
+    /**
+     *
+     * @param {string} fileName
+     * @param {Object<string,any>[]} data
+     * @param {string[]} headerData
+     */
+    static writeFileObject(fileName, data, headerData) {
+        const content = stringify(data, { columns: headerData, header: true });
+        fs.writeFileSync(fileName, content.replaceAll(/,+\n/g, "\n"));
+    }
 }
 
 export { CSV };

package/lib/exceptions.js

@@ -18,7 +18,7 @@ class Exceptions {
 
         // Read default configuration.
         this.#exceptions = readConfig(
-            Config.getPackageDir() + "/data/exceptions.json"
+            Config.getPackageDir() + "/data/exceptions.json",
         );
 
         // Add/overwrite with local configuration.
@@ -38,7 +38,7 @@ class Exceptions {
      * @param {string} name
      * @param {string} cat
      * @param {string} subcat
-     * @param {string} defaultValue
+     * @param {string} [defaultValue]
      */
     getValue(name, cat, subcat, defaultValue) {
         const taxonData = this.#exceptions[name];

package/lib/htmltaxon.js

@@ -43,6 +43,23 @@ const DEFAULT_TAXA_COLUMNS = [
 ];
 
 class HTMLTaxon {
+    /**
+     * @param {Taxon} taxon
+     * @returns {string|undefined}
+     */
+    static getCalscapeLink(taxon) {
+        const calscapeCN = taxon.getCalscapeCommonName();
+        if (!calscapeCN) {
+            return;
+        }
+        return HTML.getLink(
+            `https://www.calscape.org/${taxon.getCalscapeName().replaceAll(" ", "-")}-()`,
+            "Calscape",
+            {},
+            true,
+        );
+    }
+
     /**
      * @param {string[]|undefined} colors
      */

package/lib/taxa.js

@@ -8,6 +8,7 @@ import { Taxon } from "./taxon.js";
 import { Families } from "./families.js";
 import { FlowerColor } from "./flowercolor.js";
 import { InatPhoto } from "./inat_photo.js";
+import { TaxaCSV } from "./tools/taxacsv.js";
 
 const FLOWER_COLORS = [
     { name: "white", color: "white" },
@@ -62,8 +63,13 @@ class Taxa {
 
         this.#families = new Families();
 
-        const taxaCSV = CSV.parseFile(dataDir, "taxa.csv");
-        this.#loadTaxa(taxaCSV, inclusionList, taxonFactory, showFlowerErrors);
+        const taxaCSV = new TaxaCSV(dataDir);
+        this.#loadTaxa(
+            taxaCSV.getTaxa(),
+            inclusionList,
+            taxonFactory,
+            showFlowerErrors,
+        );
         this.#loadTaxa(
             extraTaxa,
             inclusionList,

package/lib/taxon.js

@@ -22,6 +22,7 @@ class Taxon {
     #iNatID;
     /**@type {string|undefined} */
     #iNatSyn;
+    #calscapeCN;
     #lifeCycle;
     #flowerColors;
     #bloomStart;
@@ -58,6 +59,8 @@ class Taxon {
         this.#jepsonID = data["jepson id"];
         this.#calRecNum = data["calrecnum"];
         this.#iNatID = data["inat id"];
+        this.#calscapeCN =
+            data.calscape_cn === "" ? undefined : data.calscape_cn;
         this.#lifeCycle = data.life_cycle;
         const colors = data["flower_color"];
         this.#flowerColors = colors ? colors.split(",") : undefined;
@@ -98,7 +101,7 @@ class Taxon {
      * @param {InatPhoto} photo
      */
     addPhoto(photo) {
-        this.#photos = this.#photos.concat( [photo] );
+        this.#photos = this.#photos.concat([photo]);
     }
 
     getPhotos() {
@@ -144,11 +147,26 @@ class Taxon {
             "https://www.calflora.org/app/taxon?crn=" + calfloraID,
             "Calflora",
             {},
-            true
+            true,
         );
         return this.#cfSyn ? link + " (" + this.#cfSyn + ")" : link;
     }
 
+    getCalscapeCommonName() {
+        return this.#calscapeCN;
+    }
+
+    getCalscapeName() {
+        return Taxon.getCalscapeName(this.getName());
+    }
+
+    /**
+     * @param {string} name
+     */
+    static getCalscapeName(name) {
+        return name.replace(" subsp.", " ssp.");
+    }
+
     getCESA() {
         return this.#cesa;
     }
@@ -206,7 +224,7 @@ class Taxon {
         const link = HTML.wrap(
             "span",
             HTML.getLink(href, this.getName()),
-            attributes
+            attributes,
         );
         if (isRare) {
             return HTML.getToolTip(link, this.getRPIRankAndThreatTooltip(), {
@@ -238,7 +256,7 @@ class Taxon {
             "https://www.inaturalist.org/taxa/" + iNatID,
             "iNaturalist",
             {},
-            true
+            true,
         );
         return this.#iNatSyn ? link + " (" + this.#iNatSyn + ")" : link;
     }
@@ -272,7 +290,7 @@ class Taxon {
 
     getRPIRankAndThreatTooltip() {
         return RarePlants.getRPIRankAndThreatDescriptions(
-            this.getRPIRankAndThreat()
+            this.getRPIRankAndThreat(),
         ).join("<br>");
     }
 
@@ -285,7 +303,7 @@ class Taxon {
             "https://rareplants.cnps.org/Plants/Details/" + rpiID,
             "CNPS Rare Plant Inventory",
             {},
-            true
+            true,
         );
         return link;
     }

package/lib/tools/calflora.js

@@ -0,0 +1,220 @@
+import * as path from "path";
+import { CSV } from "../csv.js";
+import { Files } from "../files.js";
+
+const CALFLORA_URL_ALL =
+    "https://www.calflora.org/app/downtext?xun=117493&table=species&format=Tab&cols=0,1,4,5,8,38,41,43&psp=lifeform::grass,Tree,Herb,Fern,Shrub,Vine!!&par=f&active=";
+const CALFLORA_URL_COUNTY =
+    "https://www.calflora.org/app/downtext?xun=117493&table=species&format=Tab&cols=0,1,4,5,8,38,41,43&psp=countylist::ALA,CCA!!&active=1";
+
+/**
+ * @typedef {{
+ * "Native Status":string,
+ * TJMTID:string
+ * "Active in Calflora?":string
+ * Calrecnum:string
+ * }} CalfloraData
+ */
+
+class Calflora {
+    /** @type {Object<string,CalfloraData>} */
+    static #taxa = {};
+
+    /**
+     *
+     * @param {string} toolsDataDir
+     * @param {Taxa} taxa
+     * @param {import("../exceptions.js").Exceptions} exceptions
+     * @param {ErrorLog} errorLog
+     */
+    static async analyze(toolsDataDir, taxa, exceptions, errorLog) {
+        /**
+         * @param {string} url
+         * @param {string} targetFile
+         */
+        async function retrieveCalfloraFile(url, targetFile) {
+            // Retrieve file if it's not there.
+            targetFile = toolsDataPath + "/" + targetFile;
+            if (Files.exists(targetFile)) {
+                return;
+            }
+            console.log("retrieving " + targetFile);
+            await Files.fetch(url, targetFile);
+        }
+
+        const toolsDataPath = toolsDataDir + "/calflora";
+        // Create data directory if it's not there.
+        Files.mkdir(toolsDataPath);
+
+        const calfloraDataFileNameActive = "calflora_taxa_active.tsv";
+        const calfloraDataFileNameCounties = "calflora_taxa_counties.tsv";
+
+        await retrieveCalfloraFile(
+            CALFLORA_URL_ALL + "1",
+            calfloraDataFileNameActive,
+        );
+        // County list and "all" lists are both incomplete; load everything to get as much as possible.
+        await retrieveCalfloraFile(
+            CALFLORA_URL_COUNTY,
+            calfloraDataFileNameCounties,
+        );
+
+        const csvActive = CSV.readFile(
+            path.join(toolsDataPath, calfloraDataFileNameActive),
+        );
+        const csvCounties = CSV.readFile(
+            path.join(toolsDataPath, calfloraDataFileNameCounties),
+        );
+
+        for (const row of csvActive) {
+            this.#taxa[row["Taxon"]] = row;
+        }
+        for (const row of csvCounties) {
+            this.#taxa[row["Taxon"]] = row;
+        }
+
+        for (const taxon of taxa.getTaxonList()) {
+            const name = taxon.getName();
+            if (name.includes(" unknown")) {
+                continue;
+            }
+            const cfName = taxon.getCalfloraName();
+            const cfData = Calflora.#taxa[cfName];
+            if (!cfData) {
+                if (
+                    !exceptions.hasException(name, "calflora", "notintaxondata")
+                ) {
+                    errorLog.log(name, "not found in Calflora files");
+                }
+                continue;
+            }
+
+            // Check native status.
+            const cfNative = cfData["Native Status"];
+            let cfIsNative = cfNative === "rare" || cfNative === "native";
+            // Override if exception is specified.
+            const nativeException = exceptions.getValue(
+                name,
+                "calflora",
+                "native",
+                undefined,
+            );
+            if (typeof nativeException === "boolean") {
+                if (nativeException === cfIsNative) {
+                    errorLog.log(
+                        name,
+                        "has unnecessary Calflora native override",
+                    );
+                }
+                cfIsNative = nativeException;
+            }
+            if (cfIsNative !== taxon.isCANative()) {
+                errorLog.log(
+                    name,
+                    "has different nativity status in Calflora",
+                    cfIsNative.toString(),
+                );
+            }
+
+            // Check if it is active in Calflora.
+            const isActive = cfData["Active in Calflora?"];
+            if (isActive !== "YES") {
+                errorLog.log(name, "is not active in Calflora", isActive);
+            }
+
+            // Check Jepson IDs.
+            const cfJepsonID = cfData.TJMTID;
+            if (cfJepsonID !== taxon.getJepsonID()) {
+                if (
+                    !exceptions.hasException(name, "calflora", "badjepsonid") &&
+                    !exceptions.hasException(name, "calflora", "notintaxondata")
+                ) {
+                    errorLog.log(
+                        name,
+                        "Jepson ID in Calflora is different than taxa.csv",
+                        cfJepsonID,
+                        taxon.getJepsonID(),
+                    );
+                }
+            }
+
+            // Check Calflora ID.
+            const cfID = cfData["Calrecnum"];
+            if (cfID !== taxon.getCalfloraID()) {
+                errorLog.log(
+                    name,
+                    "Calflora ID in Calflora is different than taxa.csv",
+                    cfID,
+                    taxon.getCalfloraID(),
+                );
+            }
+        }
+
+        this.#checkExceptions(taxa, exceptions, errorLog);
+    }
+
+    /**
+     * @param {Taxa} taxa
+     * @param {import("../exceptions.js").Exceptions} exceptions
+     * @param {ErrorLog} errorLog
+     */
+    static #checkExceptions(taxa, exceptions, errorLog) {
+        // Check the Calflora exceptions and make sure they still apply.
+        for (const [name, v] of exceptions.getExceptions()) {
+            const exceptions = v.calflora;
+            if (!exceptions) {
+                continue;
+            }
+
+            // Make sure the taxon is still in our list.
+            const taxon = taxa.getTaxon(name);
+            if (!taxon) {
+                // Don't process global exceptions if taxon is not in local list.
+                if (taxa.isSubset() && !v.local) {
+                    continue;
+                }
+                errorLog.log(
+                    name,
+                    "has Calflora exceptions but not in Taxa collection",
+                );
+                continue;
+            }
+
+            for (const [k] of Object.entries(exceptions)) {
+                const cfData = Calflora.#taxa[name];
+                switch (k) {
+                    case "badjepsonid": {
+                        // Make sure Jepson ID is still wrong.
+                        const cfID = cfData.TJMTID;
+                        const jepsID = taxon.getJepsonID();
+                        if (cfID === jepsID) {
+                            errorLog.log(
+                                name,
+                                "has Calflora badjepsonid exception but IDs are the same",
+                            );
+                        }
+                        break;
+                    }
+                    case "native":
+                        break;
+                    case "notintaxondata":
+                        if (cfData) {
+                            errorLog.log(
+                                name,
+                                "found in Calflora data but has notintaxondata exception",
+                            );
+                        }
+                        break;
+                    default:
+                        errorLog.log(
+                            name,
+                            "unrecognized Calflora exception",
+                            k,
+                        );
+                }
+            }
+        }
+    }
+}
+
+export { Calflora };

package/lib/tools/calscape.js

@@ -0,0 +1,214 @@
+import path from "node:path";
+import xlsx from "exceljs";
+import { Files } from "../files.js";
+import { TaxaCSV } from "./taxacsv.js";
+import { Taxon } from "../taxon.js";
+
+export class Calscape {
+    /**
+     * @param {string} toolsDataDir
+     * @param {string} dataDir
+     * @param {Taxa} taxa
+     * @param {import("../exceptions.js").Exceptions} exceptions
+     * @param {ErrorLog} errorLog
+     * @param {boolean} update
+     */
+    static async analyze(
+        toolsDataDir,
+        dataDir,
+        taxa,
+        exceptions,
+        errorLog,
+        update,
+    ) {
+        const calscapeData = await getCalscapeData(toolsDataDir);
+
+        for (const taxon of taxa.getTaxonList()) {
+            const taxonName = taxon.getName();
+            const taxonCN = taxon.getCalscapeCommonName();
+            const calscapeCN = getCalscapeCommonName(
+                taxonName,
+                calscapeData,
+                exceptions,
+            );
+
+            if (taxonCN !== calscapeCN) {
+                errorLog.log(
+                    taxonName,
+                    "name in Calscape data is different than taxa.csv",
+                    calscapeCN ?? "undefined",
+                    taxonCN ?? "undefined",
+                );
+            }
+            // Calscape should only have natives, so make sure it's recorded as native.
+            if (calscapeCN && !taxon.isCANative()) {
+                errorLog.log(
+                    taxonName,
+                    "is in Calscape but not native in taxa.csv",
+                );
+            }
+        }
+
+        checkExceptions(taxa, exceptions, errorLog);
+
+        if (update) {
+            updateTaxaCSV(dataDir, calscapeData, exceptions);
+        }
+    }
+}
+
+/**
+ * @param {Taxa} taxa
+ * @param {import("../exceptions.js").Exceptions} exceptions
+ * @param {ErrorLog} errorLog
+ */
+function checkExceptions(taxa, exceptions, errorLog) {
+    // Check the Calscape exceptions and make sure they still apply.
+    for (const [name, v] of exceptions.getExceptions()) {
+        const exceptions = v.calscape;
+        if (!exceptions) {
+            continue;
+        }
+
+        // Make sure the taxon is still in our list.
+        const taxon = taxa.getTaxon(name);
+        if (!taxon) {
+            // Don't process global exceptions if taxon is not in local list.
+            if (taxa.isSubset() && !v.local) {
+                continue;
+            }
+            errorLog.log(
+                name,
+                "has Calscape exceptions but not in Taxa collection",
+            );
+            continue;
+        }
+
+        for (const [k] of Object.entries(exceptions)) {
+            switch (k) {
+                case "notnative": {
+                    if (taxon.isCANative()) {
+                        errorLog.log(
+                            name,
+                            "has Calscape notnative exception but is native in taxa.csv",
+                        );
+                    }
+                    break;
+                }
+                default:
+                    errorLog.log(name, "unrecognized Calscape exception", k);
+            }
+        }
+    }
+}
+
+/**
+ * @param {string} taxonName
+ * @param {Map<string,string>} calscapeData
+ * @param {import("../exceptions.js").Exceptions} exceptions
+ * @returns {string|undefined}
+ */
+function getCalscapeCommonName(taxonName, calscapeData, exceptions) {
+    const calscapeCN = calscapeData.get(Taxon.getCalscapeName(taxonName));
+    if (
+        calscapeCN &&
+        exceptions.hasException(taxonName, "calscape", "notnative")
+    ) {
+        return;
+    }
+    return calscapeCN;
+}
+
+/**
+ * @param {string} toolsDataDir
+ * @returns {Promise<Map<string,string>>}
+ */
+async function getCalscapeData(toolsDataDir) {
+    /**
+     * @param {import("exceljs").Cell} cell
+     */
+    function getCellValue(cell) {
+        const value = cell.value;
+        if (value === null || value === undefined) {
+            return undefined;
+        }
+        return value.toString();
+    }
+
+    toolsDataDir = path.join(toolsDataDir, "calscape");
+    Files.mkdir(toolsDataDir);
+    await retrieveCalscapeFile(toolsDataDir);
+
+    /** @type {Map<string,string>} */
+    const data = new Map();
+
+    const wb = new xlsx.Workbook();
+    await wb.xlsx.readFile(getExcelFilename(toolsDataDir)).then(function () {
+        const ws = wb.worksheets[0];
+        let isInData = false;
+        for (let index = 0; index < ws.rowCount; index++) {
+            const row = ws.getRow(index);
+            const col1 = getCellValue(row.getCell(1));
+            if (!isInData) {
+                if (col1 === "Botanical Name") {
+                    isInData = true;
+                }
+                continue;
+            }
+            const col2 = getCellValue(row.getCell(2));
+            if (!col1 || !col2) {
+                continue;
+            }
+            data.set(col1, col2);
+        }
+    });
+
+    return data;
+}
+
+/**
+ * @param {string} toolsDataDir
+ */
+function getExcelFilename(toolsDataDir) {
+    return path.join(toolsDataDir, "calscape.xlsx");
+}
+
+/**
+ * @param {string} toolsDataDir
+ */
+async function retrieveCalscapeFile(toolsDataDir) {
+    // Retrieve file if it's not there.
+    const targetFile = getExcelFilename(toolsDataDir);
+    if (Files.exists(targetFile)) {
+        return;
+    }
+    console.info("retrieving " + targetFile);
+    await Files.fetch("https://www.calscape.org/export/search/", targetFile);
+}
+
+/**
+ * @param {string} dataDir
+ * @param {Map<string,string>} calscapeData
+ * @param {import("../exceptions.js").Exceptions} exceptions
+ */
+function updateTaxaCSV(dataDir, calscapeData, exceptions) {
+    const taxa = new TaxaCSV(dataDir);
+
+    for (const taxonData of taxa.getTaxa()) {
+        const taxonCN = taxonData.calscape_cn;
+        const calscapeCN = getCalscapeCommonName(
+            taxonData.taxon_name,
+            calscapeData,
+            exceptions,
+        );
+        if (taxonCN !== calscapeCN) {
+            if (calscapeCN === undefined) {
+                delete taxonData.calscape_cn;
+            } else {
+                taxonData.calscape_cn = calscapeCN;
+            }
+        }
+    }
+
+    taxa.write();
+}

package/lib/tools/taxacsv.js

@@ -0,0 +1,31 @@
+import path from "path";
+import { CSV } from "../csv.js";
+
+export class TaxaCSV {
+    #filePath;
+    #headers;
+    /** @type {TaxonData[]} */
+    #taxa;
+
+    /**
+     * @param {string} dataDir
+     */
+    constructor(dataDir) {
+        this.#filePath = path.join(dataDir, "taxa.csv");
+        const csv = CSV.readFileAndHeaders(this.#filePath);
+        // @ts-ignore
+        this.#taxa = csv.data;
+        this.#headers = csv.headers;
+    }
+
+    /**
+     * @returns {TaxonData[]}
+     */
+    getTaxa() {
+        return this.#taxa;
+    }
+
+    write() {
+        CSV.writeFileObject(this.#filePath, this.#taxa, this.#headers);
+    }
+}

package/lib/web/pagetaxon.js

@@ -36,6 +36,10 @@ class PageTaxon extends GenericPage {
         if (iNatLink) {
             links.push(iNatLink);
         }
+        const calscapeLink = HTMLTaxon.getCalscapeLink(this.#taxon);
+        if (calscapeLink) {
+            links.push(calscapeLink);
+        }
         const rpiLink = this.#taxon.getRPITaxonLink();
         if (rpiLink) {
             links.push(rpiLink);
@@ -120,7 +124,7 @@ class PageTaxon extends GenericPage {
         }
 
         return HTML.wrap("div", "<ul>" + HTML.arrayToLI(ranks) + "</ul>", {
-            class: "section",
+            class: "section nobullet",
         });
     }