@ca-plant-list/ca-plant-list 0.4.10 → 0.4.12

package/lib/tools/calscape.js

@@ -0,0 +1,214 @@
+import path from "node:path";
+import xlsx from "exceljs";
+import { Files } from "../files.js";
+import { TaxaCSV } from "./taxacsv.js";
+import { Taxon } from "../taxon.js";
+
+export class Calscape {
+    /**
+     * @param {string} toolsDataDir
+     * @param {string} dataDir
+     * @param {Taxa} taxa
+     * @param {import("../exceptions.js").Exceptions} exceptions
+     * @param {ErrorLog} errorLog
+     * @param {boolean} update
+     */
+    static async analyze(
+        toolsDataDir,
+        dataDir,
+        taxa,
+        exceptions,
+        errorLog,
+        update,
+    ) {
+        const calscapeData = await getCalscapeData(toolsDataDir);
+
+        for (const taxon of taxa.getTaxonList()) {
+            const taxonName = taxon.getName();
+            const taxonCN = taxon.getCalscapeCommonName();
+            const calscapeCN = getCalscapeCommonName(
+                taxonName,
+                calscapeData,
+                exceptions,
+            );
+
+            if (taxonCN !== calscapeCN) {
+                errorLog.log(
+                    taxonName,
+                    "name in Calscape data is different than taxa.csv",
+                    calscapeCN ?? "undefined",
+                    taxonCN ?? "undefined",
+                );
+            }
+            // Calscape should only have natives, so make sure it's recorded as native.
+            if (calscapeCN && !taxon.isCANative()) {
+                errorLog.log(
+                    taxonName,
+                    "is in Calscape but not native in taxa.csv",
+                );
+            }
+        }
+
+        checkExceptions(taxa, exceptions, errorLog);
+
+        if (update) {
+            updateTaxaCSV(dataDir, calscapeData, exceptions);
+        }
+    }
+}
+
+/**
+ * @param {Taxa} taxa
+ * @param {import("../exceptions.js").Exceptions} exceptions
+ * @param {ErrorLog} errorLog
+ */
+function checkExceptions(taxa, exceptions, errorLog) {
+    // Check the Calscape exceptions and make sure they still apply.
+    for (const [name, v] of exceptions.getExceptions()) {
+        const exceptions = v.calscape;
+        if (!exceptions) {
+            continue;
+        }
+
+        // Make sure the taxon is still in our list.
+        const taxon = taxa.getTaxon(name);
+        if (!taxon) {
+            // Don't process global exceptions if taxon is not in local list.
+            if (taxa.isSubset() && !v.local) {
+                continue;
+            }
+            errorLog.log(
+                name,
+                "has Calscape exceptions but not in Taxa collection",
+            );
+            continue;
+        }
+
+        for (const [k] of Object.entries(exceptions)) {
+            switch (k) {
+                case "notnative": {
+                    if (taxon.isCANative()) {
+                        errorLog.log(
+                            name,
+                            "has Calscape notnative exception but is native in taxa.csv",
+                        );
+                    }
+                    break;
+                }
+                default:
+                    errorLog.log(name, "unrecognized Calscape exception", k);
+            }
+        }
+    }
+}
+
+/**
+ * @param {string} taxonName
+ * @param {Map<string,string>} calscapeData
+ * @param {import("../exceptions.js").Exceptions} exceptions
+ * @returns {string|undefined}
+ */
+function getCalscapeCommonName(taxonName, calscapeData, exceptions) {
+    const calscapeCN = calscapeData.get(Taxon.getCalscapeName(taxonName));
+    if (
+        calscapeCN &&
+        exceptions.hasException(taxonName, "calscape", "notnative")
+    ) {
+        return;
+    }
+    return calscapeCN;
+}
+
+/**
+ * @param {string} toolsDataDir
+ * @returns {Promise<Map<string,string>>}
+ */
+async function getCalscapeData(toolsDataDir) {
+    /**
+     * @param {import("exceljs").Cell} cell
+     */
+    function getCellValue(cell) {
+        const value = cell.value;
+        if (value === null || value === undefined) {
+            return undefined;
+        }
+        return value.toString();
+    }
+
+    toolsDataDir = path.join(toolsDataDir, "calscape");
+    Files.mkdir(toolsDataDir);
+    await retrieveCalscapeFile(toolsDataDir);
+
+    /** @type {Map<string,string>} */
+    const data = new Map();
+
+    const wb = new xlsx.Workbook();
+    await wb.xlsx.readFile(getExcelFilename(toolsDataDir)).then(function () {
+        const ws = wb.worksheets[0];
+        let isInData = false;
+        for (let index = 0; index < ws.rowCount; index++) {
+            const row = ws.getRow(index);
+            const col1 = getCellValue(row.getCell(1));
+            if (!isInData) {
+                if (col1 === "Botanical Name") {
+                    isInData = true;
+                }
+                continue;
+            }
+            const col2 = getCellValue(row.getCell(2));
+            if (!col1 || !col2) {
+                continue;
+            }
+            data.set(col1, col2);
+        }
+    });
+
+    return data;
+}
+
+/**
+ * @param {string} toolsDataDir
+ */
+function getExcelFilename(toolsDataDir) {
+    return path.join(toolsDataDir, "calscape.xlsx");
+}
+
+/**
+ * @param {string} toolsDataDir
+ */
+async function retrieveCalscapeFile(toolsDataDir) {
+    // Retrieve file if it's not there.
+    const targetFile = getExcelFilename(toolsDataDir);
+    if (Files.exists(targetFile)) {
+        return;
+    }
+    console.info("retrieving " + targetFile);
+    await Files.fetch("https://www.calscape.org/export/search/", targetFile);
+}
+
+/**
+ * @param {string} dataDir
+ * @param {Map<string,string>} calscapeData
+ * @param {import("../exceptions.js").Exceptions} exceptions
+ */
+function updateTaxaCSV(dataDir, calscapeData, exceptions) {
+    const taxa = new TaxaCSV(dataDir);
+
+    for (const taxonData of taxa.getTaxa()) {
+        const taxonCN = taxonData.calscape_cn;
+        const calscapeCN = getCalscapeCommonName(
+            taxonData.taxon_name,
+            calscapeData,
+            exceptions,
+        );
+        if (taxonCN !== calscapeCN) {
+            if (calscapeCN === undefined) {
+                delete taxonData.calscape_cn;
+            } else {
+                taxonData.calscape_cn = calscapeCN;
+            }
+        }
+    }
+
+    taxa.write();
+}

package/lib/tools/taxacsv.js

@@ -0,0 +1,31 @@
+import path from "path";
+import { CSV } from "../csv.js";
+
+export class TaxaCSV {
+    #filePath;
+    #headers;
+    /** @type {TaxonData[]} */
+    #taxa;
+
+    /**
+     * @param {string} dataDir
+     */
+    constructor(dataDir) {
+        this.#filePath = path.join(dataDir, "taxa.csv");
+        const csv = CSV.readFileAndHeaders(this.#filePath);
+        // @ts-ignore
+        this.#taxa = csv.data;
+        this.#headers = csv.headers;
+    }
+
+    /**
+     * @returns {TaxonData[]}
+     */
+    getTaxa() {
+        return this.#taxa;
+    }
+
+    write() {
+        CSV.writeFileObject(this.#filePath, this.#taxa, this.#headers);
+    }
+}

package/lib/web/pagetaxon.js

@@ -36,6 +36,10 @@ class PageTaxon extends GenericPage {
         if (iNatLink) {
             links.push(iNatLink);
         }
+        const calscapeLink = HTMLTaxon.getCalscapeLink(this.#taxon);
+        if (calscapeLink) {
+            links.push(calscapeLink);
+        }
         const rpiLink = this.#taxon.getRPITaxonLink();
         if (rpiLink) {
             links.push(rpiLink);
@@ -120,7 +124,7 @@ class PageTaxon extends GenericPage {
         }
 
         return HTML.wrap("div", "<ul>" + HTML.arrayToLI(ranks) + "</ul>", {
-            class: "section",
+            class: "section nobullet",
         });
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@ca-plant-list/ca-plant-list",
-    "version": "0.4.10",
+    "version": "0.4.12",
     "description": "Tools to create Jekyll files for a website listing plants in an area of California.",
     "license": "MIT",
     "repository": {
@@ -16,7 +16,8 @@
     "bin": {
         "ca-plant-list": "scripts/build-site.js",
         "ca-plant-book": "scripts/build-ebook.js",
-        "cpl-photos": "scripts/cpl-photos.js"
+        "cpl-photos": "scripts/cpl-photos.js",
+        "cpl-tools": "scripts/cpl-tools.js"
     },
     "dependencies": {
         "archiver": "^5.3.1",
@@ -36,8 +37,8 @@
         "@types/markdown-it": "^14.1.2",
         "@types/node": "^22.7.8",
         "@types/unzipper": "^0.10.9",
-        "ajv-cli": "^5.0.0",
         "eslint": "^9.13.0",
+        "exceljs": "^4.4.0",
         "prettier": "^3.3.3",
         "typescript": "^5.6.3"
     }

package/schemas/exceptions.schema.json

@@ -9,12 +9,17 @@
                     "badjepsonid": {
                         "const": true
                     },
+                    "native": { "type": "boolean" },
                     "notintaxondata": {
                         "const": true
                     }
                 },
                 "additionalProperties": false
             },
+            "calscape": {
+                "type": "object",
+                "properties": { "notnative": { "const": true } }
+            },
             "comment": {
                 "type": "string"
             },
@@ -54,4 +59,4 @@
         },
         "additionalProperties": false
     }
-}
+}

package/scripts/cpl-photos.js

@@ -74,7 +74,11 @@ async function addMissingPhotos(options) {
         }
     }
 
-    CSV.writeFile(`${options.outputdir}/${PHOTO_FILE_NAME}`, data, headers);
+    CSV.writeFileArray(
+        `${options.outputdir}/${PHOTO_FILE_NAME}`,
+        data,
+        headers,
+    );
 }
 
 /**

package/scripts/cpl-tools.js

@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+
+import * as path from "node:path";
+import { Option } from "commander";
+import { Taxa } from "../lib/taxa.js";
+import { Program } from "../lib/program.js";
+import { Calflora } from "../lib/tools/calflora.js";
+import { Exceptions } from "../lib/exceptions.js";
+import { ErrorLog } from "../lib/errorlog.js";
+import { Calscape } from "../lib/tools/calscape.js";
+
+const TOOLS = {
+    CALFLORA: "calflora",
+    CALSCAPE: "calscape",
+    INAT: "inat",
+    JEPSON_EFLORA: "jepson-eflora",
+    JEPSON_FAM: "jepson-families",
+    RPI: "rpi",
+    TEXT: "text",
+};
+
+const ALL_TOOLS = [
+    TOOLS.CALFLORA,
+    TOOLS.CALSCAPE,
+    TOOLS.INAT,
+    TOOLS.JEPSON_EFLORA,
+    TOOLS.RPI,
+    TOOLS.TEXT,
+];
+
+const OPT_LOADER = "loader";
+const OPT_TOOL = "tool";
+
+const TOOLS_DATA_DIR = "./external_data";
+
+/**
+ * @param {import("commander").Command} program
+ * @param {import("commander").OptionValues} options
+ */
+async function build(program, options) {
+    let tools = options[OPT_TOOL];
+    if (!tools) {
+        program.help();
+    }
+    if (tools[0] === "all") {
+        tools = ALL_TOOLS;
+    }
+
+    const exceptions = new Exceptions(options.datadir);
+    // const config = new Config(options.datadir);
+    const taxa = await getTaxa(options);
+
+    const errorLog = new ErrorLog(options.outputdir + "/log.tsv", true);
+    for (const tool of tools) {
+        switch (tool) {
+            case TOOLS.CALFLORA:
+                await Calflora.analyze(
+                    TOOLS_DATA_DIR,
+                    taxa,
+                    exceptions,
+                    errorLog,
+                );
+                break;
+            case TOOLS.CALSCAPE:
+                await Calscape.analyze(
+                    TOOLS_DATA_DIR,
+                    options.datadir,
+                    taxa,
+                    exceptions,
+                    errorLog,
+                    !!options.update,
+                );
+                break;
+            case TOOLS.INAT:
+                // await INat.analyze(
+                //     TOOLS_DATA_DIR,
+                //     taxa,
+                //     exceptions,
+                //     errorLog,
+                //     options.inTaxafile,
+                // );
+                break;
+            case TOOLS.JEPSON_EFLORA: {
+                // const eflora = new JepsonEFlora(
+                //     TOOLS_DATA_DIR,
+                //     taxa,
+                //     errorLog,
+                //     options.efLognotes,
+                // );
+                // await eflora.analyze(exceptions);
+                break;
+            }
+            case TOOLS.JEPSON_FAM:
+                // await JepsonFamilies.build(TOOLS_DATA_DIR, options.outputdir);
+                break;
+            case TOOLS.RPI:
+                // await RPI.analyze(
+                //     TOOLS_DATA_DIR,
+                //     taxa,
+                //     config,
+                //     exceptions,
+                //     errorLog,
+                // );
+                break;
+            case TOOLS.TEXT:
+                // SupplementalText.analyze(taxa, errorLog);
+                break;
+            default:
+                console.log("unrecognized tool: " + tool);
+                return;
+        }
+    }
+
+    errorLog.write();
+}
+
+/**
+ * @param {import("commander").OptionValues} options
+ */
+async function getTaxa(options) {
+    const errorLog = new ErrorLog(options.outputdir + "/errors.tsv", true);
+
+    const loader = options[OPT_LOADER];
+    let taxa;
+    if (loader) {
+        const taxaLoaderClass = await import("file:" + path.resolve(loader));
+        taxa = await taxaLoaderClass.TaxaLoader.loadTaxa(options, errorLog);
+    } else {
+        taxa = new Taxa(
+            Program.getIncludeList(options.datadir),
+            errorLog,
+            options.showFlowerErrors,
+        );
+    }
+
+    errorLog.write();
+    return taxa;
+}
+
+const program = Program.getProgram();
+program.addOption(
+    new Option(
+        "-t, --tool <tool...>",
+        "The tools to run. Value may be any subset of the tools below.",
+    ).choices(["all"].concat(ALL_TOOLS).concat(TOOLS.JEPSON_FAM)),
+);
+program.option(
+    "--in-taxafile <file>",
+    "The name of the file containing the iNaturalist taxa. Can be used for testing on a smaller subset of the iNaturalist data.",
+    "inat_taxa.csv",
+);
+program.option(
+    "--ef-lognotes",
+    "When running the jepson-eflora tool, include eFlora notes, invalid names, etc. in the log file.",
+);
+program.option(
+    "--loader <path>",
+    "The path (relative to the current directory) of the JavaScript file containing the TaxaLoader class. If not provided, the default TaxaLoader will be used.",
+);
+program.option("--update", "Update taxa.csv to remove errors if possible.");
+program.addHelpText(
+    "after",
+    `
+Tools:
+    'all' runs the 'calflora', 'inat', 'jepson-eflora', 'rpi', and 'text' tools.
+    '${TOOLS.CALFLORA}' retrieves data from Calflora and compares with local data.
+    '${TOOLS.CALSCAPE}' retrieves data from Calscape and compares with local data.
+    '${TOOLS.INAT}' retrieves data from iNaturalist and compares with local data.
+    '${TOOLS.JEPSON_EFLORA}' retrieves data from Jepson eFlora indexes and compares with local data.
+    '${TOOLS.JEPSON_FAM}' retrieves section, family and genus data from Jepson eFlora and creates data files for use by ca-plant-list.
+    '${TOOLS.RPI}' retrieves data from the CNPS Rare Plant Inventory and compares with local data.
+    '${TOOLS.TEXT}' checks supplemental text files to make sure their names are referenced.
+    `,
+);
+program.action((options) => build(program, options));
+
+await program.parseAsync();

package/types/classes.d.ts

@@ -83,6 +83,7 @@ declare class Taxa {
     getFlowerColors(): FlowerColor[];
     getTaxon(name: string): Taxon;
     getTaxonList(): Taxon[];
+    isSubset(): boolean;
 }
 
 declare class TaxaCol {
@@ -96,8 +97,11 @@ declare class Taxon {
     getBaseFileName(): string;
     getBloomEnd(): number | undefined;
     getBloomStart(): number | undefined;
+    getCalfloraID(): string;
     getCalfloraName(): string;
     getCalfloraTaxonLink(): string | undefined;
+    getCalscapeCommonName(): string | undefined;
+    getCalscapeName(): string;
     getCESA(): string | undefined;
     getCommonNames(): string[];
     getFamily(): Family;
@@ -123,6 +127,7 @@ declare class Taxon {
     getRPITaxonLink(): string;
     getStatusDescription(config: Config): string;
     getSynonyms(): string[];
+    isCANative(): boolean;
     isNative(): boolean;
 }
 
@@ -130,6 +135,7 @@ declare class TaxonData {
     bloom_end: string;
     bloom_start: string;
     calrecnum: string;
+    calscape_cn?: string;
     CESA: string;
     "common name": string;
     CRPR: string;