@ca-plant-list/ca-plant-list 0.4.17 → 0.4.19

package/lib/tools/jepsoneflora.js

@@ -1,10 +1,17 @@
 import { scrape } from "@htmltools/scrape";
 import { Files } from "../files.js";
+import { SynCSV } from "./syncsv.js";
 
 /**
- * @typedef {{id:string,name:string,common?:string,type:string,under:string}} JepsonInfo
+ * @typedef {{
+ * id:string,
+ * name:string,
+ * common?:string,
+ * type:string,
+ * }} JepsonTaxon
  */
 
+/** @type {Object<string,string>} */
 const TYPES = {
     EX_ALIEN: "Extirpated alien",
     HYBRID_SPONT: "Spontaneous hybrid",
@@ -32,44 +39,47 @@ const TYPES = {
     WEED: "* weed*",
 };
 
-class JepsonEFlora {
+export class JepsonEFlora {
     #toolsDataPath;
     #taxa;
     #errorLog;
-    #shouldLogNotes;
 
-    /** @type {Map<string,JepsonInfo>} */
+    /** @type {Map<string,JepsonTaxon>} */
     #nameInfo = new Map();
-    /** @type {Set<string>} */
-    #loadedLetters = new Set();
+    /** @type {Map<string,string[]>} */
+    #synInfo = new Map();
+    /** @type {import("./syncsv.js").SynData[]} */
+    #synonymsToAdd = [];
 
     /**
      * @param {string} toolsDataDir
      * @param {Taxa} taxa
      * @param {ErrorLog} errorLog
-     * @param {boolean} shouldLogNotes
      */
-    constructor(toolsDataDir, taxa, errorLog, shouldLogNotes) {
+    constructor(toolsDataDir, taxa, errorLog) {
         this.#toolsDataPath = toolsDataDir + "/jepson-eflora";
         this.#taxa = taxa;
         this.#errorLog = errorLog;
-        this.#shouldLogNotes = shouldLogNotes;
     }
 
     /**
      * @param {import("../exceptions.js").Exceptions} exceptions
+     * @param {boolean} update
      */
-    async analyze(exceptions) {
+    async analyze(exceptions, update) {
         // Create data directory if it's not there.
         Files.mkdir(this.#toolsDataPath);
 
+        // Retrieve all Jepson indexes.
+        await this.#loadIndexPages();
+
         for (const taxon of this.#taxa.getTaxonList()) {
             const name = taxon.getName();
             if (name.includes(" unknown")) {
                 continue;
             }
 
-            const jepsInfo = await this.#getJepsInfo(name);
+            const jepsInfo = this.#getJepsInfo(name);
             if (jepsInfo === undefined) {
                 // Not found in the index.
                 if (!exceptions.hasException(name, "jepson", "notineflora")) {
@@ -87,12 +97,6 @@ class JepsonEFlora {
                 );
             }
 
-            if (this.#isSynonym(jepsInfo)) {
-                if (!exceptions.hasException(name, "jepson", "allowsynonym")) {
-                    this.#errorLog.log(name, "is synonym for", jepsInfo.under);
-                }
-            }
-
             const efStatus = this.#getStatusCode(jepsInfo);
             const taxonStatus = taxon.getStatus();
             if (
@@ -108,8 +112,12 @@ class JepsonEFlora {
             }
         }
 
-        await this.#checkSynonyms();
+        this.#checkSynonyms();
         this.#checkExceptions(exceptions);
+
+        if (update) {
+            this.#updateSynCSV();
+        }
     }
 
     /**
@@ -140,15 +148,6 @@ class JepsonEFlora {
             for (const [k] of Object.entries(exceptions)) {
                 const jepsonData = this.#nameInfo.get(name);
                 switch (k) {
-                    case "allowsynonym":
-                        // Make sure it really is a synonym.
-                        if (!this.#isSynonym(jepsonData)) {
-                            this.#errorLog.log(
-                                name,
-                                "has Jepson allowsynonym exception but is not a synonym",
-                            );
-                        }
-                        break;
                     case "notineflora":
                         // Make sure it is really not in eFlora.
                         if (jepsonData) {
@@ -169,42 +168,43 @@ class JepsonEFlora {
         }
     }
 
-    async #checkSynonyms() {
+    #checkSynonyms() {
         // Make sure all synonyms in eFlora are in our list.
-        for (const jepsonInfo of Object.values(this.#nameInfo)) {
-            if (!this.#isSynonym(jepsonInfo)) {
-                continue;
-            }
+        for (const [synName, targetNames] of this.#synInfo.entries()) {
+            for (const targetName of targetNames) {
+                const taxon = this.#taxa.getTaxon(targetName);
+                if (!taxon) {
+                    // We're not tracking the target.
+                    continue;
+                }
 
-            const target = jepsonInfo.under;
-            const taxon = this.#taxa.getTaxon(target);
-            if (!taxon) {
-                // We're not tracking the target.
-                continue;
-            }
+                if (taxon.getSynonyms().includes(synName)) {
+                    // Already have it.
+                    continue;
+                }
 
-            if (taxon.getSynonyms().includes(jepsonInfo.name)) {
-                // Already have it.
-                continue;
+                this.#errorLog.log(
+                    targetName,
+                    "does not have synonym",
+                    synName + "," + targetName,
+                );
+                this.#synonymsToAdd.push({
+                    Former: synName,
+                    Current: targetName,
+                });
             }
-
-            this.#errorLog.log(
-                target,
-                "does not have synonym",
-                jepsonInfo.name + "," + target,
-            );
         }
 
         // Make sure everything in our list is in eFlora.
         for (const taxon of this.#taxa.getTaxonList()) {
             for (const synonym of taxon.getSynonyms()) {
-                const jepsonInfo = await this.#getJepsInfo(synonym);
-                if (!jepsonInfo || !this.#isSynonym(jepsonInfo)) {
+                const synInfo = this.#synInfo.get(synonym);
+                if (!synInfo || !synInfo.includes(taxon.getName())) {
                     // Ignore iNat synonyms.
                     if (synonym !== taxon.getINatSyn()) {
                         this.#errorLog.log(
                             synonym,
-                            "is in synonyms.csv but is not a synonym in eFlora",
+                            `is in synonyms.csv but is not a synonym for ${taxon.getName()} in eFlora`,
                         );
                     }
                 }
@@ -214,31 +214,17 @@ class JepsonEFlora {
 
     /**
      * @param {string} name
-     * @returns {Promise<JepsonInfo|undefined>}
+     * @returns {JepsonTaxon|undefined}
      */
-    async #getJepsInfo(name) {
-        const firstLetter = name[0];
-        // See if this index has been loaded.
-        if (!this.#loadedLetters.has(firstLetter)) {
-            await this.#loadNameIndex(firstLetter);
-        }
-
+    #getJepsInfo(name) {
         return this.#nameInfo.get(name);
     }
 
     /**
-     * @param {JepsonInfo} jepsInfo
+     * @param {JepsonTaxon} jepsInfo
      * @returns {StatusCode|undefined}
      */
     #getStatusCode(jepsInfo) {
-        // If it's a synonym, return status of the target.
-        if (this.#isSynonym(jepsInfo)) {
-            const targetInfo = this.#nameInfo.get(jepsInfo.under);
-            if (!targetInfo) {
-                return;
-            }
-            return this.#getStatusCode(targetInfo);
-        }
         switch (jepsInfo.type) {
             case TYPES.NATIVE:
                 return "N";
@@ -249,19 +235,12 @@ class JepsonEFlora {
         }
     }
 
-    /**
-     * @param {JepsonInfo|undefined} jepsInfo
-     * @returns {boolean}
-     */
-    #isSynonym(jepsInfo) {
-        if (!jepsInfo) {
-            return false;
-        }
-        switch (jepsInfo.type) {
-            case TYPES.SYNONYM:
-                return true;
+    async #loadIndexPages() {
+        for (let index = 0; index < 26; index++) {
+            await this.#loadNameIndex(
+                String.fromCharCode("A".charCodeAt(0) + index),
+            );
         }
-        return false;
     }
 
     /**
@@ -293,32 +272,6 @@ class JepsonEFlora {
 
         const document = scrape.parseFile(filePath);
         this.#parseIndex(document);
-
-        this.#loadedLetters.add(firstLetter);
-    }
-
-    /**
-     * @param {JepsonInfo} taxonData
-     */
-    #logNotes(taxonData) {
-        // If we're tracking the source, log it.
-        if (this.#taxa.getTaxon(taxonData.name)) {
-            this.#errorLog.log(
-                taxonData.name,
-                "has eFlora note (as source)",
-                taxonData.type + " for",
-                taxonData.under,
-            );
-        }
-        // If we're tracking the target, log it.
-        if (this.#taxa.getTaxon(taxonData.under)) {
-            this.#errorLog.log(
-                taxonData.under,
-                "has eFlora note (as target)",
-                taxonData.type + " for",
-                taxonData.name,
-            );
-        }
     }
 
     /**
@@ -370,71 +323,83 @@ class JepsonEFlora {
                 );
             }
 
-            const under = scrape.getTextContent(cols[0]);
             const common = scrape.getTextContent(cols[1]);
+            const name = linkText;
 
-            /** @type {JepsonInfo} */
-            const taxonData = {};
-
-            taxonData.name = linkText;
             const href = scrape.getAttr(links[0], "href");
             if (!href) {
                 throw new Error();
             }
-            taxonData.id = href.split("=")[1];
-            taxonData.type = type;
-            if (taxonData.common) {
-                taxonData.common = common;
-            }
+            const id = href.split("=")[1];
 
-            if (under) {
-                const m = under.match(reUnder);
+            const sciNameText = scrape.getTextContent(cols[0]);
+            let under;
+            if (sciNameText) {
+                const m = sciNameText.match(reUnder);
                 if (m) {
-                    taxonData.under = m[1];
+                    under = m[1];
                 }
             }
 
-            // If we're not tracking either the source or target, ignore this entry.
-            if (
-                !this.#taxa.getTaxon(taxonData.name) &&
-                !this.#taxa.getTaxon(taxonData.under) &&
-                !this.#taxa.hasSynonym(taxonData.name)
-            ) {
-                continue;
+            switch (type) {
+                case TYPES.NATIVE:
+                case TYPES.NATIVITY_UNCERTAIN:
+                case TYPES.NATURALIZED:
+                case TYPES.NATURALIZED_UW:
+                case TYPES.SYNONYM:
+                case TYPES.WAIF:
+                case TYPES.WEED:
+                    break;
+                default:
+                    continue;
             }
 
-            switch (type) {
-                case TYPES.ILLEGITIMATE:
-                case TYPES.INVALID:
-                case TYPES.INVALID_NOTED:
-                case TYPES.INVALID_SUPERFLUOUS:
-                case TYPES.MISAPPLIED:
-                case TYPES.MISAPP_PART:
-                case TYPES.MISAPP_UNABRIDGED:
-                case TYPES.SYN_INED:
-                case TYPES.SYN_ORTH_VARIANT:
-                case TYPES.SYN_PART:
-                case TYPES.SYN_PART_UN:
-                case TYPES.MENTIONED:
-                    // Not a valid synonym or active taxon. Log it for further investigation.
-                    if (this.#shouldLogNotes) {
-                        this.#logNotes(taxonData);
-                    }
+            if (type === TYPES.SYNONYM) {
+                // Should have "under".
+                if (!under) {
+                    throw new Error();
+                }
+                // If we're not tracking the target, ignore this entry.
+                if (!this.#taxa.getTaxon(under)) {
                     continue;
+                }
+
+                // Add to synonyms.
+                let targetNames = this.#synInfo.get(name);
+                if (!targetNames) {
+                    targetNames = [];
+                    this.#synInfo.set(name, targetNames);
+                }
+                targetNames.push(under);
+                continue;
             }
 
-            if (this.#nameInfo.get(taxonData.name)) {
-                this.#errorLog.log(
-                    taxonData.name,
-                    "has multiple entries in eFlora",
-                );
-                // Disable the current entry, since we don't know which one is correct.
-                this.#nameInfo.delete(taxonData.name);
+            // Not a synonym. Should not have "under".
+            if (under) {
+                throw new Error(`under = ${under} for ${name}`);
+            }
+
+            // If we're not tracking either the source, ignore this entry.
+            if (!this.#taxa.getTaxon(name)) {
                 continue;
             }
-            this.#nameInfo.set(taxonData.name, taxonData);
+
+            if (this.#nameInfo.get(name)) {
+                throw new Error();
+            }
+            this.#nameInfo.set(name, {
+                id: id,
+                type: type,
+                name: name,
+                common: common,
+            });
         }
     }
-}
 
-export { JepsonEFlora };
+    #updateSynCSV() {
+        const csv = new SynCSV("./data");
+        const data = csv.getData();
+        data.push(...this.#synonymsToAdd);
+        csv.write();
+    }
+}

package/lib/tools/rpi.js

@@ -12,7 +12,7 @@ class RPI {
 
     /**
      * @param {string} toolsDataDir
-     * @param {Taxa} taxa
+     * @param {import("../taxa.js").Taxa} taxa
      * @param {Config} config
      * @param {import("../exceptions.js").Exceptions} exceptions
      * @param {ErrorLog} errorLog
@@ -202,8 +202,7 @@ class RPI {
     }
 
     /**
-     *
-     * @param {Taxa} taxa
+     * @param {import("../taxa.js").Taxa} taxa
      * @param {Config} config
      * @param {import("../exceptions.js").Exceptions} exceptions
      * @param {ErrorLog} errorLog
@@ -367,7 +366,7 @@ class RPI {
 
     /**
      * @param {string} toolsDataDir
-     * @param {Taxa} taxa
+     * @param {import("../taxa.js").Taxa} taxa
      * @param {import("../exceptions.js").Exceptions} exceptions
      * @param {ErrorLog} errorLog
      */

package/lib/tools/syncsv.js

@@ -0,0 +1,41 @@
+import path from "path";
+import { CSV } from "../csv.js";
+
+/**
+ * @typedef {{Former:string,Current:string,Type?:"INAT"|undefined}} SynData
+ */
+
+export class SynCSV {
+    #filePath;
+    #headers;
+    /** @type {SynData[]} */
+    #data;
+
+    /**
+     * @param {string} dataDir
+     */
+    constructor(dataDir) {
+        this.#filePath = path.join(dataDir, "synonyms.csv");
+        const csv = CSV.readFileAndHeaders(this.#filePath);
+        this.#data = csv.data;
+        this.#headers = csv.headers;
+    }
+
+    /**
+     * @returns {SynData[]}
+     */
+    getData() {
+        return this.#data;
+    }
+
+    write() {
+        this.#data.sort((a, b) => {
+            const former = a.Former.localeCompare(b.Former);
+            if (former !== 0) {
+                return former;
+            }
+            return a.Current.localeCompare(b.Current);
+        });
+        CSV.writeFileObject(this.#filePath, this.#data, this.#headers);
+    }
+}

package/lib/web/pagetaxon.js

@@ -11,8 +11,8 @@ class PageTaxon extends GenericPage {
 
     /**
      * @param {string} outputDir
-     * @param {Config} config
-     * @param {Taxon} taxon
+     * @param {import("../config.js").Config} config
+     * @param {import("../taxon.js").Taxon} taxon
      */
     constructor(outputDir, config, taxon) {
         super(outputDir, taxon.getName(), taxon.getBaseFileName());
@@ -189,7 +189,7 @@ class PageTaxon extends GenericPage {
 
         html += HTMLTaxon.getFooterHTML(this.#taxon);
 
-        const photos = this.#taxon.getPhotos().slice( 0, 5 );
+        const photos = this.#taxon.getPhotos().slice(0, 5);
         if (photos.length > 0) {
             let photosHtml = "";
             for (const photo of photos) {
@@ -202,9 +202,7 @@ class PageTaxon extends GenericPage {
               />
             </a>
             <figcaption>
-              ${photo.rights === "CC0" ? "By" : "(c)"}
-              ${photo.rightsHolder}
-              ${photo.rights && `(${photo.rights})`}
+              ${photo.getAttribution()}
             </figcaption>
           </figure>
         `;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@ca-plant-list/ca-plant-list",
-    "version": "0.4.17",
+    "version": "0.4.19",
     "description": "Tools to create Jekyll files for a website listing plants in an area of California.",
     "license": "MIT",
     "repository": {
@@ -22,8 +22,9 @@
     },
     "types": "./lib/index.d.ts",
     "scripts": {
-        "check": "npm run eslint && npm run tsc",
+        "check": "npm run eslint && npm run tsc && npm run jest",
         "eslint": "npx eslint",
+        "jest": "node --experimental-vm-modules node_modules/jest/bin/jest.js tests",
         "prettier": "npx prettier -l .",
         "tsc": "npx tsc"
     },
@@ -35,6 +36,7 @@
         "inatobsphotos": "scripts/inatobsphotos.js"
     },
     "dependencies": {
+        "@htmltools/scrape": "^0.1.0",
         "archiver": "^5.3.1",
         "cli-progress": "^3.12.0",
         "commander": "^12.1.0",
@@ -48,13 +50,14 @@
         "unzipper": "^0.12.3"
     },
     "devDependencies": {
-        "@htmltools/scrape": "^0.1.0",
         "@types/archiver": "^6.0.2",
         "@types/cli-progress": "^3.11.6",
+        "@types/jest": "^29.5.14",
         "@types/markdown-it": "^14.1.2",
         "@types/node": "^22.10.3",
         "@types/unzipper": "^0.10.9",
         "eslint": "^9.17.0",
+        "jest": "^29.7.0",
         "prettier": "^3.4.2",
         "typescript": "^5.7.2"
     }

package/scripts/cpl-tools.js

@@ -85,13 +85,8 @@ async function build(program, options) {
                 );
                 break;
             case TOOLS.JEPSON_EFLORA: {
-                const eflora = new JepsonEFlora(
-                    TOOLS_DATA_DIR,
-                    taxa,
-                    errorLog,
-                    options.efLognotes,
-                );
-                await eflora.analyze(exceptions);
+                const eflora = new JepsonEFlora(TOOLS_DATA_DIR, taxa, errorLog);
+                await eflora.analyze(exceptions, !!options.update);
                 break;
             }
             case TOOLS.JEPSON_FAM:
@@ -130,10 +125,6 @@ program.option(
     "The name of the file containing the iNaturalist taxa. Can be used for testing on a smaller subset of the iNaturalist data.",
     "inat_taxa.csv",
 );
-program.option(
-    "--ef-lognotes",
-    "When running the jepson-eflora tool, include eFlora notes, invalid names, etc. in the log file.",
-);
 program.option("--update", "Update taxa.csv to remove errors if possible.");
 program.addHelpText(
     "after",

package/types/classes.d.ts

@@ -23,7 +23,6 @@ declare class ErrorLog {
 declare class Families {
     getFamilies(): Family[];
     getFamily(name: string): Family;
-    renderPages(outputDir: string, cols?: TaxaCol[]): void;
 }
 
 declare class Family {
@@ -41,7 +40,6 @@ declare class FlowerColor {
 }
 
 declare class Genera {
-    addTaxon(taxon: Taxon): void;
     getGenus(name: string): Genus;
 }
 
@@ -87,15 +85,8 @@ declare class Taxa {
     isSubset(): boolean;
 }
 
-declare class TaxaCol {
-    class?: string;
-    data: (taxon: Taxon) => string;
-    title: string;
-}
-
 type StatusCode = "N" | "NC" | "U" | "X";
 declare class Taxon {
-    constructor(data: TaxonData, genera: Genera, meta: any);
     getBaseFileName(): string;
     getBloomEnd(): number | undefined;
     getBloomStart(): number | undefined;
@@ -104,8 +95,6 @@ declare class Taxon {
     getCalfloraTaxonLink(): string | undefined;
     getCalscapeCommonName(): string | undefined;
     getCalscapeName(): string;
-    getCESA(): string | undefined;
-    getCNDDBRank(): string | undefined;
     getCommonNames(): string[];
     getFamily(): Family;
     getFESA(): string | undefined;
@@ -125,7 +114,6 @@ declare class Taxon {
     getJepsonID(): string;
     getLifeCycle(): string;
     getName(): string;
-    getPhotos(): Photo[];
     getRPIID(): string | undefined;
     getRPIRank(): string;
     getRPIRankAndThreat(): string;
@@ -160,21 +148,6 @@ declare class TaxonData {
 
 type PhotoRights = "CC0" | "CC BY" | "CC BY-NC" | "C" | null;
 
-declare class Photo {
-    url?: string;
-    rightsHolder: null | string;
-    rights?: PhotoRights;
-    getExt(): string;
-    getId(): number;
-    getUrl(): string;
-    getSourceUrl(): string;
-}
-
-declare class InatPhoto extends Photo {
-    inatPhotoId: number;
-    ext: string;
-}
-
 type InatPhotoInfo = {
     id: string;
     ext: string;
@@ -219,7 +192,7 @@ declare class InatApiTaxon {
 declare class InatApiObservation {
     observation_photos: {
         photo: InatApiPhoto;
-    }[]
+    }[];
 }
 
 declare class InatObsPhotosCommandLineOptions extends CommandLineOptions {