@ca-plant-list/ca-plant-list 0.4.32 → 0.4.35

package/lib/ebook/plantbook.js

@@ -11,16 +11,19 @@ import { TaxonPage } from "./pages/taxonpage.js";
 import { TOCPage } from "./pages/tocpage.js";
 
 class PlantBook extends EBook {
+    #config;
     #taxa;
     #glossary;
     #images;
+    #maxTaxa;
 
     /**
      * @param {string} outputDir
      * @param {import("../config.js").Config} config
      * @param {import("../types.js").Taxa} taxa
+     * @param {number|undefined} maxTaxa
      */
-    constructor(outputDir, config, taxa) {
+    constructor(outputDir, config, taxa, maxTaxa) {
         super(
             outputDir,
             getRequiredConfigValue(config, "filename"),
@@ -28,16 +31,21 @@ class PlantBook extends EBook {
             getRequiredConfigValue(config, "title"),
         );
 
+        this.#config = config;
         this.#taxa = taxa;
         const generator = new EBookSiteGenerator(config, this.getContentDir());
         this.#glossary = new GlossaryPages(generator);
         this.#images = new Images(generator, this.getContentDir(), taxa);
+        this.#maxTaxa = maxTaxa;
     }
 
     async createPages() {
         const contentDir = this.getContentDir();
 
-        const taxonList = this.#taxa.getTaxonList();
+        const allTaxa = this.#taxa.getTaxonList();
+        const taxonList = this.#maxTaxa
+            ? allTaxa.slice(0, this.#maxTaxa)
+            : allTaxa;
 
         await this.#images.createImages(taxonList);
 
@@ -45,9 +53,15 @@ class PlantBook extends EBook {
             "creating taxon pages",
             taxonList.length,
         );
+
         for (let index = 0; index < taxonList.length; index++) {
             const taxon = taxonList[index];
-            new TaxonPage(contentDir, taxon, this.#images).create();
+            new TaxonPage(
+                contentDir,
+                this.#config,
+                taxon,
+                this.#images,
+            ).create();
             meter.update(index + 1);
         }
         meter.stop();

package/lib/externalsites.js

@@ -30,6 +30,18 @@ export class ExternalSites {
         return new URL("https://www.calflora.org/app/taxon?crn=" + calfloraID);
     }
 
+    /**
+     * @param {import("./types.js").Taxon} taxon
+     * @returns {URL|undefined}
+     */
+    static getCalIPCRefLink(taxon) {
+        const calipcID = taxon.getCalIPCID();
+        if (!calipcID) {
+            return;
+        }
+        return new URL(`https://www.cal-ipc.org/plants/profile/${calipcID}/`);
+    }
+
     /**
      * @param {import("./types.js").Taxon} taxon
      * @returns {URL|undefined}

package/lib/htmltaxon.js

@@ -74,6 +74,10 @@ const REFLINKS = {
         label: "Calflora",
         href: (taxon) => ExternalSites.getCalfloraRefLink(taxon),
     },
+    calipc: {
+        label: "Cal-IPC",
+        href: (taxon) => ExternalSites.getCalIPCRefLink(taxon),
+    },
     calscape: {
         label: "Calscape",
         href: (taxon) => ExternalSites.getCalscapeLink(taxon),

package/lib/index.d.ts

@@ -8,6 +8,7 @@ type PhotoRights = "CC0" | "CC BY" | "CC BY-NC" | "C" | null;
 
 type RefSourceCode =
     | "calflora"
+    | "calipc"
     | "calscape"
     | "cch"
     | "fna"
@@ -40,6 +41,7 @@ export type TaxonData = TaxonomyData & {
     CRPR: string;
     FESA: string;
     fna: string;
+    calipc: string;
     flower_color: string;
     GRank: string;
     "inat id": string;

package/lib/taxonomy/taxon.js

@@ -13,6 +13,7 @@ class Taxon extends Taxonomy {
     #cch2id;
     #fnaName;
     #calscapeCN;
+    #calipcID;
     #lifeCycle;
     #flowerColors;
     #bloomStart;
@@ -51,6 +52,7 @@ class Taxon extends Taxonomy {
         this.#iNatID = data["inat id"];
         this.#cch2id = data.cch2_id;
         this.#fnaName = data.fna ?? "";
+        this.#calipcID = data.calipc ?? "";
         this.#calscapeCN =
             data.calscape_cn === "" ? undefined : data.calscape_cn;
         this.#lifeCycle = data.life_cycle;
@@ -118,12 +120,20 @@ class Taxon extends Taxonomy {
         return this.#bloomStart;
     }
 
+    getCalfloraID() {
+        return this.#calRecNum;
+    }
+
     getCalfloraName() {
         return this.getName().replace(" subsp.", " ssp.").replace("×", "X");
     }
 
-    getCalfloraID() {
-        return this.#calRecNum;
+    getCalIPCID() {
+        return this.#calipcID;
+    }
+
+    getCalIPCName() {
+        return this.getName().replace(" subsp.", " ssp.");
     }
 
     getCalscapeCommonName() {

package/lib/tools/calflora.js

@@ -70,12 +70,10 @@ export class Calflora {
         );
 
         /** @type {CalfloraData[]} */
-        // @ts-ignore
         const csvActive = CSV.readFile(
             path.join(toolsDataPath, calfloraDataFileNameActive),
         );
         /** @type {CalfloraData[]} */
-        // @ts-ignore
         const csvCounties = CSV.readFile(
             path.join(toolsDataPath, calfloraDataFileNameCounties),
         );

package/lib/tools/calipc.js

@@ -0,0 +1,111 @@
+import * as path from "path";
+import { Files } from "../files.js";
+import { TaxaCSV } from "./taxacsv.js";
+import { scrape } from "@htmltools/scrape";
+
+export class CalIPC {
+    /**
+     * @param {string} toolsDataDir
+     * @param {string} dataDir
+     * @param {import("../types.js").Taxa} taxa
+     * @param {import("../exceptions.js").Exceptions} exceptions
+     * @param {import("../errorlog.js").ErrorLog} errorLog
+     * @param {boolean} update
+     */
+    static async analyze(
+        toolsDataDir,
+        dataDir,
+        taxa,
+        exceptions,
+        errorLog,
+        update,
+    ) {
+        const toolsDataPath = toolsDataDir + "/calipc";
+        // Create data directory if it's not there.
+        Files.mkdir(toolsDataPath);
+
+        const calipcFileName = path.join(toolsDataPath, "inventory.html");
+        if (!Files.exists(calipcFileName)) {
+            console.info("retrieving " + calipcFileName);
+            await Files.fetch(
+                "https://www.cal-ipc.org/plants/inventory/",
+                calipcFileName,
+            );
+        }
+
+        const parsed = scrape.parseFile(calipcFileName);
+        const links = scrape.getSubtrees(parsed, (e) => {
+            if (e.tagName !== "td") {
+                return false;
+            }
+            const className = scrape.getAttr(e, "class");
+            return className === "it-latin";
+        });
+
+        /** @type {Map<string,string>} */
+        const calipcTaxa = new Map();
+        for (const link of links) {
+            const name = scrape.getTextContent(link);
+            const url = scrape.getAttr(link.children[0], "href");
+            if (!url) {
+                console.warn(`Cal-IPC url not found for ${name}`);
+                continue;
+            }
+            const id = url.match(/\/profile\/(.*)\//);
+            if (!id) {
+                console.warn(`Cal-IPC url mismatch for ${url}`);
+                continue;
+            }
+            calipcTaxa.set(name, id[1]);
+        }
+
+        const idsToUpdate = new Map();
+
+        for (const taxon of taxa.getTaxonList()) {
+            const name = taxon.getName();
+            if (name.includes(" unknown")) {
+                continue;
+            }
+            const calipcName = taxon.getCalIPCName();
+            const calipcData = calipcTaxa.get(calipcName);
+            if (!calipcData) {
+                continue;
+            }
+
+            // Check native status.
+            if (taxon.isCANative()) {
+                errorLog.log(name, "is native but listed in Cal-IPC");
+            }
+
+            if (calipcData !== taxon.getCalIPCID()) {
+                errorLog.log(
+                    name,
+                    "Cal-IPC ID in Cal-IPC is different than taxa.csv",
+                    calipcData,
+                    taxon.getCalIPCID(),
+                );
+                idsToUpdate.set(name, calipcData);
+            }
+        }
+
+        if (update) {
+            this.#updateIds(dataDir, idsToUpdate);
+        }
+    }
+
+    /**
+     * @param {string} dataDir
+     * @param {Map<string,string>} idsToUpdate
+     */
+    static #updateIds(dataDir, idsToUpdate) {
+        const taxa = new TaxaCSV(dataDir);
+        for (const taxonData of taxa.getTaxa()) {
+            const id = idsToUpdate.get(taxonData.taxon_name);
+            if (!id) {
+                continue;
+            }
+            taxonData["calipc"] = id;
+        }
+        taxa.write();
+    }
+}

package/lib/tools/taxacsv.js

@@ -5,16 +5,17 @@ const HEADERS = [
     "taxon_name",
     "common name",
     "status",
+    "life_cycle",
+    "flower_color",
+    "bloom_start",
+    "bloom_end",
     "jepson id",
     "calrecnum",
     "inat id",
     "cch2_id",
     "fna",
     "calscape_cn",
-    "life_cycle",
-    "flower_color",
-    "bloom_start",
-    "bloom_end",
+    "calipc",
     "RPI ID",
     "CRPR",
     "CESA",

package/lib/utils/eleventyGenerator.js

@@ -66,6 +66,8 @@ export class EleventyGenerator extends SiteGenerator {
                     const passThroughPatterns = [
                         "assets",
                         "i",
+                        "calflora_list_*.txt",
+                        "inat_list_*.txt",
                         "errors.tsv",
                         ...generator.getPassThroughPatterns(),
                     ];

package/lib/utils/inat-tools.js

@@ -69,7 +69,11 @@ export function convertToCSVPhoto(apiPhoto) {
  */
 async function fetchInatTaxa(inatTaxonIDs) {
     const url = `https://api.inaturalist.org/v2/taxa/${inatTaxonIDs.join(",")}?fields=(taxon_photos:(photo:(medium_url:!t,attribution:!t,license_code:!t)))`;
-    const resp = await fetch(url);
+    const resp = await getResponse(url);
+    if (resp instanceof Error) {
+        console.error(`unable to fetch taxa: ${resp.message}`);
+        return [];
+    }
     if (!resp.ok) {
         const error = await resp.text();
         throw new Error(`Failed to fetch taxa from iNat: ${error}`);
@@ -163,7 +167,7 @@ export async function getObsPhotosForTaxa(taxaToUpdate, locationOptions) {
             continue;
         }
 
-        // Just get the CC-licensed ones, 5 per taxon should be fine (max is 20 on iNat). Whether or not
+        // Just get the CC-licensed ones, 5 per taxon should be fine (max is 20 on iNat).
         const rawPhotoInfo = observations
             .map((obs) =>
                 obs.observation_photos.map((op) => {
@@ -182,7 +186,7 @@ export async function getObsPhotosForTaxa(taxaToUpdate, locationOptions) {
             if (!obj) {
                 continue;
             }
-            processedPhotoInfo.push(obj);
+            processedPhotoInfo.push({ obsId: photo.obsId.toString(), ...obj });
         }
         photos.set(taxon.getName(), processedPhotoInfo);
 
@@ -270,7 +274,7 @@ function getAttribution(rawAttribution) {
 
 let lastQueryTime = Date.now();
 /**
- * @param {URL} url
+ * @param {URL|string} url
  * @returns {Promise<Response|Error>}
  */
 async function getResponse(url) {

package/lib/web/pageTaxon.js

@@ -35,6 +35,7 @@ export class PageTaxon extends GenericPage {
         HTMLTaxon.addRefLink(links, this.#taxon, "fna");
         HTMLTaxon.addRefLink(links, this.#taxon, "cch");
         HTMLTaxon.addRefLink(links, this.#taxon, "calscape");
+        HTMLTaxon.addRefLink(links, this.#taxon, "calipc");
         return links;
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@ca-plant-list/ca-plant-list",
-    "version": "0.4.32",
+    "version": "0.4.35",
     "description": "Tools to create files for a website listing plants in an area of California.",
     "license": "MIT",
     "repository": {

package/scripts/build-ebook.js

@@ -8,9 +8,16 @@ import { Program } from "../lib/program.js";
 import { Taxa } from "../lib/taxonomy/taxa.js";
 
 const program = Program.getProgram();
-program
-    .option("-l, --locationsdir <dir>", "directory containing location data")
-    .action(build);
+program.option(
+    "-l, --locationsdir <dir>",
+    "directory containing location data",
+);
+program.option(
+    "--location <name>",
+    "name of location to generate (otherwise all ebooks will be generated)",
+);
+program.option("--max-taxa <number>", "maximum number of taxa to include");
+program.action(build);
 
 await program.parseAsync();
 
@@ -26,6 +33,11 @@ async function build(options) {
         const outputBase = options.outputdir;
         const subdirs = Files.getDirEntries(locationsDir);
         for (const subdir of subdirs) {
+            // If a single location was specified, ignore the others.
+            if (options.location && subdir !== options.location) {
+                continue;
+            }
+
             console.log("Generating " + subdir);
             const suffix = "/" + subdir;
             const path = locationsDir + suffix;
@@ -34,6 +46,7 @@ async function build(options) {
                     outputBase + suffix,
                     path,
                     options.showFlowerErrors,
+                    options.maxTaxa,
                 );
             }
         }
@@ -43,6 +56,7 @@ async function build(options) {
             options.outputdir,
             options.datadir,
             options.showFlowerErrors,
+            options.maxTaxa,
         );
     }
 }
@@ -51,8 +65,9 @@ async function build(options) {
  * @param {string} outputDir
  * @param {string} dataDir
  * @param {boolean} showFlowerErrors
+ * @param {number|undefined} maxTaxa
  */
-async function buildBook(outputDir, dataDir, showFlowerErrors) {
+async function buildBook(outputDir, dataDir, showFlowerErrors, maxTaxa) {
     const errorLog = new ErrorLog(outputDir + "/errors.tsv");
 
     const taxa = new Taxa(
@@ -62,7 +77,7 @@ async function buildBook(outputDir, dataDir, showFlowerErrors) {
     );
 
     const config = new Config(dataDir);
-    const ebook = new PlantBook(outputDir, config, taxa);
+    const ebook = new PlantBook(outputDir, config, taxa, maxTaxa);
     await ebook.create();
     errorLog.write();
 }

package/scripts/cpl-photos.js

@@ -232,7 +232,7 @@ async function checkObsPhotos(options, errorLog) {
         for (const photo of photos) {
             const obsId = photo.obsId;
             if (!obsId) {
-                throw new Error();
+                throw new Error(`no obsId in ${JSON.stringify(photo)}`);
             }
             const currentPhotos = photosById.get(obsId);
             if (!currentPhotos) {
@@ -641,11 +641,11 @@ if (!isLocal) {
 
 const checkCommand = program.command("check");
 checkCommand
-    .description("Check photo data to ensure information is current.")
+    .description("Check photo data to ensure information is current")
     .action((options) => check(program.opts(), options));
 if (!isLocal) {
-    checkCommand.option("--observations", `Check ${OBS_PHOTO_FILE_NAME}.`);
-    checkCommand.option("--taxa", `Check ${TAXON_PHOTO_FILE_NAME}.`);
+    checkCommand.option("--observations", `Check ${OBS_PHOTO_FILE_NAME}`);
+    checkCommand.option("--taxa", `Check ${TAXON_PHOTO_FILE_NAME}`);
 }
 
 program

package/scripts/cpl-tools.js

@@ -15,9 +15,11 @@ import { SupplementalText } from "../lib/tools/supplementaltext.js";
 import { JepsonFamilies } from "../lib/tools/jepsonfamilies.js";
 import { CCH2 } from "../lib/tools/cch2.js";
 import { FNA } from "../lib/tools/fna.js";
+import { CalIPC } from "../lib/tools/calipc.js";
 
 const TOOLS = {
     CALFLORA: "calflora",
+    CAL_IPC: "calipc",
     CALSCAPE: "calscape",
     CCH2: "cch",
     FNA: "fna",
@@ -30,6 +32,7 @@ const TOOLS = {
 
 const ALL_TOOLS = [
     TOOLS.CALFLORA,
+    TOOLS.CAL_IPC,
     TOOLS.CALSCAPE,
     TOOLS.CCH2,
     TOOLS.FNA,
@@ -76,6 +79,16 @@ async function build(program, options) {
                     !!options.update,
                 );
                 break;
+            case TOOLS.CAL_IPC:
+                await CalIPC.analyze(
+                    TOOLS_DATA_DIR,
+                    options.datadir,
+                    taxa,
+                    exceptions,
+                    errorLog,
+                    !!options.update,
+                );
+                break;
             case TOOLS.CALSCAPE:
                 await Calscape.analyze(
                     TOOLS_DATA_DIR,
@@ -167,8 +180,9 @@ program.addHelpText(
     "after",
     `
 Tools:
-    'all' runs the 'calflora', '${TOOLS.CALSCAPE}', '${TOOLS.CCH2}, '${TOOLS.FNA}, 'inat', 'jepson-eflora', 'rpi', and 'text' tools.
+    'all' runs the 'calflora', '${TOOLS.CAL_IPC}', '${TOOLS.CALSCAPE}', '${TOOLS.CCH2}, '${TOOLS.FNA}, 'inat', 'jepson-eflora', 'rpi', and 'text' tools.
     '${TOOLS.CALFLORA}' retrieves data from Calflora and compares with local data.
+    '${TOOLS.CAL_IPC}' retrieves data from Cal-IPC and compares with local data.
     '${TOOLS.CALSCAPE}' retrieves data from Calscape and compares with local data.
     '${TOOLS.CCH2}' retrieves data from CCH2 and compares with local data.
     '${TOOLS.FNA}' retrieves data from Flora of North America and compares with local data.