@ca-plant-list/ca-plant-list 0.4.9 → 0.4.11

package/lib/taxa.js

@@ -48,7 +48,7 @@ class Taxa {
         taxonFactory = (td, g) => new Taxon(td, g),
         extraTaxa = [],
         extraSynonyms = [],
-        includePhotos = true
+        includePhotos = true,
     ) {
         this.#isSubset = inclusionList !== true;
 
@@ -68,7 +68,7 @@ class Taxa {
             extraTaxa,
             inclusionList,
             taxonFactory,
-            showFlowerErrors
+            showFlowerErrors,
         );
 
         // Make sure everything in the inclusionList has been loaded.
@@ -79,12 +79,11 @@ class Taxa {
         }
 
         this.#sortedTaxa = Object.values(this.#taxa).sort((a, b) =>
-            a.getName().localeCompare(b.getName())
+            a.getName().localeCompare(b.getName()),
         );
 
-
-        if ( includePhotos ) {
-            this.#loadInatPhotos( dataDir );
+        if (includePhotos) {
+            this.#loadInatPhotos(dataDir);
         }
 
         const synCSV = CSV.parseFile(dataDir, "synonyms.csv");
@@ -95,22 +94,24 @@ class Taxa {
     /**
      * @param {string} dataDir
      */
-    #loadInatPhotos( dataDir ) {
+    #loadInatPhotos(dataDir) {
         const photosFileName = "inattaxonphotos.csv";
-        if ( fs.existsSync( path.join( dataDir, photosFileName ) ) ) {
+        if (fs.existsSync(path.join(dataDir, photosFileName))) {
             /** @type {InatCsvPhoto[]} */
-            const csvPhotos = CSV.parseFile( dataDir, photosFileName );
-            for ( const csvPhoto of csvPhotos ) {
+            const csvPhotos = CSV.parseFile(dataDir, photosFileName);
+            for (const csvPhoto of csvPhotos) {
                 const taxon = this.getTaxon(csvPhoto.name);
-                if(!taxon) {
+                if (!taxon) {
                     continue;
                 }
-                taxon.addPhoto(new InatPhoto(
-                    csvPhoto.id,
-                    csvPhoto.ext,
-                    csvPhoto.licenseCode,
-                    csvPhoto.attrName
-                ) );
+                taxon.addPhoto(
+                    new InatPhoto(
+                        csvPhoto.id,
+                        csvPhoto.ext,
+                        csvPhoto.licenseCode,
+                        csvPhoto.attrName,
+                    ),
+                );
             }
         }
     }
@@ -121,7 +122,7 @@ class Taxa {
 
     getFlowerColors() {
         return Object.values(this.#flower_colors).filter(
-            (fc) => fc.getTaxa().length > 0
+            (fc) => fc.getTaxa().length > 0,
         );
     }
 
@@ -209,10 +210,10 @@ class Taxa {
                     const color = this.#flower_colors[colorName];
                     if (!color) {
                         throw new Error(
-                            "flower color \"" +
+                            'flower color "' +
                                 colorName +
-                                "\" not found for " +
-                                name
+                                '" not found for ' +
+                                name,
                         );
                     }
                     color.addTaxon(taxon);
@@ -229,7 +230,7 @@ class Taxa {
                     ) {
                         this.#errorLog.log(
                             name,
-                            "does not have all flower info"
+                            "does not have all flower info",
                         );
                     }
                 } else {

package/lib/util.js

@@ -3,8 +3,9 @@
  * https://github.com/you-dont-need/You-Dont-Need-Lodash-Underscore?tab=readme-ov-file#_chunk
  * @param {any[]} input
  * @param {number} size
+ * @returns {any[][]}
  */
-export function chunk( input, size ) {
+export function chunk(input, size) {
     return input.reduce((arr, item, idx) => {
         return idx % size === 0
             ? [...arr, [item]]
@@ -15,6 +16,6 @@ export function chunk( input, size ) {
 /**
  * @param {number} time
  */
-export async function sleep( time ) {
-    return new Promise( resolve => setTimeout( resolve, time ) );
+export async function sleep(time) {
+    return new Promise((resolve) => setTimeout(resolve, time));
 }

package/lib/utils/inat-tools.js

@@ -0,0 +1,90 @@
+import { ProgressMeter } from "../progressmeter.js";
+import { chunk, sleep } from "../util.js";
+
+const ALLOWED_LICENSE_CODES = ["cc0", "cc-by", "cc-by-nc"];
+
+/**
+ * @param {Taxon[]} taxa
+ * @return {Promise<InatApiTaxon[]>}
+ */
+async function fetchInatTaxa(taxa) {
+    const inatTaxonIDs = taxa.map((taxon) => taxon.getINatID()).filter(Boolean);
+    const url = `https://api.inaturalist.org/v2/taxa/${inatTaxonIDs.join(",")}?fields=(taxon_photos:(photo:(medium_url:!t,attribution:!t,license_code:!t)))`;
+    const resp = await fetch(url);
+    if (!resp.ok) {
+        const error = await resp.text();
+        throw new Error(`Failed to fetch taxa from iNat: ${error}`);
+    }
+    const json = await resp.json();
+    return json.results;
+}
+
+/**
+ * @param {Taxon[]} taxaToUpdate
+ * @returns {Promise<Map<string,InatPhotoInfo[]>>}
+ */
+export async function getTaxonPhotos(taxaToUpdate) {
+    /** @type {Map<string,string>} */
+    const idMap = new Map();
+
+    for (const taxon of taxaToUpdate) {
+        if (taxon.getINatID()) {
+            idMap.set(taxon.getINatID(), taxon.getName());
+        }
+    }
+
+    /** @type {Map<string,InatPhotoInfo[]>} */
+    const photos = new Map();
+
+    const meter = new ProgressMeter("retrieving taxa", taxaToUpdate.length);
+    let taxaRetrieved = 0;
+
+    for (const batch of chunk(taxaToUpdate, 30)) {
+        const inatTaxa = await fetchInatTaxa(batch);
+        for (const iNatTaxon of inatTaxa) {
+            const iNatTaxonPhotos = iNatTaxon.taxon_photos
+                .filter((tp) =>
+                    ALLOWED_LICENSE_CODES.includes(tp.photo.license_code),
+                )
+                .slice(0, 5);
+
+            const taxonName = idMap.get(iNatTaxon.id.toString());
+            if (!taxonName) {
+                throw new Error(`iNat id ${iNatTaxon.id} not found`);
+            }
+            /** @type {InatPhotoInfo[]} */
+            const taxonPhotos = [];
+            for (const taxonPhoto of iNatTaxonPhotos) {
+                const ext = taxonPhoto.photo.medium_url.split(".").at(-1);
+                if (!ext) {
+                    continue;
+                }
+                /** @type {InatPhotoInfo} */
+                const obj = {
+                    id: taxonPhoto.photo.id.toString(),
+                    ext: ext,
+                    licenseCode: taxonPhoto.photo.license_code,
+                    attrName:
+                        // Photographers retain copyright for most CC licenses,
+                        // except CC0, so attribution is a bit different
+                        taxonPhoto.photo.attribution.match(
+                            /\(c\) (.*?),/,
+                        )?.[1] ||
+                        taxonPhoto.photo.attribution.match(
+                            /uploaded by (.*)/,
+                        )?.[1],
+                };
+
+                taxonPhotos.push(obj);
+            }
+            photos.set(taxonName, taxonPhotos);
+        }
+        taxaRetrieved += batch.length;
+        meter.update(taxaRetrieved);
+        await sleep(1_100);
+    }
+
+    meter.stop();
+
+    return photos;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@ca-plant-list/ca-plant-list",
-    "version": "0.4.9",
+    "version": "0.4.11",
     "description": "Tools to create Jekyll files for a website listing plants in an area of California.",
     "license": "MIT",
     "repository": {
@@ -16,7 +16,7 @@
     "bin": {
         "ca-plant-list": "scripts/build-site.js",
         "ca-plant-book": "scripts/build-ebook.js",
-        "photo-tools": "scripts/photos.js"
+        "cpl-photos": "scripts/cpl-photos.js"
     },
     "dependencies": {
         "archiver": "^5.3.1",

package/scripts/cpl-photos.js

@@ -0,0 +1,179 @@
+#!/usr/bin/env node
+
+import path from "path";
+import { ErrorLog } from "../lib/errorlog.js";
+import { Program } from "../lib/program.js";
+import { Taxa } from "../lib/taxa.js";
+import { getTaxonPhotos } from "../lib/utils/inat-tools.js";
+import { existsSync } from "fs";
+import { CSV } from "../lib/csv.js";
+
+const PHOTO_FILE_NAME = "inattaxonphotos.csv";
+
+const OPT_LOADER = "loader";
+
+const MAX_PHOTOS = 5;
+
+/**
+ * @param {import("commander").OptionValues} options
+ */
+async function addMissingPhotos(options) {
+    const taxaMissingPhotos = [];
+
+    const taxa = await getTaxa(options);
+    const errorLog = new ErrorLog(options.outputdir + "/log.tsv", true);
+
+    for (const taxon of taxa.getTaxonList()) {
+        const photos = taxon.getPhotos();
+        if (photos.length < MAX_PHOTOS) {
+            taxaMissingPhotos.push(taxon);
+        }
+    }
+
+    const newPhotos = await getTaxonPhotos(taxaMissingPhotos);
+    const currentTaxaPhotos = readPhotos();
+
+    for (const [taxonName, photos] of newPhotos) {
+        let currentPhotos = currentTaxaPhotos.get(taxonName);
+        if (!currentPhotos) {
+            currentPhotos = [];
+            currentTaxaPhotos.set(taxonName, currentPhotos);
+        }
+        for (const photo of photos) {
+            if (currentPhotos.length === MAX_PHOTOS) {
+                break;
+            }
+            if (
+                currentPhotos.some(
+                    (currentPhoto) => currentPhoto.id === photo.id,
+                )
+            ) {
+                continue;
+            }
+            currentPhotos.push(photo);
+            errorLog.log("adding photo", taxonName, photo.id);
+        }
+    }
+
+    errorLog.write();
+
+    // Write updated photo file.
+    const headers = ["name", "id", "ext", "licenseCode", "attrName"];
+    /** @type {string[][]} */
+    const data = [];
+    for (const taxonName of [...currentTaxaPhotos.keys()].sort()) {
+        // @ts-ignore
+        for (const photo of currentTaxaPhotos.get(taxonName)) {
+            data.push([
+                taxonName,
+                photo.id,
+                photo.ext,
+                photo.licenseCode,
+                photo.attrName ?? "",
+            ]);
+        }
+    }
+
+    CSV.writeFile(`${options.outputdir}/${PHOTO_FILE_NAME}`, data, headers);
+}
+
+/**
+ * @param {import("commander").OptionValues} options
+ * @param {import("commander").OptionValues} commandOptions
+ */
+async function checkmissing(options, commandOptions) {
+    const taxa = await getTaxa(options);
+    const errorLog = new ErrorLog(options.outputdir + "/log.tsv", true);
+
+    const minPhotos = commandOptions.minphotos;
+    for (const taxon of taxa.getTaxonList()) {
+        const photos = taxon.getPhotos();
+        if (
+            minPhotos === undefined
+                ? photos.length !== MAX_PHOTOS
+                : photos.length < minPhotos
+        ) {
+            errorLog.log(taxon.getName(), photos.length.toString());
+        }
+    }
+
+    errorLog.write();
+}
+
+/**
+ * @param {import("commander").OptionValues} options
+ * @return {Promise<Taxa>}
+ */
+async function getTaxa(options) {
+    const errorLog = new ErrorLog(options.outputdir + "/errors.tsv", true);
+
+    const loader = options[OPT_LOADER];
+    let taxa;
+    if (loader) {
+        const taxaLoaderClass = await import("file:" + path.resolve(loader));
+        taxa = await taxaLoaderClass.TaxaLoader.loadTaxa(options, errorLog);
+    } else {
+        taxa = new Taxa(
+            Program.getIncludeList(options.datadir),
+            errorLog,
+            options.showFlowerErrors,
+        );
+    }
+
+    errorLog.write();
+    return taxa;
+}
+
+/**
+ * @returns {Map<string,InatPhotoInfo[]>}
+ */
+function readPhotos() {
+    const photosFileName = `./data/${PHOTO_FILE_NAME}`;
+    if (!existsSync(photosFileName)) {
+        return new Map();
+    }
+
+    /** @type {Map<string,{id:string,ext:string,licenseCode:string,attrName:string}[]>} */
+    const taxonPhotos = new Map();
+
+    /** @type {InatCsvPhoto[]} */
+    const csvPhotos = CSV.readFile(photosFileName);
+    for (const csvPhoto of csvPhotos) {
+        const taxonName = csvPhoto.name;
+        let photos = taxonPhotos.get(taxonName);
+        if (!photos) {
+            photos = [];
+            taxonPhotos.set(taxonName, photos);
+        }
+        photos.push({
+            id: csvPhoto.id.toString(),
+            ext: csvPhoto.ext,
+            licenseCode: csvPhoto.licenseCode,
+            attrName: csvPhoto.attrName,
+        });
+    }
+
+    return taxonPhotos;
+}
+
+const program = Program.getProgram();
+program
+    .command("checkmissing")
+    .description("List taxa with less than the maximum number of photos")
+    .option(
+        "--minphotos <number>",
+        "Minimum number of photos. Taxa with fewer than this number will be listed.",
+    )
+    .action((options) => checkmissing(program.opts(), options));
+if (process.env.npm_package_name === "@ca-plant-list/ca-plant-list") {
+    // Only allow updates in ca-plant-list.
+    program
+        .command("addmissing")
+        .description("Add photos to taxa with fewer than the maximum")
+        .action(() => addMissingPhotos(program.opts()));
+}
+program.option(
+    "--loader <path>",
+    "The path (relative to the current directory) of the JavaScript file containing the TaxaLoader class. If not provided, the default TaxaLoader will be used.",
+);
+await program.parseAsync();

package/tmp/config.json

@@ -0,0 +1,21 @@
+{
+    "calflora": {
+        "counties": [
+            "ALA",
+            "CCA"
+        ]
+    },
+    "inat": {
+        "project": "ebcnps"
+    },
+    "labels": {
+        "introduced": "Introduced to the East Bay",
+        "native": "Native to the East Bay",
+        "status-NC": "California native introduced to the East Bay"
+    },
+    "ebook": {
+        "filename": "ebplants",
+        "pub_id": "ebplants",
+        "title": "East Bay Plants"
+    }
+}

package/tmp/exceptions.json

@@ -0,0 +1,93 @@
+{
+    "Campanula exigua": {
+        "rpi": {
+            "translation-to-rpi": "Ravenella exigua"
+        }
+    },
+    "Campanula sharsmithiae": {
+        "rpi": {
+            "translation-to-rpi": "Ravenella sharsmithiae"
+        }
+    },
+    "Castilleja ambigua subsp. ambigua": {
+        "rpi": {
+            "translation-to-rpi": "Castilleja ambigua var. ambigua"
+        }
+    },
+    "Castilleja ambigua var. ambigua": {
+        "rpi": {
+            "translation": "Castilleja ambigua subsp. ambigua"
+        }
+    },
+    "Downingia ornatissima var. mirabilis": {
+        "inat": {
+            "notintaxondata": true
+        }
+    },
+    "Erysimum capitatum var. angustatum": {
+        "calflora": {
+            "badjepsonid": true
+        },
+        "jepson": {
+            "allowsynonym": true
+        }
+    },
+    "Heterotheca oregona var. rudis": {
+        "inat": {
+            "notintaxondata": true
+        }
+    },
+    "Horkelia californica var. frondosa": {
+        "inat": {
+            "notintaxondata": true
+        }
+    },
+    "Lupinus littoralis var. variicolor": {
+        "inat": {
+            "notintaxondata": true
+        }
+    },
+    "Malacothamnus hallii": {
+        "comment": "in CNPS Rare Plant Inventory",
+        "calflora": {
+            "notintaxondata": true
+        },
+        "jepson": {
+            "allowsynonym": true
+        },
+        "rpi": {
+            "translation": "Malacothamnus arcuatus var. elmeri"
+        }
+    },
+    "Malacothamnus arcuatus var. elmeri": {
+        "comment": "in CNPS Rare Plant Inventory as Malacothamnus hallii",
+        "rpi": {
+            "translation-to-rpi": "Malacothamnus hallii"
+        }
+    },
+    "Myosurus minimus subsp. apus": {
+        "comment": "in CNPS Rare Plant Inventory",
+        "jepson": {
+            "notineflora": true
+        }
+    },
+    "Ravenella exigua": {
+        "comment": "in CNPS Rare Plant Inventory as Ravenella",
+        "rpi": {
+            "translation": "Campanula exigua"
+        }
+    },
+    "Ravenella sharsmithiae": {
+        "rpi": {
+            "translation": "Campanula sharsmithiae"
+        }
+    },
+    "Streptanthus albidus subsp. peramoenus": {
+        "calflora": {
+            "notintaxondata": true
+        },
+        "jepson": {
+            "allowsynonym": true
+        }
+    }
+}