@ca-plant-list/ca-plant-list 0.4.29 → 0.4.30

package/data/taxa.csv

@@ -207,7 +207,8 @@ Balsamorhiza deltoidea,,N,1634,1047,70383,202414,true,Balsam Deltoid
 Balsamorhiza macrolepis,,N,1639,1051,69665,211633,true,California Balsamroot,,,,,350,1B.2,,,S2,G2
 Barbarea orthoceras,winter cress,N,15481,1057,52976,11356,true,American Wintercress,,yellow,3,7
 Bassia hyssopifolia,,X,15511,1060,58125,109432,true
-Bellardia trixago var. trixago,,X,8635,14706,1115038,416937,Bellardia trixago
+Bellardia trixago var. trixago,Mediterranean lineseed,X,8635,14706,1115038,416937,Bellardia trixago,,annual,"white,pink",4,8
+Bellardia viscosa,yellow glandweed,X,103729,14707,537967,206574,true,,annual,yellow,4,10
 Bellis perennis,English daisy,X,1652,1065,55563,202429,true
 Berberis aquifolium var. aquifolium,,N,71483,1068,126887,45198,Berberis aquifolium,Piper's Barberry
 Berberis aquifolium var. dictyota,,N,76967,1069,775436,45199,Berberis dictyota,Shining Netvein Barberry

package/lib/csv.js

@@ -5,7 +5,7 @@ import { parse as parseSync } from "csv-parse/sync";
 import { parse } from "csv-parse";
 import { stringify } from "csv-stringify/sync";
 
-class CSV {
+export class CSV {
     /**
      * @param {string} fileName
      * @param {import("csv-parse").ColumnOption[]|boolean|function (string[]):string[]} columns
@@ -95,10 +95,11 @@ class CSV {
     }
 
     /**
+     * @template T
      * @param {string} fileName
      * @param {boolean|import("csv-parse").ColumnOption[]|function (string[]):string[]} [columns]
      * @param {string} [delimiter]
-     * @returns {Object<string,string>[]}
+     * @returns {T[]}
      */
     static readFile(fileName, columns = true, delimiter) {
         const content = fs.readFileSync(fileName);
@@ -155,5 +156,3 @@ class CSV {
         fs.writeFileSync(fileName, content.replaceAll(/,+\n/g, "\n"));
     }
 }
-
-export { CSV };

package/lib/index.d.ts

@@ -82,10 +82,7 @@ export class Config {
 }
 
 export class CSV {
-    static readFile(
-        fileName: string,
-        delimeter?: string,
-    ): Record<string, string>[];
+    static readFile<T>(fileName: string, delimeter?: string): T[];
     static writeFileObject(
         fileName: string,
         data: Record<string, any>[],
@@ -129,6 +126,21 @@ export class Files {
 
 export class Genera {}
 
+export class GenericPage {
+    constructor(
+        siteGenerator: SiteGenerator,
+        title: string,
+        baseFileName: string,
+        js?: string,
+    );
+    getBaseFileName(): string;
+    getFrontMatter(): string;
+    getOutputDir(): string;
+    getSiteGenerator(): SiteGenerator;
+    getTitle(): string;
+    writeFile(content: string): void;
+}
+
 export class Genus<T extends Taxon> {
     getTaxa(): T[];
 }

package/lib/index.js

@@ -6,6 +6,7 @@ import { Exceptions } from "./exceptions.js";
 import { ExternalSites } from "./externalsites.js";
 import { Families } from "./taxonomy/families.js";
 import { Files } from "./files.js";
+import { GenericPage } from "./web/pageGeneric.js";
 import { HTML } from "./html.js";
 import { HTMLFragments } from "./utils/htmlFragments.js";
 import { HTMLTaxon } from "./htmltaxon.js";
@@ -24,6 +25,7 @@ export {
     ExternalSites,
     Families,
     Files,
+    GenericPage,
     HTML,
     HTMLFragments,
     HTMLTaxon,

package/lib/photo.js

@@ -60,6 +60,15 @@ export class Photo {
      * @returns {string} The URL to retrieve the image file.
      */
     getUrl() {
-        return `https://inaturalist-open-data.s3.amazonaws.com/photos/${this.#id}/medium.${this.#ext}`;
+        return Photo.getUrl(this.#id, this.#ext);
+    }
+
+    /**
+     * @param {number|string} id
+     * @param {string} ext
+     * @returns {string} The URL to retrieve the image file.
+     */
+    static getUrl(id, ext) {
+        return `https://inaturalist-open-data.s3.amazonaws.com/photos/${id}/medium.${ext}`;
     }
 }

package/lib/util.js

@@ -1,16 +1,19 @@
 /**
  * Break an array into chunks of a desired size
  * https://github.com/you-dont-need/You-Dont-Need-Lodash-Underscore?tab=readme-ov-file#_chunk
- * @param {any[]} input
+ * @template T
+ * @param {T[]} input
  * @param {number} size
- * @returns {any[][]}
+ * @returns {T[][]}
  */
 export function chunk(input, size) {
+    /** @type {T[][]} */
+    const result = [];
     return input.reduce((arr, item, idx) => {
         return idx % size === 0
             ? [...arr, [item]]
             : [...arr.slice(0, -1), [...arr.slice(-1)[0], item]];
-    }, []);
+    }, result);
 }
 
 /**

package/lib/utils/httpUtils.js

@@ -0,0 +1,10 @@
+export class HttpUtils {
+    /**
+     * @param {URL|string} url
+     * @returns {Promise<boolean>}
+     */
+    static async UrlExists(url) {
+        const response = await fetch(url, { method: "HEAD" });
+        return response.status === 200;
+    }
+}

package/lib/utils/inat-tools.js

@@ -41,11 +41,10 @@ import { chunk, sleep } from "../util.js";
 const ALLOWED_LICENSE_CODES = ["cc0", "cc-by", "cc-by-nc"];
 
 /**
- * @param {import("../types.js").Taxon[]} taxa
+ * @param {string[]} inatTaxonIDs
  * @return {Promise<InatApiTaxon[]>}
  */
-async function fetchInatTaxa(taxa) {
-    const inatTaxonIDs = taxa.map((taxon) => taxon.getINatID()).filter(Boolean);
+async function fetchInatTaxa(inatTaxonIDs) {
     const url = `https://api.inaturalist.org/v2/taxa/${inatTaxonIDs.join(",")}?fields=(taxon_photos:(photo:(medium_url:!t,attribution:!t,license_code:!t)))`;
     const resp = await fetch(url);
     if (!resp.ok) {
@@ -77,13 +76,11 @@ export async function getTaxonPhotos(taxaToUpdate) {
     let taxaRetrieved = 0;
 
     for (const batch of chunk(taxaToUpdate, 30)) {
-        const inatTaxa = await fetchInatTaxa(batch);
+        const inatTaxa = await fetchInatTaxa(batch.map((t) => t.getINatID()));
         for (const iNatTaxon of inatTaxa) {
-            const iNatTaxonPhotos = iNatTaxon.taxon_photos
-                .filter((tp) =>
-                    ALLOWED_LICENSE_CODES.includes(tp.photo.license_code),
-                )
-                .slice(0, 5);
+            const iNatTaxonPhotos = iNatTaxon.taxon_photos.filter((tp) =>
+                ALLOWED_LICENSE_CODES.includes(tp.photo.license_code),
+            );
 
             const taxonName = idMap.get(iNatTaxon.id.toString());
             if (!taxonName) {

package/lib/web/pageGeneric.js

@@ -27,8 +27,7 @@ export class GenericPage {
     }
 
     getDefaultIntro() {
-        let html = this.getFrontMatter();
-        return html + this.getMarkdown();
+        return this.getFrontMatter() + this.getMarkdown();
     }
 
     getFrontMatter() {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@ca-plant-list/ca-plant-list",
-    "version": "0.4.29",
+    "version": "0.4.30",
     "description": "Tools to create files for a website listing plants in an area of California.",
     "license": "MIT",
     "repository": {

package/scripts/cpl-photos.js

@@ -7,9 +7,12 @@ import { Taxa } from "../lib/taxonomy/taxa.js";
 import { getTaxonPhotos } from "../lib/utils/inat-tools.js";
 import { existsSync } from "fs";
 import { CSV } from "../lib/csv.js";
+import { HttpUtils } from "../lib/utils/httpUtils.js";
+import { ProgressMeter } from "../lib/progressmeter.js";
+import { Photo } from "../lib/photo.js";
 
-const PHOTO_FILE_NAME = "inattaxonphotos.csv";
-const PHOTO_FILE_PATH = `./data/${PHOTO_FILE_NAME}`;
+const OBS_PHOTO_FILE_NAME = "inatobsphotos.csv";
+const TAXON_PHOTO_FILE_NAME = "inattaxonphotos.csv";
 
 const OPT_LOADER = "loader";
 
@@ -32,7 +35,8 @@ async function addMissingPhotos(options) {
     }
 
     const newPhotos = await getTaxonPhotos(taxaMissingPhotos);
-    const currentTaxaPhotos = readPhotos();
+    const csvFilePath = getPhotoFilePath(TAXON_PHOTO_FILE_NAME, options);
+    const currentTaxaPhotos = readPhotos(csvFilePath);
 
     for (const [taxonName, photos] of newPhotos) {
         let currentPhotos = currentTaxaPhotos.get(taxonName);
@@ -59,14 +63,110 @@ async function addMissingPhotos(options) {
     errorLog.write();
 
     // Write updated photo file.
-    writePhotos(currentTaxaPhotos);
+    writePhotos(csvFilePath, currentTaxaPhotos);
+}
+
+/**
+ * @param {import("commander").OptionValues} options
+ */
+async function check(options) {
+    const csvFilePath = getPhotoFilePath(TAXON_PHOTO_FILE_NAME, options);
+    const taxa = await getTaxa(options);
+    const csvPhotos = readPhotos(csvFilePath);
+    const taxaPhotos = await getTaxonPhotos(taxa.getTaxonList());
+    const csvNames = Array.from(csvPhotos.keys());
+
+    const errorLog = new ErrorLog(options.outputdir + "/log.tsv", false);
+
+    const meter = new ProgressMeter("checking taxa photos", csvPhotos.size);
+    let errors = 0;
+    let counter = 0;
+
+    for (const name of csvNames) {
+        const taxon = taxa.getTaxon(name);
+        if (taxon) {
+            const csvTaxonPhotos = csvPhotos.get(name) ?? [];
+            const iNatTaxonPhotos = taxaPhotos.get(name) ?? [];
+
+            // Make sure each of the CSV photos is still referenced.
+            /** @type {string[]} */
+            const idsToDelete = [];
+            for (const csvPhoto of csvTaxonPhotos) {
+                const photoId = csvPhoto.id;
+                const iNatPhoto = iNatTaxonPhotos.find(
+                    (tp) => tp.id === photoId,
+                );
+                if (iNatPhoto) {
+                    /**
+                     * @param {"attrName"|"ext"|"licenseCode"} colName
+                     * @param {string|undefined} csvVal
+                     * @param {string|undefined} iNatVal
+                     */
+                    function checkCol(colName, csvVal, iNatVal) {
+                        iNatVal = iNatVal ?? "";
+                        if (csvVal !== iNatVal) {
+                            errors++;
+                            errorLog.log(
+                                name,
+                                `photo in CSV has different ${colName}`,
+                                photoId,
+                                csvVal,
+                                iNatVal,
+                            );
+                            if (options.update) {
+                                csvPhoto[colName] = iNatVal;
+                            }
+                        }
+                    }
+                    checkCol("attrName", csvPhoto.attrName, iNatPhoto.attrName);
+                    checkCol("ext", csvPhoto.ext, iNatPhoto.ext);
+                    checkCol(
+                        "licenseCode",
+                        csvPhoto.licenseCode,
+                        iNatPhoto.licenseCode,
+                    );
+                } else {
+                    if (options.update) {
+                        idsToDelete.push(photoId);
+                    }
+                    errors++;
+                    errorLog.log(
+                        name,
+                        `photo id ${photoId} not found in iNat taxon photos`,
+                    );
+                }
+            }
+
+            if (idsToDelete.length > 0) {
+                csvPhotos.set(
+                    name,
+                    csvTaxonPhotos.filter((p) => !idsToDelete.includes(p.id)),
+                );
+            }
+        } else {
+            errors++;
+            errorLog.log(name, "not found in taxa list");
+        }
+        counter++;
+        meter.update(counter, {
+            custom: ` | ${errors} errors | ${name}`,
+        });
+    }
+
+    meter.stop();
+
+    if (options.update) {
+        writePhotos(csvFilePath, csvPhotos);
+    }
+
+    errorLog.write();
 }
 
 /**
  * @param {import("commander").OptionValues} options
  * @param {import("commander").OptionValues} commandOptions
  */
-async function checkmissing(options, commandOptions) {
+async function checkmax(options, commandOptions) {
     const taxa = await getTaxa(options);
     const errorLog = new ErrorLog(options.outputdir + "/log.tsv", true);
 
@@ -85,6 +185,68 @@ async function checkmissing(options, commandOptions) {
     errorLog.write();
 }
 
+/**
+ * @param {import("commander").OptionValues} options
+ */
+async function checkUrl(options) {
+    checkUrlFile(TAXON_PHOTO_FILE_NAME, options);
+}
+
+/**
+ * @param {string} fileName
+ * @param {import("commander").OptionValues} options
+ */
+async function checkUrlFile(fileName, options) {
+    /**
+     * @param {string} name
+     * @param {import("../lib/utils/inat-tools.js").InatPhotoInfo[]} photoList
+     */
+    async function checkTaxon(name, photoList) {
+        const urls = photoList.map((p) =>
+            HttpUtils.UrlExists(Photo.getUrl(p.id, p.ext)),
+        );
+        const resolved = await Promise.all(urls);
+        for (let index = 0; index < resolved.length; index++) {
+            if (!resolved[index]) {
+                const id = photoList[index].id;
+                errorLog.log(name, id);
+                invalid.push({ name: name, id: id });
+            }
+        }
+    }
+
+    const invalid = [];
+
+    const csvFilePath = getPhotoFilePath(fileName, options);
+    const photos = readPhotos(csvFilePath);
+    const errorLog = new ErrorLog(options.outputdir + "/log.tsv", false);
+
+    const meter = new ProgressMeter("checking taxa URLs", photos.size);
+    let counter = 0;
+    const names = Array.from(photos.keys());
+
+    for (const name of names) {
+        const photoList = photos.get(name);
+        // @ts-ignore
+        await checkTaxon(name, photoList);
+        meter.update(++counter, {
+            custom: ` | ${invalid.length} errors | ${name}`,
+        });
+    }
+    meter.stop();
+
+    errorLog.write();
+}
+
+/**
+ * @param {string} fileName
+ * @param {import("commander").OptionValues} options
+ * @returns {string}
+ */
+function getPhotoFilePath(fileName, options) {
+    return path.join(".", options.datadir, fileName);
+}
+
 /**
  * @param {import("commander").OptionValues} options
  * @return {Promise<Taxa>}
@@ -113,16 +275,26 @@ async function getTaxa(options) {
  * @param {{outputdir:string,update:boolean}} options
  */
 async function prune(options) {
+    await pruneFile(TAXON_PHOTO_FILE_NAME, options);
+    await pruneFile(OBS_PHOTO_FILE_NAME, options);
+}
+
+/**
+ * @param {string} fileName
+ * @param {{outputdir:string,update:boolean}} options
+ */
+async function pruneFile(fileName, options) {
     const taxa = await getTaxa(options);
     const errorLog = new ErrorLog(options.outputdir + "/log.tsv", true);
-    const currentTaxaPhotos = readPhotos();
+    const csvFilePath = getPhotoFilePath(fileName, options);
+    const currentTaxaPhotos = readPhotos(csvFilePath);
 
     const invalidNames = new Set();
 
     for (const name of currentTaxaPhotos.keys()) {
         const taxon = taxa.getTaxon(name);
         if (!taxon) {
-            errorLog.log(name, `is in ${PHOTO_FILE_NAME} but not in taxa list`);
+            errorLog.log(name, `is in ${csvFilePath} but not in taxa list`);
             invalidNames.add(name);
         }
     }
@@ -131,17 +303,18 @@ async function prune(options) {
         for (const name of invalidNames) {
             currentTaxaPhotos.delete(name);
         }
-        writePhotos(currentTaxaPhotos);
+        writePhotos(csvFilePath, currentTaxaPhotos);
     }
 
     errorLog.write();
 }
 
 /**
+ * @param {string} csvFilePath
  * @returns {Map<string,import("../lib/utils/inat-tools.js").InatPhotoInfo[]>}
  */
-function readPhotos() {
-    if (!existsSync(PHOTO_FILE_PATH)) {
+function readPhotos(csvFilePath) {
+    if (!existsSync(csvFilePath)) {
         return new Map();
     }
 
@@ -150,7 +323,7 @@ function readPhotos() {
 
     /** @type {import("../lib/utils/inat-tools.js").InatCsvPhoto[]} */
     // @ts-ignore
-    const csvPhotos = CSV.readFile(PHOTO_FILE_PATH);
+    const csvPhotos = CSV.readFile(csvFilePath);
     for (const csvPhoto of csvPhotos) {
         const taxonName = csvPhoto.name;
         let photos = taxonPhotos.get(taxonName);
@@ -170,16 +343,17 @@ function readPhotos() {
 }
 
 /**
- * @param {Map<string,import("../lib/utils/inat-tools.js").InatPhotoInfo[]>} currentTaxaPhotos
+ * @param {string} filePath
+ * @param {Map<string,import("../lib/utils/inat-tools.js").InatPhotoInfo[]>} currentPhotos
  */
-function writePhotos(currentTaxaPhotos) {
+function writePhotos(filePath, currentPhotos) {
     // Write updated photo file.
     const headers = ["name", "id", "ext", "licenseCode", "attrName"];
     /** @type {string[][]} */
     const data = [];
-    for (const taxonName of [...currentTaxaPhotos.keys()].sort()) {
+    for (const taxonName of [...currentPhotos.keys()].sort()) {
         // @ts-ignore - should always be defined at this point
-        for (const photo of currentTaxaPhotos.get(taxonName)) {
+        for (const photo of currentPhotos.get(taxonName)) {
             data.push([
                 taxonName,
                 photo.id,
@@ -190,20 +364,28 @@ function writePhotos(currentTaxaPhotos) {
         }
     }
 
-    CSV.writeFileArray(PHOTO_FILE_PATH, data, headers);
+    CSV.writeFileArray(filePath, data, headers);
 }
 
 const program = Program.getProgram();
 program
-    .command("checkmissing")
+    .command("checkmax")
     .description("List taxa with less than the maximum number of photos")
     .option(
         "--minphotos <number>",
         "Minimum number of photos. Taxa with fewer than this number will be listed.",
     )
-    .action((options) => checkmissing(program.opts(), options));
+    .action((options) => checkmax(program.opts(), options));
+program
+    .command("checkurl")
+    .description("Make sure URLs are valid")
+    .action(() => checkUrl(program.opts()));
 if (process.env.npm_package_name === "@ca-plant-list/ca-plant-list") {
     // Only allow updates in ca-plant-list.
+    program
+        .command("check")
+        .description("Check taxa photos to ensure information is current.")
+        .action(() => check(program.opts()));
     program
         .command("addmissing")
         .description("Add photos to taxa with fewer than the maximum")