imerss-bioinfo 1.18.0 → 1.19.0

package/docs/css/data-table.css

@@ -61,6 +61,10 @@
     min-height: 0;
     overflow-x: scroll;
     overflow-y: auto;
+    /* Only the rows near those in view are rendered, between spacers. The browser's scroll anchoring would otherwise
+       move the scrolled position each time a new block of rows replaces the old, so that scrolling runs away from the
+       pointer */
+    overflow-anchor: none;
 }
 
 .hortis-dataTable-scroll table {

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "imerss-bioinfo",
     "description": "Working data of the IMERSS biodiversity informatics working group, together with algorithms",
-    "version": "1.18.0",
+    "version": "1.19.0",
     "author": "Antranig Basman",
     "scripts": {
         "lint": "fluid-lint-all",

package/src/assignBNames.js

@@ -4,6 +4,9 @@
 
 const fluid = require("infusion");
 const minimist = require("minimist");
+const JSON5 = require("json5");
+const fs = require("fs");
+const path = require("path");
 
 fluid.require("%imerss-bioinfo");
 
@@ -142,23 +145,85 @@ const strategies = {
     }
 };
 
-const parsedArgs = minimist(process.argv.slice(2), {boolean: [...Object.keys(strategies), "noRank"]});
+// Strategy fields which may be overridden from the command line or a driver file
+const overridableFields = ["phylum", "phylumField", "rank", "rankField", "rawName", "iNatName"];
 
-const swapsFile = parsedArgs.swaps || fluid.module.resolvePath("%imerss-bioinfo/data/b-team/taxon-swaps-2025.csv");
+// Paths held in a driver file, which are stored relative to the driver file's own directory
+const driverPaths = ["input", "swaps", "output", "taxa", "mismatches"];
 
-const noRank = parsedArgs.noRank;
+/**
+ * Load a driver file, resolving any paths it holds against the driver file's directory.
+ * @param {String} driverFile - Path to a JSON5 driver file
+ * @return {Object} The driver record with its paths made absolute
+ */
+hortis.readDriver = function (driverFile) {
+    const driver = JSON5.parse(fs.readFileSync(driverFile, "utf8"));
+    const driverDir = path.dirname(path.resolve(driverFile));
+    driverPaths.forEach(function (key) {
+        if (driver[key] !== undefined) {
+            driver[key] = path.resolve(driverDir, fluid.module.resolvePath(driver[key]));
+        }
+    });
+    return driver;
+};
+
+/**
+ * Write a driver file which will reproduce this run, with paths relative to the driver file's directory.
+ * @param {String} driverFile - Path at which to write the JSON5 driver file
+ * @param {Object} driver - The fully resolved driver record for this run
+ */
+hortis.writeDriver = function (driverFile, driver) {
+    const driverDir = path.dirname(path.resolve(driverFile));
+    const togo = fluid.copy(driver);
+    driverPaths.forEach(function (key) {
+        if (togo[key] !== undefined) {
+            togo[key] = path.relative(driverDir, path.resolve(fluid.module.resolvePath(togo[key]))).replace(/\\/g, "/");
+        }
+    });
+    fs.writeFileSync(driverFile, JSON5.stringify(togo, null, 4) + "\n");
+    console.log("Written driver file " + driverFile);
+};
+
+const parsedArgs = minimist(process.argv.slice(2), {boolean: [...Object.keys(strategies), "noRank"], string: overridableFields});
+
+const fileDriver = parsedArgs.driver ? hortis.readDriver(parsedArgs.driver) : {};
+
+const argStrategy = parsedArgs.strategy || Object.keys(strategies).find(strategy => parsedArgs[strategy]);
+
+const argOverrides = {};
+overridableFields.forEach(function (key) {
+    if (parsedArgs[key] !== undefined) {
+        argOverrides[key] = parsedArgs[key];
+    }
+});
+
+// Values supplied on the command line take priority over those in any driver file
+const driver = {
+    strategy: argStrategy || fileDriver.strategy,
+    input: parsedArgs._[0] || fileDriver.input || fluid.module.resolvePath("%imerss-bioinfo/data/b-team/plant-pollinators-Carril-normalised.csv"),
+    swaps: parsedArgs.swaps || fileDriver.swaps || fluid.module.resolvePath("%imerss-bioinfo/data/b-team/taxon-swaps-2025.csv"),
+    noRank: parsedArgs.noRank || fileDriver.noRank || false,
+    // Default to DwC column names for phylum and rank - other layouts supply these via e.g. --phylumField, --rank
+    overrides: {phylumField: "phylum", rankField: "taxonRank", ...fileDriver.overrides, ...argOverrides}
+};
 
-const inputFile = parsedArgs._[0] || fluid.module.resolvePath("%imerss-bioinfo/data/b-team/plant-pollinators-Carril-normalised.csv");
+if (!driver.strategy || !strategies[driver.strategy]) {
+    fluid.fail("Please supply strategy argument via --strategy=<name> or one of ", Object.keys(strategies).map(key => "--" + key).join(", "));
+}
+
+const inputTrunk = hortis.inputFileToTrunk(driver.input);
 
-const inputTrunk = hortis.inputFileToTrunk(inputFile);
+driver.output = parsedArgs.o || fileDriver.output || inputTrunk + "-assigned.csv";
+driver.taxa = parsedArgs.taxa || fileDriver.taxa || inputTrunk + "-assigned-taxa.csv";
+driver.mismatches = parsedArgs.mismatches || fileDriver.mismatches || inputTrunk + "-mismatches.csv";
 
-const outputFile = parsedArgs.o || inputTrunk + "-assigned.csv";
-const outputTaxaFile = parsedArgs.taxa || inputTrunk + "-assigned-taxa.csv";
+const driverOutFile = parsedArgs.driverOut || inputTrunk + "-driver.json5";
 
-const mismatchFile = parsedArgs.mismatches || inputTrunk + "-mismatches.csv";
+const strategy = driver.strategy;
+const noRank = driver.noRank;
 
 const swapsReader = hortis.csvReaderWithMap({
-    inputFile: swapsFile,
+    inputFile: driver.swaps,
     mapColumns: {
         "preferred": "Preferred name",
         "supplied": "Supplied name",
@@ -170,22 +235,25 @@ const source = hortis.iNatTaxonSource({
 //     disableNameCache: true
 });
 
-
-const strategy = Object.keys(strategies).find(strategy => parsedArgs[strategy]);
-
-if (!strategy) {
-    fluid.fail("Please supply strategy argument via one of ", Object.keys(strategies).map(key => "--" + key).join(", "));
-}
-
 console.log("Applying strategy ", strategy);
-const strategyBigRec = strategies[strategy];
-
+const strategyBigRec = strategy === "bees" ? strategies.bees : {...strategies[strategy], ...driver.overrides};
 
 const reader = hortis.csvReaderWithoutMap({
-    inputFile: inputFile,
+    inputFile: driver.input,
     csvOptions: strategyBigRec.csvOptions
 });
 
+/**
+ * Determine a value such as phylum or rank for a row, taking it from the configured field if present,
+ * falling back to any configured default.
+ * @param {Object} row - The row being processed
+ * @param {String} [field] - The name of the field holding the value
+ * @param {String} [defaultValue] - Value to use if the field is absent or empty
+ * @return {String} The resolved value, or undefined
+ */
+hortis.rowValue = function (row, field, defaultValue) {
+    return (field && row[field]) || defaultValue;
+};
 
 // Added in Epifamily so we can include Bees [Epifamily Anthophila]
 const storeRanks = ["stateofmatter", "kingdom", "phylum", "subphylum", "class", "order", "epifamily", "family", "genus"];
@@ -327,23 +395,19 @@ Promise.all([reader.completionPromise, swapsReader.completionPromise, source.eve
         if (strategy === "bees") {
             await hortis.applyName(row, i, "Tracheophyta", null, invertedSwaps, taxa, unmappedTaxa, strategyBigRec.plant);
             await hortis.applyName(row, i, "Arthropoda", null, invertedSwaps, taxa, unmappedTaxa, strategyBigRec.pollinator);
-        } else if (strategy === "reintegrated") {
-            await hortis.applyName(row, i, row.Phylum, row.Rank, invertedSwaps, taxa, unmappedTaxa, strategyBigRec);
-        } else if (strategy.startsWith("DwC")) {
-            await hortis.applyName(row, i, row.phylum, row.taxonRank, invertedSwaps, taxa, unmappedTaxa, strategyBigRec);
-        } else if (strategy === "iNat") {
-            await hortis.applyName(row, i, row.phylum, row.taxon_rank, invertedSwaps, taxa, unmappedTaxa, strategyBigRec);
-        } else if (strategy === "Saanich") {
-            await hortis.applyName(row, i, "Tracheophyta", "species", invertedSwaps, taxa, unmappedTaxa, strategyBigRec);
+        } else {
+            const s = strategyBigRec;
+            await hortis.applyName(row, i, hortis.rowValue(row, s.phylumField, s.phylum), hortis.rowValue(row, s.rankField, s.rank),
+                invertedSwaps, taxa, unmappedTaxa, s);
         }
 
         mapped.push(row);
     }
-    const resolveOut = fluid.module.resolvePath(outputFile);
+    const resolveOut = fluid.module.resolvePath(driver.output);
     await hortis.writeCSV(resolveOut, Object.keys(mapped[0]), mapped, fluid.promise());
 
     const taxaRows = hortis.flattenTaxa(taxa);
-    await hortis.writeCSV(outputTaxaFile, Object.keys(taxaRows[0]), taxaRows, fluid.promise());
+    await hortis.writeCSV(driver.taxa, Object.keys(taxaRows[0]), taxaRows, fluid.promise());
 
     const unmapped = Object.keys(unmappedTaxa);
     if (unmapped.length > 0) {
@@ -353,7 +417,7 @@ Promise.all([reader.completionPromise, swapsReader.completionPromise, source.eve
             console.log(key + ", original name " + scientificName);
             return {taxonName: key, originalName: scientificName};
         });
-        await hortis.writeCSV(mismatchFile, Object.keys(unmappedRows[0]), unmappedRows, fluid.promise());
+        await hortis.writeCSV(driver.mismatches, Object.keys(unmappedRows[0]), unmappedRows, fluid.promise());
     }
-
+    hortis.writeDriver(driverOutFile, driver);
 });

package/src/client/css/data-table.css

@@ -61,6 +61,10 @@
     min-height: 0;
     overflow-x: scroll;
     overflow-y: auto;
+    /* Only the rows near those in view are rendered, between spacers. The browser's scroll anchoring would otherwise
+       move the scrolled position each time a new block of rows replaces the old, so that scrolling runs away from the
+       pointer */
+    overflow-anchor: none;
 }
 
 .hortis-dataTable-scroll table {

package/src/client/sfc/components/hortis-dataTable.vue

@@ -30,6 +30,10 @@ fluid.def("hortis.dataTable", {
     scrollTop: 0,
     viewHeight: 0,
     rowHeight: 26.5,
+    // The least change in the measured height of a row for which `rowHeight` is updated. Rows differ in height by a
+    // fraction of a pixel, by their content and the display's scaling, and since every row not rendered is reckoned at
+    // `rowHeight`, following each such difference would change the height of a long table by thousands of pixels
+    rowHeightTolerance: 0.5,
     resolvedColumns: "$compute:hortis.dataTable.resolveColumns({self}.rows, {self}.columns)",
     displayRows: "$compute:hortis.dataTable.sortRows({self}.rows, {self}.sortColumn, {self}.sortDirection)",
     window: "$compute:hortis.dataTable.window({self}.displayRows.length, {self}.scrollTop, {self}.viewHeight, {self}.rowHeight, {self}.minBlockSize)",
@@ -178,10 +182,28 @@ hortis.dataTable.pageInfo = function (window) {
     return window.total > 0 ? `${window.first + 1}–${window.last} of ${window.total}` : "No data";
 };
 
+/**
+ * Measure the height of the rows rendered in a table's scrolled element, as their mean, since they may differ by a
+ * fraction of a pixel.
+ * @param {Element} scrollEl - The table's scrolled element
+ * @return {Number} The mean height of the rendered rows, or 0 if none is rendered
+ */
+hortis.dataTable.measureRowHeight = function (scrollEl) {
+    const rows = scrollEl.querySelectorAll("tbody tr:not(.hortis-dataTable-spacer)");
+    const first = rows[0], last = rows[rows.length - 1];
+    return rows.length > 0 ? (last.getBoundingClientRect().bottom - first.getBoundingClientRect().top) / rows.length : 0;
+};
+
 /**
  * Track the scrolled position and the height of the view of a table's scrolled element, writing them into the table
  * on each animation frame in which they change, and the height of its head into the element's CSS variable
  * --data-table-head-height.
+ * The table's `rowHeight`, by which the heights of the spacers standing for the rows not rendered are reckoned, is
+ * corrected only when the rows' measured height departs from it by more than `rowHeightTolerance`, and never whilst
+ * the pointer is pressed within the scrolled element. A change of `rowHeight` changes the height of the whole table,
+ * and whilst the scrollbar's thumb is dragged, the browser holds the thumb at the pointer by moving the scrolled
+ * position in proportion - which rendered other rows, measured afresh, in a loop which ran the table to its end. Once
+ * `rowHeight` is corrected, the scrolled position is moved so that the same row stays at the top of the view.
  * @param {hortis.dataTable} that - The table
  * @param {Element} container - The table's rendered container
  * @param {Window} window - The window holding the table
@@ -190,32 +212,50 @@ hortis.dataTable.pageInfo = function (window) {
 hortis.dataTable.track = function (that, container, window) {
     const scrollEl = container.querySelector(".hortis-dataTable-scroll");
     let frame = null;
+    // Whether the pointer is pressed within the scrolled element, e.g. dragging its scrollbar's thumb
+    let pressed = false;
     const update = () => {
         frame = null;
         const theadHeight = scrollEl.querySelector("thead")?.getBoundingClientRect().height || 0;
-        const row = scrollEl.querySelector("tbody tr:not(.hortis-dataTable-spacer)");
-        const rowHeight = row?.getBoundingClientRect().height;
         // Confines the vertical scrollbar's track to the rows, beneath the sticky head - see data-table.css
         scrollEl.style.setProperty("--data-table-head-height", theadHeight + "px");
-        that.scrollTop = scrollEl.scrollTop;
-        that.viewHeight = Math.max(scrollEl.clientHeight - theadHeight, 0);
-        if (rowHeight > 0 && Math.abs(rowHeight - that.rowHeight) > 0.01) {
+        const rowHeight = hortis.dataTable.measureRowHeight(scrollEl);
+        if (!pressed && rowHeight > 0 && Math.abs(rowHeight - that.rowHeight) > that.rowHeightTolerance) {
+            const topRow = scrollEl.scrollTop / that.rowHeight;
             that.rowHeight = rowHeight;
+            scrollEl.scrollTop = topRow * rowHeight;
         }
+        that.scrollTop = scrollEl.scrollTop;
+        that.viewHeight = Math.max(scrollEl.clientHeight - theadHeight, 0);
     };
     const schedule = () => {
         if (!frame) {
             frame = window.requestAnimationFrame(update);
         }
     };
+    const press = () => {
+        pressed = true;
+    };
+    const release = () => {
+        if (pressed) {
+            pressed = false;
+            schedule();
+        }
+    };
     scrollEl.addEventListener("scroll", schedule);
+    scrollEl.addEventListener("pointerdown", press);
+    window.addEventListener("pointerup", release);
+    window.addEventListener("pointercancel", release);
     const observer = new window.ResizeObserver(schedule);
     observer.observe(scrollEl);
-    return {scrollEl, schedule, observer, window, cancel: () => frame && window.cancelAnimationFrame(frame)};
+    return {scrollEl, schedule, press, release, observer, window, cancel: () => frame && window.cancelAnimationFrame(frame)};
 };
 
 hortis.dataTable.untrack = function (record) {
     record.scrollEl.removeEventListener("scroll", record.schedule);
+    record.scrollEl.removeEventListener("pointerdown", record.press);
+    record.window.removeEventListener("pointerup", record.release);
+    record.window.removeEventListener("pointercancel", record.release);
     record.observer.disconnect();
     record.cancel();
 };

package/src/dataProcessing/sqlite.js

@@ -8,6 +8,8 @@ const hortis = fluid.registerNamespace("hortis");
 const sqlite = require("sqlite");
 const sqlite3 = require("sqlite3");
 const pako = require("pako");
+const fs = require("fs");
+const nodePath = require("path");
 
 fluid.registerNamespace("hortis.codecs.zlib");
 
@@ -60,6 +62,8 @@ hortis.sqliteDB.openDB = async function (dbFile) {
     const path = fluid.module.resolvePath(dbFile);
     try {
         fluid.log("Opening SQLite database at ", path);
+        // Ensure the database's directory exists, since SQLite will create the file but not its parents
+        fs.mkdirSync(nodePath.dirname(path), {recursive: true});
         return sqlite.open({
             filename: path,
             driver: sqlite3.Database

package/src/dataProcessing/writeCSV.js

@@ -7,8 +7,8 @@ const csvWriterModule = require("csv-writer");
 
 const hortis = fluid.registerNamespace("hortis");
 
-require("./readCSVWithmap.js");
-require("./readCSVWithoutMap.js");
+require("./readCSVwithMap.js");
+require("./readCSVwithoutMap.js");
 require("../utils/utils.js");
 
 hortis.writeCSV = function (fileName, headersOrMap, rows, completionPromise) {

package/src/iNaturalist/obsAPI.js

@@ -155,3 +155,95 @@ fluid.defaults("hortis.iNatObsAPI.byIdDBSource", {
         query: "SELECT id from iNatObsId"
     }
 });
+
+// A cached source for pages of the observations search API, so that an interrupted run of inatObs.js can be resumed
+// rapidly. The cache is short-lived since observations change much more rapidly than taxa.
+// Accepts the same directModel as hortis.iNat.obsSource, e.g. {per_page, id_above}, and resolves to a record
+// {key, fetched_at, doc} where doc is the API response.
+// Expected to be instantiated as a child of hortis.iNatTaxonSource in order to share its database and JWT.
+fluid.defaults("hortis.cachediNatObsPage", {
+    gradeNames: "hortis.cachedApiSource",
+    refreshInDays: 3,
+    // Pages are large and each is requested only once per run, so holding them in memory would exhaust the heap
+    useInMemoryCache: false,
+    // paramMap: Object - search parameters for the observations query, merged with those in hortis.iNat.obsSource
+    components: {
+        apiSource: {
+            type: "hortis.iNat.obsSource",
+            options: {
+                gradeNames: "hortis.withINatRateLimit",
+                paramMap: "{cachediNatObsPage}.options.paramMap"
+            }
+        },
+        dbSource: {
+            type: "hortis.iNatObsAPI.pageDBSource",
+            options: {
+                invokers: {
+                    encodeQueryKey: "hortis.cachediNatObsPage.encodeQueryKey({arguments}.0, {cachediNatObsPage}.apiSource)"
+                }
+            }
+        }
+    },
+    invokers: {
+        upgradeLiveDocument: "hortis.cachediNatObsPage.upgradeLiveDocument({arguments}.0, {arguments}.1, {arguments}.2, {that}.apiSource)"
+    }
+});
+
+/** Compute the database key for an observations page query - this is the full resolved API URL
+ * @param {Object} query - The directModel for the query, or a record already holding its `key`
+ * @param {Component} apiSource - The hortis.iNat.obsSource issuing the requests
+ * @return {Object} The query, supplemented with member `key`
+ */
+hortis.cachediNatObsPage.encodeQueryKey = function (query, apiSource) {
+    let key = query.key;
+    if (!key) {
+        key = apiSource.resolveUrl(apiSource.options.url, apiSource.options.termMap, query);
+    }
+    return {
+        ...query,
+        key
+    };
+};
+
+/** Convert a live observations page response into the record form stored in the database
+ * @param {Object} query - The directModel for the query
+ * @param {Object} live - The response from the observations API
+ * @param {Function} writer - Function accepting (query, record) which writes the record to the caches
+ * @param {Component} apiSource - The hortis.iNat.obsSource issuing the requests
+ * @return {Promise<Object>} A promise for the record {key, fetched_at, doc}
+ */
+hortis.cachediNatObsPage.upgradeLiveDocument = async function (query, live, writer, apiSource) {
+    const toWrite = {
+        key: hortis.cachediNatObsPage.encodeQueryKey(query, apiSource).key,
+        fetched_at: new Date().toISOString(),
+        doc: live
+    };
+    // Don't cache the final empty page, so that a rerun will always pick up observations added since
+    if (live && live.results && live.results.length > 0) {
+        await writer(query, toWrite);
+    }
+    return toWrite;
+};
+
+fluid.defaults("hortis.iNatObsAPI.pageDBSource", {
+    gradeNames: ["hortis.listableSqliteSource"],
+    createString: "CREATE TABLE IF NOT EXISTS iNatObsPage (key TEXT PRIMARY KEY, fetched_at TEXT, doc BLOB)",
+    columnCodecs: {
+        doc: "zlib"
+    },
+    readQuery: {
+        query: "SELECT key, fetched_at, doc from iNatObsPage WHERE key = ?",
+        args: ["%key"]
+    },
+    writeQuery: {
+        query: "INSERT OR REPLACE INTO iNatObsPage (key, fetched_at, doc) VALUES ($key, $fetched_at, $doc)",
+        args: {
+            $key: "%key",
+            $fetched_at: "%fetched_at",
+            $doc: "%doc"
+        }
+    },
+    listQuery: {
+        query: "SELECT key from iNatObsPage"
+    }
+});

package/src/iNaturalist/taxonAPI.js

@@ -158,8 +158,14 @@ hortis.DAY_IN_MS = 24 * 60 * 60 * 1000;
 
 fluid.defaults("hortis.cachedApiSource", {
     gradeNames: ["fluid.dataSource", "fluid.dataSource.noencoding"],
-    refreshInDays: 180,
+    refreshInDays: 365,
     disableCache: false,
+    // Set to false for sources whose documents are large and never requested twice in a run, e.g. observation pages
+    useInMemoryCache: true,
+    retry: {
+        attempts: 6,
+        initialDelay: 5000 // doubled after each failed attempt
+    },
     components: {
         inMemorySource: {
             type: "fluid.inMemoryCachedSource"
@@ -179,6 +185,40 @@ fluid.defaults("hortis.cachedApiSource", {
     }
 });
 
+/** Issue a get against a dataSource, retrying with exponential backoff on failure (e.g. transient DNS or network
+ * errors such as EAI_AGAIN or ECONNRESET)
+ * @param {fluid.dataSource} source - The dataSource to read from
+ * @param {Object} query - The directModel for the request
+ * @param {Object} retry - Retry policy, holding members `attempts` and `initialDelay` (in ms)
+ * @return {Promise<any>} A promise for the dataSource's response, rejecting with the final error if all attempts fail
+ */
+hortis.getWithRetry = async function (source, query, retry) {
+    const attempts = retry?.attempts || 1;
+    let delay = retry?.initialDelay || 5000;
+    let result, lastError;
+    let succeeded = false;
+    for (let attempt = 1; attempt <= attempts && !succeeded; ++attempt) {
+        try {
+            result = await source.get(query);
+            succeeded = true;
+        } catch (e) {
+            lastError = e;
+            if (attempt < attempts) {
+                console.log("Got API error on attempt " + attempt + " of " + attempts + ", retrying in " + delay + "ms: ", e);
+                await new Promise(resolve => setTimeout(resolve, delay));
+                delay *= 2;
+            } else {
+                console.log("Got API error on final attempt " + attempt + ", giving up: ", e);
+            }
+        }
+    }
+    if (succeeded) {
+        return result;
+    } else {
+        throw lastError;
+    }
+};
+
 hortis.readNoValue = function (value) {
     return value === fluid.NO_VALUE ? undefined : value;
 };
@@ -189,7 +229,8 @@ hortis.writeNoValue = function (value) {
 
 hortis.cachedApiSource.read = async function (that, payload, options) {
     const query = options.directModel;
-    const firstLevel = await that.inMemorySource.get(query);
+    const useInMemory = that.options.useInMemoryCache;
+    const firstLevel = useInMemory ? await that.inMemorySource.get(query) : undefined;
     if (firstLevel) {
         return hortis.readNoValue(firstLevel);
     } else {
@@ -197,27 +238,23 @@ hortis.cachedApiSource.read = async function (that, payload, options) {
         if (cached && !that.options.disableCache) {
             const staleness = Date.now() - Date.parse(cached.fetched_at);
             if (staleness < that.options.refreshInDays * hortis.DAY_IN_MS) {
-                await that.inMemorySource.set(query, cached);
+                if (useInMemory) {
+                    await that.inMemorySource.set(query, cached);
+                }
                 return cached;
             } else {
                 console.log("Staleness is " + moment.duration(staleness).humanize() + " so fetching live document");
             }
         }
-        let live;
-        try {
-            live = await that.apiSource.get(query);
-        } catch (e) {
-            console.log("Got API error ", e);
-            await new Promise(resolve => setTimeout(resolve, 5000));
-            debugger;
-            live = await that.apiSource.get(query);
-        }
+        const live = await hortis.getWithRetry(that.apiSource, query, that.options.retry);
         const writer = async function (query, toWrite) {
             if (toWrite.doc) { // Only write to db if we got live
                 await that.dbSource.set(query, toWrite);
             }
-            // Always write to top-level cache
-            await that.inMemorySource.set(query, hortis.writeNoValue(toWrite));
+            // Always write to top-level cache if enabled
+            if (useInMemory) {
+                await that.inMemorySource.set(query, hortis.writeNoValue(toWrite));
+            }
         };
         const toWrite = await that.upgradeLiveDocument(query, live, writer);
 
@@ -340,7 +377,7 @@ hortis.genusMatch = function (a, b) {
 
     // Return everything before the whitespace
     return j > 0 ? a.slice(0, j - 1) : "";
-}
+};
 
 // Highest iNat rank is 100 for Life - we want the most specific taxon of those selected
 hortis.scoreNameMatch = async function (result, query, byIdSource) {

package/src/inatObs.js

@@ -190,17 +190,23 @@ const projectArgs = hortis.iNatProjects[parsedArgs._[0] || "GalianoData"];
 
 const jwt = hortis.readJSONSync("jwt.json", "reading JWT token file");
 
-const source = hortis.iNat.obsSource({
-    headers: {
-        Authorization: "Bearer " + jwt.api_token
-    },
-    paramMap: projectArgs.paramMap
-});
-
+// Observation pages are fetched via a short-lived cache held in the same database as taxa, so that an interrupted run
+// can be resumed quickly. Pass --disableObsCache to force all pages to be fetched live.
 const iNatTaxonSource = hortis.iNatTaxonSource({
-    jwt: jwt
+    jwt: jwt,
+    components: {
+        obsPages: {
+            type: "hortis.cachediNatObsPage",
+            options: {
+                paramMap: projectArgs.paramMap,
+                refreshInDays: parsedArgs.disableObsCache ? 0 : 3
+            }
+        }
+    }
 });
 
+const source = iNatTaxonSource.obsPages;
+
 const fileVars = {
     Date: new Date().toISOString().substring(0, 10).replaceAll("-", "_")
 };
@@ -234,9 +240,8 @@ hortis.applyResponse = async function (data) {
         const lastId = fluid.peek(rows).id;
         console.log("got last id " + lastId);
         directModel.id_above = lastId;
-        setTimeout(function () {
-            hortis.makeObsRequest(directModel);
-        }, 1000);
+        // Live requests are rate-limited by hortis.iNatAPILimiter, cached pages are served immediately
+        hortis.makeObsRequest(directModel);
     } else {
         hortis.writeObs("obsoutput.csv", rows).then(function () {
             const fileTarget = fluid.stringTemplate(projectArgs.outputFile, fileVars);
@@ -249,12 +254,13 @@ hortis.applyResponse = async function (data) {
 
 hortis.makeObsRequest = function (directModel) {
     console.log("Making request ", directModel);
-    const promise = source.get(directModel);
+    const promise = source.get({...directModel});
 
-    promise.then(async function (data) {
-        await hortis.applyResponse(data);
+    promise.then(async function (record) {
+        await hortis.applyResponse(record.doc);
     }, function (error) {
-        console.log("Got ERROUR: ", error);
+        console.log("Failed to fetch observations after retries, rerun to resume from cache: ", error);
+        process.exitCode = 1;
     });
 };