frugal-iot-server 0.3.4 → 0.3.6

package/frugal-iot-server.js

@@ -30,6 +30,7 @@
  *Δ /admin (get) dashboard for administrators - includes OTA and will include permission management
  P /ota_update (post) protected place to upload new binaries (OTAUPDATE)
  XP /ota_list/:org list all ota files for an organization
+ XP /ota_get/:org/*remainingpath download a binary so the client can flash it over USB
  O  /private Serve up private files under authentication - currently unused
  * /register (post) register a new user
  */
@@ -111,6 +112,8 @@ import { createPushManager } from './lib/farm-platform-push.js';
 import { APIError } from './lib/api-errors.js';
 
 import { access, constants, createReadStream, mkdir, readdir, readFile, rm } from 'fs'; // https://nodejs.org/api/fs.html
+import { createGunzip } from 'zlib'; // https://nodejs.org/api/zlib.html - for serving compressed readings
+import { startHousekeeping } from './lib/housekeeping.js';
 import { detectSeries } from 'async'; // https://caolan.github.io/async/v3/docs.html
 import { createMD5 } from 'hash-wasm';
 import multer from 'multer'; // https://www.npmjs.com/package/multer
@@ -678,11 +681,20 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
     // each organization's mqtt_password, which should not be going to the console and the journal.
     console.log("Broker", config.mqtt.broker, "- organizations:", Object.keys(config.organizations).join(", ") || "(none)");
     // Could genericize config defaults
-    if (!config.morgan) {
-      config.morgan = ':method :url :req[range] :status :res[content-length] :response-time ms :req[referer]'
+    // HTTP request logging. "morgan: false" in config.d/server.yaml turns it off completely, which
+    // matters on a machine running from an SD card: every request logged is a line to the journal,
+    // and so a write that wears the card. Absent means log, so an existing server is unaffected by
+    // this setting arriving. The older top-level config.yaml setting still works if anyone set it.
+    let morganFormat = (config.server.morgan !== undefined) ? config.server.morgan : config.morgan;
+    if (morganFormat === false) {
+      console.log("Not logging HTTP requests (morgan: false in config.d/server.yaml)");
+    } else {
+      if (!morganFormat || (morganFormat === true)) { // Unset, or "morgan: true" meaning just turn it on
+        morganFormat = ':method :url :req[range] :status :res[content-length] :response-time ms :req[referer]'
+      }
+      // Seems to be writing to syslog which is being cycled.
+      app.use(morgan(morganFormat)); // see https://www.npmjs.com/package/morgan )
     }
-    // Seems to be writing to syslog which is being cycled.
-    app.use(morgan(config.morgan)); // see https://www.npmjs.com/package/morgan )
 
     if (config.server.otadir.startsWith("./")) {
       config.server.otadir = process.cwd() + config.server.otadir.substring(1);
@@ -920,6 +932,24 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
             });
           }
         );
+        // Download an OTA binary so the client can flash it over USB (see FLASH_PLAN.md in
+        // frugal-iot-client). Deliberately not /ota_update, which is unauthenticated for devices
+        // and answers 304 rather than sending bytes.
+        app.get('/ota_get/:org/*remainingpath',
+          loggedInOrFail,
+          can_OTAUPDATE,
+          (req,res) => {
+            let remainingpath = req.params.remainingpath.join('/');
+            let filepath = `${config.server.otadir}/${req.params.org}/${sanitize(remainingpath)}/firmware.bin`;
+            console.log("Sending OTA file", filepath);
+            res.sendFile(filepath, {}, (err) => {
+              if (err) {
+                console.error("Error sending ota file:", filepath, err);
+                if (!res.headersSent) res.status(404).send(err.message);
+              }
+            });
+          }
+        );
         app.get('/people_list/:org',
           loggedInOrFail,
           can_ADMIN, // Gets org from URL
@@ -985,6 +1015,10 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
 
         // Serve frugal-iot-logger data at /data but configure where to get them.
         console.log("Serving /data from", config.server.datadir);
+        // Nothing ever removed old readings, so a server left running filled its disk - which on a
+        // field node takes the whole machine down. This compresses days gone by and, if asked to,
+        // removes the oldest ones. See lib/housekeeping.js for what the settings mean.
+        startHousekeeping(config.server.datadir, config.server.housekeeping);
         //TODO-N89 should be authenticated to correct org
         const routerData = express.Router();
         app.use('/data', routerData);
@@ -995,7 +1029,39 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
             res.locals.org = req.url.split("/")[1];
             next(); }, //
           can_READ,
-          (req,res,next) => { console.log("/data handler authenticated by session for", req.url); next(); },
+          // Nothing logged here on purpose - morgan already reports the URL and status of this same
+          // request, and on an SD card each duplicated line is another write.
+          // The logger holds recent readings in memory rather than writing each one to the card as
+          // it arrives, so write them out before serving the files, or today's would be short of
+          // the last few minutes. Reading data is rare compared with recording it, and this costs
+          // nothing when there is nothing waiting.
+          // The guard is for an installation whose logger predates flush() - it would rather serve
+          // data a few minutes out of date than fail the request
+          (req, res, next) => { if (mqttLogger.flush) { mqttLogger.flush(() => next()); } else { next(); } },
+          // Readings from days gone by get compressed (see lib/housekeeping.js) but are still
+          // asked for by their ".csv" name, so answer one of those with the ".csv.gz" if that is
+          // what is on disk. A browser decompresses a gzip Content-Encoding itself, so the client
+          // needed no change; anything that says it does not want gzip - curl, by default - gets
+          // it decompressed here instead.
+          (req, res, next) => {
+            if (!req.path.endsWith('.csv')) { next(); return; }
+            // req.path comes from the URL, so it has to be checked before being used as a path
+            const wanted = path.resolve(config.server.datadir, '.' + path.normalize(req.path));
+            if (!wanted.startsWith(path.resolve(config.server.datadir) + path.sep)) { next(); return; }
+            access(wanted, constants.R_OK, (err) => {
+              if (!err) { next(); return; } // Not compressed - serve it in the ordinary way
+              access(wanted + '.gz', constants.R_OK, (err1) => {
+                if (err1) { next(); return; } // Neither exists - let express.static give the 404
+                res.setHeader('Content-Type', 'text/csv; charset=utf-8');
+                if (req.acceptsEncodings('gzip')) {
+                  res.setHeader('Content-Encoding', 'gzip');
+                  createReadStream(wanted + '.gz').pipe(res);
+                } else {
+                  createReadStream(wanted + '.gz').pipe(createGunzip()).pipe(res);
+                }
+              });
+            });
+          },
           express.static(config.server.datadir, {immutable: false})
         );
 

package/lib/api-routes.js

@@ -76,6 +76,9 @@ export function createAPIRouter(db, dataDir, loggerClient, pushManager = null) {
         validateTimeRange(fromTime, toTime);
 
         console.log("API fetching data from", device, fromTime*1000, toTime*1000);
+        // Recent readings are held in memory rather than written to disk as they arrive, so write
+        // them out first - otherwise a request covering "up to now" would stop a few minutes short
+        await loggerClient.flush();
         // Load data from disk
         const readings = await loadDeviceData(device, dataDir, fromTime, toTime);
 

package/lib/data-loader.js

@@ -4,7 +4,11 @@
  */
 
 import { readdir, readFile, stat } from 'fs/promises';
+import { gunzip as gunzipCallback } from 'zlib';
+import { promisify } from 'util';
 import { join } from 'path';
+
+const gunzip = promisify(gunzipCallback);
 import { parseDeviceId, toSenML, filterSenMLByTime } from './senml-utils.js';
 import { APIError } from './api-errors.js';
 
@@ -29,7 +33,8 @@ function unixToDateString(unixTime) {
  * @returns {number|null} Unix timestamp at start of day (UTC), or null if invalid
  */
 function dateFilenameToUnix(filename) {
-  const match = filename.match(/^(\d{4})-(\d{2})-(\d{2})\.csv$/);
+  // ".csv.gz" as well as ".csv" - days gone by are compressed by lib/housekeeping.js
+  const match = filename.match(/^(\d{4})-(\d{2})-(\d{2})\.csv(\.gz)?$/);
   if (!match) return null;
 
   const year = parseInt(match[1]);
@@ -121,8 +126,8 @@ export async function loadDeviceData(deviceId, dataDir, fromTime, toTime) {
               // console.log(`[loadDeviceData] Field ${module}/${field} has files: ${files.join(', ')}`);
 
               for (const file of files) {
-                // Only process .csv files
-                if (!file.endsWith('.csv')) continue;
+                // Only process reading files - ".csv", or ".csv.gz" once housekeeping has been over them
+                if (!file.endsWith('.csv') && !file.endsWith('.csv.gz')) continue;
 
                 // Parse date from filename and check if it's in range
                 const fileUnixTime = dateFilenameToUnix(file);
@@ -150,7 +155,9 @@ export async function loadDeviceData(deviceId, dataDir, fromTime, toTime) {
                 const filePath = join(fieldPath, file);
 
                 try {
-                  const content = await readFile(filePath, 'utf-8');
+                  const content = file.endsWith('.gz')
+                    ? (await gunzip(await readFile(filePath))).toString('utf-8')
+                    : await readFile(filePath, 'utf-8');
                   const lines = content.trim().split('\n');
                   // console.log(`[loadDeviceData] File ${file} has ${lines.length} lines`);
 

package/lib/housekeeping.js

@@ -0,0 +1,251 @@
+/*
+ * Housekeeping for the data directory.
+ *
+ * Readings accumulate one file per topic per day and nothing ever removed them, so a server left
+ * running eventually fills its disk - which on a field node running from an SD card takes the whole
+ * machine down, and is a good deal more likely than the card wearing out.
+ *
+ * Two things happen here, once a day:
+ *
+ *  - Days older than "compressafterdays" are gzipped. A day's file never changes once the day is
+ *    over, and they compress to about a third. Nothing needs to know: the server serves a
+ *    ".csv.gz" in answer to a request for the ".csv", and lib/data-loader.js reads either.
+ *
+ *  - Days older than "deleteafterdays" are deleted, and separately, if the disk has less than
+ *    "freespacepercent" free, the oldest days are deleted until it has. Deleting is off unless
+ *    asked for, except the free space rule, which is there to keep the machine alive.
+ *
+ * Today's file and yesterday's are never touched, whatever the settings say - the logger is still
+ * appending to them.
+ */
+
+import { createReadStream, createWriteStream, readdir, rename, stat, statfs, unlink, readFile, writeFile } from 'fs';
+import { createGzip, gunzip, gzip } from 'zlib';
+import path from 'path';
+import { each, eachSeries, waterfall } from 'async';
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+// A file for a day still being written to must never be compressed or deleted. Two days of margin
+// covers the server and the reader disagreeing about the timezone, and a node with a slow clock.
+const MINIMUM_AGE_DAYS = 2;
+
+// Day-stamped reading files, e.g. "2026-08-19.csv" or "2026-08-19.csv.gz"
+const DATAFILE = /^(\d{4}-\d{2}-\d{2})\.csv(\.gz)?$/;
+
+/*
+ * Every reading file under a directory, with the day it holds and whether it is compressed.
+ * callback(err, [ {filepath, day, ageDays, compressed, size} ])
+ */
+function findDataFiles(dir, callback) {
+  let found = [];
+  let today = Date.now();
+  function walk(d, cb) {
+    readdir(d, {withFileTypes: true}, (err, entries) => {
+      if (err) {
+        // A directory disappearing under us is not worth stopping for
+        if (err.code === 'ENOENT') { cb(null); } else { cb(err); }
+        return;
+      }
+      each(entries, (entry, cb1) => {
+        let full = path.join(d, entry.name);
+        if (entry.isDirectory()) {
+          walk(full, cb1);
+        } else {
+          let m = entry.name.match(DATAFILE);
+          if (!m) { cb1(null); return; }
+          stat(full, (err1, st) => {
+            if (err1) { cb1(null); return; } // Gone, or unreadable - leave it alone
+            found.push({
+              filepath: full,
+              day: m[1],
+              ageDays: (today - Date.parse(m[1] + 'T00:00:00Z')) / DAY_MS,
+              compressed: !!m[2],
+              size: st.size,
+            });
+            cb1(null);
+          });
+        }
+      }, cb);
+    });
+  }
+  walk(dir, (err) => callback(err, found));
+}
+
+/*
+ * Compress one day's file. If a compressed file for that day already exists - which happens when a
+ * node with a wrong clock reports a reading for a day already dealt with - the two are joined into
+ * one, because a request for that day is answered with a single file.
+ */
+function compressFile(file, callback) {
+  let gzpath = file.filepath + '.gz';
+  let tmppath = gzpath + '.tmp';
+  stat(gzpath, (err) => {
+    if (err) {
+      // The ordinary case: nothing compressed for this day yet, so stream it out. Streaming rather
+      // than reading it all in keeps this affordable on a machine with 512 MB.
+      let source = createReadStream(file.filepath);
+      let destination = createWriteStream(tmppath);
+      source.on('error', callback);
+      destination.on('error', callback);
+      destination.on('finish', () => {
+        // Only once the compressed copy is complete on disk is the original removed, so an
+        // interruption anywhere in here leaves the readings intact in one file or the other
+        rename(tmppath, gzpath, (err1) => {
+          if (err1) { callback(err1); return; }
+          unlink(file.filepath, callback);
+        });
+      });
+      source.pipe(createGzip()).pipe(destination);
+    } else {
+      // Rare: join the existing compressed day and the new rows into a single compressed file
+      waterfall([
+        (cb) => readFile(gzpath, cb),
+        (gzdata, cb) => gunzip(gzdata, cb),
+        (existing, cb) => readFile(file.filepath, (err1, added) => cb(err1, existing, added)),
+        (existing, added, cb) => gzip(Buffer.concat([existing, added]), cb),
+        (combined, cb) => writeFile(tmppath, combined, cb),
+        (cb) => rename(tmppath, gzpath, cb),
+        (cb) => unlink(file.filepath, cb),
+      ], callback);
+    }
+  });
+}
+
+/*
+ * Compress every day old enough to be finished with. One file at a time: this runs on a machine
+ * with one core, and there is nothing waiting on it.
+ */
+function compressOldFiles(files, afterDays, callback) {
+  let due = files.filter((f) => !f.compressed && (f.ageDays >= Math.max(afterDays, MINIMUM_AGE_DAYS)));
+  if (!due.length) { callback(null, 0); return; }
+  let done = 0;
+  eachSeries(due, (f, cb) => {
+    compressFile(f, (err) => {
+      if (err) { console.error("Housekeeping could not compress", f.filepath, "-", err.message); } else { done++; }
+      cb(null); // One failure should not stop the others
+    });
+  }, (err) => callback(err, done));
+}
+
+/*
+ * Delete whole days, oldest first, either because they are older than asked for or because the disk
+ * is filling up. Never touches a day recent enough that the logger might still be writing to it.
+ */
+function deleteFiles(files, callback) {
+  let removed = 0;
+  let freed = 0;
+  eachSeries(files, (f, cb) => {
+    unlink(f.filepath, (err) => {
+      if (err) { console.error("Housekeeping could not delete", f.filepath, "-", err.message); }
+      else { removed++; freed += f.size; }
+      cb(null);
+    });
+  }, (err) => callback(err, removed, freed));
+}
+
+/*
+ * How much of the filesystem holding "dir" is free, as a percentage, or null if it cannot be told.
+ */
+function freePercent(dir, callback) {
+  if (typeof statfs !== 'function') { callback(null, null); return; } // Older node
+  statfs(dir, (err, st) => {
+    if (err || !st || !st.blocks) { callback(null, null); return; }
+    callback(null, (st.bavail / st.blocks) * 100);
+  });
+}
+
+/*
+ * One pass. Exported so it can be run on demand and tested without waiting a day.
+ * config: { compressafterdays, deleteafterdays, freespacepercent }
+ * callback(err, summary)
+ */
+export function housekeep(datadir, config, callback) {
+  callback = callback || (() => {});
+  let compressAfter = (config.compressafterdays === undefined) ? 2 : config.compressafterdays;
+  let deleteAfter = config.deleteafterdays || 0;          // 0 = keep readings for ever
+  let freeWanted = (config.freespacepercent === undefined) ? 10 : config.freespacepercent;
+
+  findDataFiles(datadir, (err, files) => {
+    if (err) { console.error("Housekeeping could not read", datadir, "-", err.message); callback(err); return; }
+    if (!files.length) { callback(null, {files: 0}); return; }
+
+    // Old enough that the logger has certainly finished with it
+    let finished = files.filter((f) => f.ageDays >= MINIMUM_AGE_DAYS);
+    // Oldest first, so that freeing space takes the least useful days
+    finished.sort((a, b) => (a.day < b.day) ? -1 : (a.day > b.day) ? 1 : 0);
+
+    let byAge = deleteAfter ? finished.filter((f) => f.ageDays >= deleteAfter) : [];
+
+    deleteFiles(byAge, (err1, removedByAge, freedByAge) => {
+      // A Set, not "byAge.includes": there can be tens of thousands of these files
+      let gone = new Set(byAge.map((f) => f.filepath));
+      let left = finished.filter((f) => !gone.has(f.filepath));
+      freePercent(datadir, (err2, free) => {
+        // Nothing to do about the disk if the amount free cannot be established
+        if ((free === null) || (free >= freeWanted)) {
+          finish(removedByAge, freedByAge, 0, free);
+          return;
+        }
+        // Delete oldest days until there is enough room. The shortfall is worked out in bytes so
+        // that the number of days removed matches what is actually needed.
+        console.log(`Housekeeping: ${free.toFixed(1)}% of the disk free, below the ${freeWanted}% wanted - removing the oldest readings`);
+        statfs(datadir, (err3, st) => {
+          let shortfall = (err3 || !st) ? 0 : ((freeWanted / 100) * st.blocks - st.bavail) * st.bsize;
+          let toRemove = [];
+          let planned = 0;
+          for (let f of left) {
+            if (planned >= shortfall) break;
+            toRemove.push(f);
+            planned += f.size;
+          }
+          deleteFiles(toRemove, (err4, removedForSpace, freedForSpace) => {
+            if (toRemove.length === left.length) {
+              console.error("Housekeeping deleted every reading it was allowed to and the disk is still short of space");
+            }
+            finish(removedByAge, freedByAge + freedForSpace, removedForSpace, free);
+          });
+        });
+      });
+
+      function finish(removedByAge, freed, removedForSpace, free) {
+        // Compress last, so that anything about to be deleted is not compressed first for nothing
+        findDataFiles(datadir, (err5, remaining) => {
+          compressOldFiles(remaining || [], compressAfter, (err6, compressed) => {
+            let summary = {
+              files: files.length,
+              compressed: compressed,
+              deletedByAge: removedByAge,
+              deletedForSpace: removedForSpace,
+              freedBytes: freed,
+              freePercent: free,
+            };
+            if (compressed || removedByAge || removedForSpace) {
+              console.log("Housekeeping:", compressed, "day(s) compressed,",
+                removedByAge + removedForSpace, "deleted,",
+                (freed / 1048576).toFixed(1), "MB freed");
+            }
+            callback(null, summary);
+          });
+        });
+      }
+    });
+  });
+}
+
+/*
+ * Run it now and then once a day. Called from the server at startup.
+ * The first pass is delayed, because startup is the busiest moment on a small board and none of
+ * this is urgent.
+ */
+export function startHousekeeping(datadir, config) {
+  config = config || {};
+  if (config.enabled === false) {
+    console.log("Housekeeping of", datadir, "turned off in config.d/server.yaml");
+    return null;
+  }
+  let run = () => housekeep(datadir, config);
+  setTimeout(run, 5 * 60 * 1000).unref();
+  let timer = setInterval(run, DAY_MS);
+  timer.unref(); // Never a reason to keep the server alive just for this
+  return timer;
+}

package/lib/logger-client.js

@@ -267,6 +267,22 @@ export class LoggerClient {
     const cacheKey = `${org}/${project}/${deviceId}`;
     this.schemaCache.delete(cacheKey);
   }
+
+  /**
+   * Write out readings the logger is still holding in memory.
+   *
+   * The logger collects readings and writes them in batches, because writing each one as it
+   * arrives wears out an SD card. Call this before reading the data back, or the most recent
+   * few minutes will be missing from the files.
+   *
+   * @returns {Promise<void>} Resolves once everything waiting has been written
+   */
+  flush() {
+    // The guard is for an installation whose logger predates flush() - better to answer with data a
+    // few minutes out of date than to fail the request
+    if (!this.mqttLogger.flush) return Promise.resolve();
+    return new Promise((resolve) => this.mqttLogger.flush(resolve));
+  }
 }
 
 /**

package/package.json

@@ -8,12 +8,14 @@
   "dependencies": {
     "@gebsl/senml-js": "^1.1.0",
     "array.prototype.tospliced": "^1.1.5",
+    "async": "^3.2.6",
     "cookie-parser": "^1.4.7",
     "express": "^5.2.1",
     "express-session": "^1.19.0",
-    "frugal-iot-client": "^1.3.11",
-    "frugal-iot-logger": "^1.1.17",
+    "frugal-iot-client": "^1.3.12",
+    "frugal-iot-logger": "^1.1.18",
     "hash-wasm": "^4.12.0",
+    "js-yaml": "^4.2.0",
     "morgan": "^1.11.0",
     "multer": "^2.1.1",
     "passport": "^0.7.0",
@@ -24,7 +26,8 @@
     "frugal-iot-server": "frugal-iot-server.js",
     "frugal-iot-init": "scripts/init.zsh",
     "frugal-iot-addorganization": "scripts/addorganization.zsh",
-    "frugal-iot-setpassword": "scripts/setpassword.zsh"
+    "frugal-iot-setpassword": "scripts/setpassword.zsh",
+    "frugal-iot-diagnostic": "scripts/diagnostic.zsh"
   },
   "description": "Frugal IoT client and server",
   "//files": "An allowlist, not an ignore list - anything not named here is never published, so a stray file (a database backup, a credential) cannot leak by default. Check with: npm pack --dry-run",
@@ -63,8 +66,10 @@
   "scripts": {
     "test": "vitest run",
     "test:watch": "vitest",
-    "test:coverage": "vitest run --coverage"
+    "test:coverage": "vitest run --coverage",
+    "check-schema": "node scripts/check-schema.js",
+    "prerelease": "zsh scripts/prerelease.zsh"
   },
   "type": "module",
-  "version": "0.3.4"
+  "version": "0.3.6"
 }

package/public/service-worker.js

@@ -1,6 +1,6 @@
 // Change this version number if want existing installs of PWA to refresh their cache
 // TODO-28 - probably make this the same as the app version in frugal-iot-client
-const CACHE_NAME = 'frugal-iot-cache-1.3.11';
+const CACHE_NAME = 'frugal-iot-cache-1.3.12';
 const urlsToCache = [
     '/',
     '/index.html',

package/scripts/addorganization.zsh

@@ -2,13 +2,13 @@
 #
 # Add a new organization to Frugal IoT: config file, user account, permissions, and mqtt credentials.
 # Run from the server's own directory, the one holding frugal-iot.db (all paths below are relative
-# to there) - prepare it first with "npx frugal-iot-init".
+# to there) - prepare it first with "npx --no frugal-iot-init".
 #
 # Usage:
-#   npx frugal-iot-addorganization <org-id> <org-name> <email> <phone> <password>
+#   npx --no frugal-iot-addorganization <org-id> <org-name> <email> <phone> <password>
 #   scripts/addorganization.zsh <org-id> <org-name> <email> <phone> <password>
 # Example:
-#   npx frugal-iot-addorganization abc "Clever People" foo@abc.org +61123456 "secret!123"
+#   npx --no frugal-iot-addorganization abc "Clever People" foo@abc.org +61123456 "secret!123"
 
 set -euo pipefail
 
@@ -59,7 +59,7 @@ fi
 
 if [[ ! -f "$DB" ]]; then
   echo "Error: database ${DB} not found in this directory - prepare the directory first with:" >&2
-  echo "  npx frugal-iot-init" >&2
+  echo "  npx --no frugal-iot-init" >&2
   exit 1
 fi
 
@@ -75,13 +75,50 @@ if ! command -v mosquitto_passwd >/dev/null; then
   MOSQUITTO_PASSWD_MISSING=1
 fi
 
-# Prefer the real system password file; if it's not present (e.g. mosquitto isn't installed on this
-# machine), fall back to editing the local copy under extras/ so the script still does something useful.
-MOSQUITTO_PASSWD_FILE="/etc/mosquitto/mosquitto_passwords"
+# Ask the broker's own configuration where its password file is, rather than guessing - that way
+# this works whatever layout a machine uses, including installations older than this script.
+# Mosquitto reads /etc/mosquitto/mosquitto.conf and then the .conf files in conf.d, last one wins.
+MOSQUITTO_PASSWD_FILE=$(grep -hE '^[[:space:]]*password_file[[:space:]]' \
+  /etc/mosquitto/mosquitto.conf /etc/mosquitto/conf.d/*.conf(N) 2>/dev/null | tail -1 | awk '{print $2}')
 USED_LOCAL_MOSQUITTO_FILE=0
-if [[ ! -e "$MOSQUITTO_PASSWD_FILE" && -e "extras/mosquitto_passwords" ]]; then
-  MOSQUITTO_PASSWD_FILE="extras/mosquitto_passwords"
-  USED_LOCAL_MOSQUITTO_FILE=1
+if [[ -n "$MOSQUITTO_PASSWD_FILE" ]]; then
+  MOSQUITTO_PASSWD_FROM_CONFIG=1
+else
+  MOSQUITTO_PASSWD_FROM_CONFIG=0
+  # No mosquitto configuration here - fall back to the usual locations, and then to the copy under
+  # extras/ so that a machine without mosquitto installed still gets something useful.
+  for f in /var/lib/mosquitto/passwords /etc/mosquitto/mosquitto_passwords; do
+    if [[ -e "$f" ]]; then
+      MOSQUITTO_PASSWD_FILE="$f"
+      break
+    fi
+  done
+  if [[ -z "$MOSQUITTO_PASSWD_FILE" ]]; then
+    if [[ -e "extras/mosquitto_passwords" ]]; then
+      MOSQUITTO_PASSWD_FILE="extras/mosquitto_passwords"
+      USED_LOCAL_MOSQUITTO_FILE=1
+    else
+      MOSQUITTO_PASSWD_FILE="/var/lib/mosquitto/passwords"
+    fi
+  fi
+fi
+
+# Run mosquitto_passwd as whoever owns that file. Mosquitto warns unless the file belongs to the user
+# opening it, and mosquitto_passwd writes a temporary backup beside it, so it needs the directory too
+# - neither of which we get by running as ourselves against a file owned by mosquitto or by root.
+# On Raspberry Pi OS the first user has passwordless sudo, so this is invisible.
+MOSQUITTO_PASSWD_CMD=(mosquitto_passwd)
+if [[ ! -w "${MOSQUITTO_PASSWD_FILE:h}" || ( -e "$MOSQUITTO_PASSWD_FILE" && ! -w "$MOSQUITTO_PASSWD_FILE" ) ]]; then
+  if command -v sudo >/dev/null; then
+    # stat's spelling differs between Linux (-c) and BSD/macOS (-f)
+    MOSQUITTO_PASSWD_OWNER=$(stat -c '%U' "$MOSQUITTO_PASSWD_FILE" 2>/dev/null \
+      || stat -f '%Su' "$MOSQUITTO_PASSWD_FILE" 2>/dev/null || true)
+    if [[ -n "$MOSQUITTO_PASSWD_OWNER" && "$MOSQUITTO_PASSWD_OWNER" != "$(id -un)" && "$MOSQUITTO_PASSWD_OWNER" != "root" ]]; then
+      MOSQUITTO_PASSWD_CMD=(sudo -u "$MOSQUITTO_PASSWD_OWNER" mosquitto_passwd)
+    else
+      MOSQUITTO_PASSWD_CMD=(sudo mosquitto_passwd)
+    fi
+  fi
 fi
 
 # ---- Escaping helpers ----
@@ -168,17 +205,24 @@ MOSQUITTO_PASSWD_FAILED=0
 MOSQUITTO_PASSWD_ERROR=""
 if [[ "$MOSQUITTO_PASSWD_MISSING" -eq 1 ]]; then
   echo "Warning: mosquitto_passwd command not found - skipped setting MQTT broker password" >&2
+elif [[ "$MOSQUITTO_PASSWD_FROM_CONFIG" -eq 1 && ! -e "$MOSQUITTO_PASSWD_FILE" ]]; then
+  # Nothing to add an account to - and mosquitto will not be running either, for the same reason.
+  echo "Warning: mosquitto is configured to use ${MOSQUITTO_PASSWD_FILE}, but that file does not exist," >&2
+  echo "so the MQTT broker password was NOT set. Create it, then add this organization to it:" >&2
+  echo "  sudo install -o mosquitto -g mosquitto -m 600 /dev/null \"${MOSQUITTO_PASSWD_FILE}\"" >&2
+  echo "  sudo -u mosquitto mosquitto_passwd -b \"${MOSQUITTO_PASSWD_FILE}\" \"${ORG_ID}\" \"${PASSWORD}\"" >&2
+  MOSQUITTO_PASSWD_FAILED=1
 else
   # Capture stderr without letting `set -e` abort the whole script on failure - the org's config file
   # and DB rows are already written by this point, so a failure here should be reported, not fatal.
-  MOSQUITTO_PASSWD_ERROR=$(mosquitto_passwd -b "$MOSQUITTO_PASSWD_FILE" "${ORG_ID}" "${PASSWORD}" 2>&1 >/dev/null) || MOSQUITTO_PASSWD_FAILED=1
+  MOSQUITTO_PASSWD_ERROR=$("${MOSQUITTO_PASSWD_CMD[@]}" -b "$MOSQUITTO_PASSWD_FILE" "${ORG_ID}" "${PASSWORD}" 2>&1 >/dev/null) || MOSQUITTO_PASSWD_FAILED=1
   if [[ "$MOSQUITTO_PASSWD_FAILED" -eq 1 ]]; then
     echo "Warning: mosquitto_passwd failed to set the MQTT broker password:" >&2
     echo "  ${MOSQUITTO_PASSWD_ERROR}" >&2
-    if [[ "$MOSQUITTO_PASSWD_ERROR" == *"Permission denied"* || ! -w "$MOSQUITTO_PASSWD_FILE" ]]; then
-      echo "This looks like a permissions problem - re-run that command with sudo:" >&2
-      echo "  sudo mosquitto_passwd -b \"${MOSQUITTO_PASSWD_FILE}\" \"${ORG_ID}\" \"${PASSWORD}\"" >&2
-    fi
+    echo "Everything else was done, so set just the broker password by hand:" >&2
+    echo "  sudo mosquitto_passwd -b \"${MOSQUITTO_PASSWD_FILE}\" \"${ORG_ID}\" \"${PASSWORD}\"" >&2
+    echo "If it complains about creating a backup file, the directory holding that file is not" >&2
+    echo "writable by you - which is why the command above uses sudo." >&2
   else
     echo "Set mosquitto password for ${ORG_ID} in ${MOSQUITTO_PASSWD_FILE}"
   fi