frugal-iot-server 2.0.11 → 2.0.12

package/scripts/diagnostic.zsh

@@ -133,36 +133,59 @@ if [[ -f frugal-iot.db ]] && have sqlite3; then
 fi
 
 section "Mosquitto configuration"
-if [[ -f /etc/mosquitto/mosquitto.conf ]]; then
-  item "/etc/mosquitto/mosquitto.conf: $(fileinfo /etc/mosquitto/mosquitto.conf)"
+# Which files the broker reads, asked of its own configuration - production's is several hand-made
+# files in conf.d, a Pi's is one frugal-iot.conf. scripts/broker-config.zsh explains the sudo retry,
+# which is per file: a root-only bridge file used to be skipped silently among readable ones.
+# docs/INSTALLATION.md has this script copied on its own to a machine not yet installed, so the
+# helper may not be beside it - look in the installed package too before giving up.
+BROKER_HELPER=""
+for f in "${0:A:h}/broker-config.zsh" node_modules/frugal-iot-server/scripts/broker-config.zsh; do
+  [[ -r "$f" ]] && { BROKER_HELPER="$f"; break; }
+done
+if [[ -z "$BROKER_HELPER" ]]; then
+  print -u3 "diagnostic.zsh needs broker-config.zsh beside it - copy that from the same scripts/ directory"
+  exit 2
+fi
+source "$BROKER_HELPER"
+broker_conf_read
+if (( ! ${#BROKER_CONF_FILES} )); then
+  item "$(broker_conf_main): missing - is mosquitto installed?"
 else
-  item "/etc/mosquitto/mosquitto.conf: missing - is mosquitto installed?"
-fi
-item "files in /etc/mosquitto/conf.d:"
-for f in /etc/mosquitto/conf.d/*(N); do
-  item "  ${f}: $(fileinfo $f)"
-  grep -nE '^[[:space:]]*(listener|protocol|password_file|allow_anonymous)' $f 2>/dev/null | sed 's/^/      /'
+  item "files the broker reads, in order:"
+  for f in $BROKER_CONF_FILES; do
+    item "  ${f}: $(fileinfo $f)"
+    # Read through the helper rather than grep, so a root-only file is shown as well
+    print -r -- "$(broker_conf_cat $f 2>/dev/null)" | grep -nE '^[[:space:]]*(listener|protocol|password_file|allow_anonymous)' | sed 's/^/      /'
+  done
+  (( ${#BROKER_CONF_FILES} == 1 )) && problem "$BROKER_CONF_FILES reads no other files - the Frugal IoT config was never copied into conf.d (step 5)"
+fi
+if (( ${#BROKER_CONF_UNREADABLE} )); then
+  item "NOT READABLE as $(id -un), even with sudo -n: ${(j:, :)BROKER_CONF_UNREADABLE}"
+  item "  everything below about the broker is missing whatever those files say - run with sudo"
+  problem "Could not read ${(j:, :)BROKER_CONF_UNREADABLE} - re-run with sudo for a complete answer about the broker"
+fi
+item "files the configuration names, and what a backup does with each (the files above are taken too):"
+# NAMED_FILE, not "path": zsh ties $path to $PATH, so reading into it wipes out every command after
+broker_named_files | while read -r setting NAMED_FILE role; do
+  case $role in
+    backup)      role="taken" ;;
+    tls)         role="taken (TLS)" ;;
+    rebuildable) role="taken - its admin account matches secrets.yaml; the rest is rebuilt from the db" ;;
+    package)     role="not taken - part of mosquitto" ;;
+    live)        role="not taken - running state" ;;
+  esac
+  # A certificate under /etc/letsencrypt/live sits in a root-only directory, so it is invisible to
+  # an ordinary user - which is not the same thing as missing.
+  if [[ ! -e "$NAMED_FILE" ]] && have sudo && sudo -n test -e "$NAMED_FILE" 2>/dev/null; then
+    item "  ${(r:22:)setting} ${NAMED_FILE}: present, needs root to see"
+  else
+    item "  ${(r:22:)setting} ${NAMED_FILE}: $(fileinfo $NAMED_FILE)"
+  fi
+  item "      -> ${role}"
 done
-[[ -z "$(print -r -- /etc/mosquitto/conf.d/*(N))" ]] && problem "Nothing in /etc/mosquitto/conf.d - the Frugal IoT config was never copied there (step 5)"
-
-# The files the broker actually reads. mosquitto's include_dir only loads names ending in .conf,
-# so look at exactly those - and the last setting wins, hence "tail -1" everywhere below.
-BROKER_CONF_FILES=(/etc/mosquitto/mosquitto.conf(N) /etc/mosquitto/conf.d/*.conf(N))
 
-# Everything the broker reads, as one stream - with sudo if reading it plainly gives nothing. A
-# file under /etc can be readable only by root, and an ordinary user then gets silence rather than
-# an error, which is indistinguishable from "that setting is not configured". Left unguarded, the
-# comparison below reported a broker as missing EVERY setting this release ships, which as a
-# periodic check is worse than no check at all.
-broker_conf_text() {
-  local t
-  (( ${#BROKER_CONF_FILES} )) || return 0
-  t=$(cat "${BROKER_CONF_FILES[@]}" 2>/dev/null) || true
-  if [[ -z "$t" ]] && have sudo; then
-    t=$(sudo -n cat "${BROKER_CONF_FILES[@]}" 2>/dev/null) || true
-  fi
-  print -r -- "$t"
-}
+# Everything the broker reads, as one stream, comments removed
+broker_conf_text() { print -r -- "$BROKER_CONF_TEXT"; }
 
 # Where the packaged copy of a shipped file is. What is in ./extras is NOT it: frugal-iot-init
 # copies extras/ in and then never overwrites it, so once a release changes one of these files the
@@ -170,8 +193,38 @@ broker_conf_text() {
 PKG_EXTRAS="node_modules/frugal-iot-server/extras"
 [[ -d "$PKG_EXTRAS" ]] || PKG_EXTRAS="${0:A:h:h}/extras"
 
+# What this release ships, against what the broker is actually running.
+#
+# This used to be "diff ./extras/mosquitto.conf /etc/mosquitto/conf.d/frugal-iot.conf", which
+# reported a problem on every machine there has ever been, for three separate reasons: ./extras is
+# a fossil frugal-iot-init will not overwrite, so it differs from the release for ever; the
+# installed copy has the dynamic-security plugin's path substituted, so it differs from the shipped
+# one by design; and "diff -q" against a frugal-iot.conf that does not exist fails too, which is
+# the normal state of a broker whose configuration was assembled by hand as several small files in
+# conf.d. A check that is always wrong is one you stop reading, so this asks the question that
+# actually matters instead: is the broker missing a setting this release ships?
+if [[ -f "${PKG_EXTRAS}/mosquitto.conf" ]] && (( ${#BROKER_CONF_FILES} )); then
+  # Setting by setting against what the broker reads, wherever it reads it from - see
+  # broker_conf_compare. A file kept at another path is information, not a fault.
+  CONF_FINDINGS=$(broker_conf_compare "${PKG_EXTRAS}/mosquitto.conf")
+  if [[ -z "$CONF_FINDINGS" ]]; then
+    item "the broker has every setting this release ships"
+  else
+    item "compared with this release's mosquitto.conf:"
+    CONF_REAL=0
+    for l in ${(f)CONF_FINDINGS}; do
+      case ${l%%$'\t'*} in
+        missing)   item "    missing:   ${l#*$'\t'}"; (( CONF_REAL++ )) || true ;;
+        differs)   item "    different: ${l#*$'\t'}"; (( CONF_REAL++ )) || true ;;
+        elsewhere) item "    (fine)     ${l#*$'\t'}" ;;
+      esac
+    done
+    (( CONF_REAL )) && problem "The broker's configuration differs from this release's in ${CONF_REAL} setting(s) - see \"Mosquitto configuration\" above. Add them to a file in /etc/mosquitto/conf.d/ (or re-run install-pi.sh on a Pi it set up), then: sudo systemctl restart mosquitto"
+  fi
+fi
+
 # Which password file does the running configuration actually name?
-CONFIGURED_PWFILE=$(broker_conf_text | grep -hE '^[[:space:]]*password_file[[:space:]]' | tail -1 | awk '{print $2}')
+broker_setting password_file; CONFIGURED_PWFILE="$BROKER_SETTING"
 section "Mosquitto password file"
 if [[ -n "$CONFIGURED_PWFILE" ]]; then
   item "configuration names: $CONFIGURED_PWFILE"
@@ -180,44 +233,15 @@ if [[ -n "$CONFIGURED_PWFILE" ]]; then
     problem "Mosquitto is configured to use $CONFIGURED_PWFILE but that file does not exist - it will refuse to start"
   else
     # Account names only, never the hashes
-    item "  accounts in it:    $(cut -d: -f1 "$CONFIGURED_PWFILE" 2>/dev/null | tr '\n' ' ')"
+    # Read through the helper: the file is mosquitto:mosquitto 600, so a plain cut as the service
+    # user returns nothing, which looks exactly like a file with no accounts in it
+    item "  accounts in it:    $(broker_conf_cat "$CONFIGURED_PWFILE" | cut -d: -f1 | tr '\n' ' ')"
     [[ ! -s "$CONFIGURED_PWFILE" ]] && item "  (the file is empty - no accounts yet)"
   fi
 else
   item "no password_file line found in any mosquitto configuration"
   problem "No password_file configured - the broker would allow anonymous access or reject everything"
 fi
-# What this release ships, against what the broker is actually running.
-#
-# This used to be "diff ./extras/mosquitto.conf /etc/mosquitto/conf.d/frugal-iot.conf", which
-# reported a problem on every machine there has ever been, for three separate reasons: ./extras is
-# a fossil frugal-iot-init will not overwrite, so it differs from the release for ever; the
-# installed copy has the dynamic-security plugin's path substituted, so it differs from the shipped
-# one by design; and "diff -q" against a frugal-iot.conf that does not exist fails too, which is
-# the normal state of a broker whose configuration was assembled by hand as several small files in
-# conf.d. A check that is always wrong is one you stop reading, so this asks the question that
-# actually matters instead: is the broker missing a setting this release ships?
-if [[ -f "${PKG_EXTRAS}/mosquitto.conf" ]]; then
-  item "this release's mosquitto.conf names: $(grep -hE '^[[:space:]]*password_file' "${PKG_EXTRAS}/mosquitto.conf" 2>/dev/null | awk '{print $2}')"
-  if (( ${#BROKER_CONF_FILES} )); then
-    # The plugin line is normalised: the installer substitutes a path that varies by distribution.
-    # Comparing settings rather than files means this works whether the configuration arrived as one
-    # frugal-iot.conf or as several hand-written pieces.
-    MISSING_CONF=$(comm -23 \
-      <(sed -E 's#^plugin .*#plugin -#' "${PKG_EXTRAS}/mosquitto.conf" | grep -vE '^[[:space:]]*(#|$)' | sort -u) \
-      <(broker_conf_text | sed -E 's#^plugin .*#plugin -#' | grep -vE '^[[:space:]]*(#|$)' | sort -u) 2>/dev/null)
-    # password_file and the listeners are allowed to be sited differently on an older installation,
-    # so a difference in those is reported without being called a fault.
-    MISSING_REAL=$(print -r -- "$MISSING_CONF" | grep -vE '^[[:space:]]*(password_file|listener|protocol)[[:space:]]' || true)
-    if [[ -n "$MISSING_CONF" ]]; then
-      item "settings this release ships that the broker does not have:"
-      print -r -- "$MISSING_CONF" | while IFS= read -r l; do [[ -n "$l" ]] && item "    $l"; done
-    else
-      item "the broker has every setting this release ships"
-    fi
-    [[ -n "$MISSING_REAL" ]] && problem "The broker is missing $(print -r -- "$MISSING_REAL" | grep -c .) setting(s) this release ships - see \"Mosquitto password file\" above. Re-run install-pi.sh, or add them by hand if this broker's config.d was written by hand."
-  fi
-fi
 item "candidate locations, whether or not configured:"
 for f in /var/lib/mosquitto/passwords /etc/mosquitto/mosquitto_passwords; do
   item "  ${f}: $(fileinfo $f)"
@@ -238,7 +262,7 @@ else
   item "systemctl not available on this machine"
 fi
 # Debian's mosquitto logs to a file rather than the journal, so this is where the real reason is.
-MOSQUITTO_LOG=$(grep -hE '^[[:space:]]*log_dest[[:space:]]+file' /etc/mosquitto/mosquitto.conf /etc/mosquitto/conf.d/*(N) 2>/dev/null | tail -1 | awk '{print $3}')
+MOSQUITTO_LOG=$(broker_conf_lines log_dest | awk '$2 == "file" {print $3}' | tail -1)
 [[ -z "$MOSQUITTO_LOG" ]] && MOSQUITTO_LOG=/var/log/mosquitto/mosquitto.log
 item "its own log file: ${MOSQUITTO_LOG} ($(fileinfo $MOSQUITTO_LOG))"
 if [[ -r "$MOSQUITTO_LOG" ]]; then
@@ -347,11 +371,6 @@ section "Broker access control"
 # An acl_file confines each account to its own organization's topics; without one, every account
 # that can log in reaches every topic on the broker. Reported here because a broker installed before
 # the ACL shipped will not have one, and nothing else says so.
-BROKER_SETTING=""
-broker_setting() {  # name -> value from the broker's own configuration, sudo only if needed
-  # broker_conf_text carries the sudo fallback this used to spell out for itself
-  BROKER_SETTING=$(broker_conf_text | grep -hE "^[[:space:]]*$1[[:space:]]" | tail -1 | awk '{print $2}') || true
-}
 broker_setting acl_file; ACLFILE="$BROKER_SETTING"
 if [[ -z "$ACLFILE" ]]; then
   item "acl_file: NOT SET - every account that can log in can reach every topic, across all organizations"
@@ -495,24 +514,13 @@ section "Bridge to a production server (optional)"
 # A bridge relays this Pi's readings to a production server - see
 # extras/mosquitto-bridge.conf.example. It is configured by a "connection" line in one of
 # mosquitto's own config files, so most installations have none, which is not a problem.
-BRIDGECONFS=(/etc/mosquitto/mosquitto.conf(N) /etc/mosquitto/conf.d/*.conf(N))
 # The bridge's own file is 600 root, because it holds the bridge's password - so read as an ordinary
 # user it yields nothing, which looked exactly like "no bridge configured" on a Pi whose bridge was
-# configured and failing. Read each file with sudo when it cannot be read plainly, and say so when
-# even that fails rather than report an answer that was never looked for.
-BRIDGETEXT=""; BRIDGEUNREAD=()
-for f in $BRIDGECONFS; do
-  if [[ -r $f ]]; then
-    BRIDGETEXT+="$(<$f)"$'\n'
-  elif have sudo && t=$(sudo -n cat $f 2>/dev/null); then
-    BRIDGETEXT+="$t"$'\n'
-  else
-    BRIDGEUNREAD+=($f)
-  fi
-done
-BRIDGENAMES=$(print -r -- "$BRIDGETEXT" | grep -E '^[[:space:]]*connection[[:space:]]' | awk '{print $2}')
-if [[ -z "$BRIDGENAMES" ]] && (( ${#BRIDGEUNREAD} )); then
-  item "cannot tell whether a bridge is configured: not readable as $(id -un) - ${(j:, :)BRIDGEUNREAD}"
+# configured and failing. broker_conf_read retried it with sudo; say so when even that failed rather
+# than report an answer that was never looked for.
+BRIDGENAMES=$(broker_conf_lines connection | awk '{print $2}')
+if [[ -z "$BRIDGENAMES" ]] && (( ${#BROKER_CONF_UNREADABLE} )); then
+  item "cannot tell whether a bridge is configured: not readable as $(id -un) - ${(j:, :)BROKER_CONF_UNREADABLE}"
   item "  sudo npx --no frugal-iot-diagnostic   reads them"
 elif [[ -z "$BRIDGENAMES" ]]; then
   item "no bridge configured - readings stay on this Pi"
@@ -520,7 +528,7 @@ else
   for b in ${(f)BRIDGENAMES}; do item "configured bridge: $b"; done
   # What it is set to relay and where. remote_password is deliberately not among these.
   item "its settings:"
-  print -r -- "$BRIDGETEXT" | grep -E '^[[:space:]]*(address|remote_username|remote_clientid|topic|cleansession|restart_timeout|notifications)[[:space:]]' | sed 's/^/      /'
+  broker_conf_lines 'address|remote_username|remote_clientid|topic|cleansession|restart_timeout|notifications' | sed 's/^/      /'
   # Whether it is actually up, rather than what the configuration hoped for. With "notifications"
   # left on, the bridge publishes a retained 1 or 0 here each time it connects or drops.
   if have mosquitto_sub; then
@@ -583,6 +591,37 @@ if [[ -d data ]]; then
   fi
 fi
 
+section "Backups"
+# A nightly timer that has quietly stopped looks exactly like one that works, so say how old the
+# newest snapshot is. Only a problem once the timer is installed - before that, nothing promised one.
+BACKUP_DIR=/var/backups/frugal-iot
+BACKUP_TIMER=""
+have systemctl && { BACKUP_TIMER=$(systemctl is-enabled frugal-iot-backup.timer 2>/dev/null) || true; }
+# is-enabled answers "not-found" for a unit that does not exist, rather than nothing
+[[ "$BACKUP_TIMER" == not-found ]] && BACKUP_TIMER=""
+item "timer:       ${BACKUP_TIMER:-not installed - npx --no frugal-iot-backup install}"
+# The directory is 700, owned by the account the server runs as - so sudo for anyone else
+NEWEST_BACKUP=$( (ls -1t "$BACKUP_DIR" 2>/dev/null || { have sudo && sudo -n ls -1t "$BACKUP_DIR" 2>/dev/null; }) |
+  grep -m1 '^frugal-iot-state-.*\.tar\.gz$') || true
+if [[ -n "$NEWEST_BACKUP" ]]; then
+  # The time is in the name, which saves a stat that might need sudo too
+  BACKUP_STAMP=${${NEWEST_BACKUP%.tar.gz}[-15,-1]}     # YYYYmmdd-HHMMSS
+  zmodload zsh/datetime 2>/dev/null
+  if strftime -r -s BACKUP_SECS '%Y%m%d-%H%M%S' "$BACKUP_STAMP" 2>/dev/null; then
+    BACKUP_HOURS=$(( (EPOCHSECONDS - BACKUP_SECS) / 3600 ))
+    item "newest:      ${BACKUP_DIR}/${NEWEST_BACKUP} (${BACKUP_HOURS} hours old)"
+    [[ "$BACKUP_TIMER" == enabled ]] && (( BACKUP_HOURS > 48 )) &&
+      problem "The newest backup is ${BACKUP_HOURS} hours old although the nightly timer is enabled - see: journalctl -u frugal-iot-backup"
+  else
+    item "newest:      ${BACKUP_DIR}/${NEWEST_BACKUP}"
+  fi
+elif [[ "$BACKUP_TIMER" == enabled ]]; then
+  item "newest:      none in ${BACKUP_DIR} (or not readable as $(id -un))"
+  problem "The backup timer is enabled but there is no snapshot in ${BACKUP_DIR} - see: journalctl -u frugal-iot-backup"
+else
+  item "newest:      none"
+fi
+
 section "Wear on the SD card"
 # An SD card wears out from being written to, and a server logging sensor readings writes constantly
 # unless told not to. This reports how much is actually being written and whether the settings that

package/scripts/init.zsh

@@ -31,18 +31,33 @@ fi
 COPIED=()      # which files this run actually created, so existing ones are never rewritten below
 DIFFER_TO=()   # files kept that no longer match what this release ships, and where to compare them
 DIFFER_FROM=()
+DIFFER_OURS=() # ...and the lines in each that this release does not have
 # copy_if_missing <packaged file> <local file> [compare]
-# "compare" asks for a warning when the file exists but differs from the one this release ships.
-# Only pass it for files nobody is expected to edit - config.yaml, mqtt.yaml and server.yaml are
-# this installation's own settings, so of course they differ, and warning about them every time
-# would train you to ignore the warning that matters.
+# "compare" is for a file a release extends - the schema - and that someone may also have added to
+# here. When it differs, the question is whether this copy has anything the release does not:
+#  - nothing: every line here is still in the release's version, so all the release did was add
+#    (a new sensor type, a new field). Installing it loses nothing, so it is installed. Asking
+#    instead was a question with only one answer, put to someone who then had to read a diff of a
+#    file they never touched to find that out.
+#  - something: either it was added here, or the release changed or removed it - and only a person
+#    can tell which. So it is kept, and those lines, and only those, are shown.
+# Only pass "compare" for files like that - config.yaml, mqtt.yaml and server.yaml are this
+# installation's own settings, so of course they differ, and warning about them every time would
+# train you to ignore the warning that matters.
 copy_if_missing() {
-  local from=$1 to=$2 compare=${3:-}
+  local from=$1 to=$2 compare=${3:-} ours
   if [[ -e "$to" ]]; then
     if [[ "$compare" == compare && -e "$from" ]] && ! cmp -s "$from" "$to"; then
-      DIFFER_TO+=("$to")
-      DIFFER_FROM+=("$from")
-      echo "  kept    ${to} (DIFFERS from this release - see below)"
+      ours=$(diff "$to" "$from" | sed -n 's/^< //p') || true
+      if [[ -z "$ours" ]]; then
+        cp "$from" "$to"
+        echo "  updated ${to} (this release only adds to it)"
+      else
+        DIFFER_TO+=("$to")
+        DIFFER_FROM+=("$from")
+        DIFFER_OURS+=("$ours")
+        echo "  kept    ${to} (has lines this release does not - see below)"
+      fi
     else
       echo "  kept    ${to} (already present)"
     fi
@@ -192,115 +207,177 @@ else
   echo "  kept    ${DB} permissions.project (already present)"
 fi
 
+# ---- 3b. Dependency overrides, which npm reads only from THIS directory's package.json ----
+# frugal-iot-server's package.json carries "overrides" that fix vulnerable transitive dependencies
+# (uuid, below firebase-admin in the logger). npm honours overrides only in the top-level project,
+# and here that is this directory's package.json, which just lists frugal-iot-server - so the
+# package's own overrides are ignored, and "npm audit" on an install reports what it reports clean
+# in a checkout. Copied here so that they apply. A dependency needing an override has to have it
+# repeated in the server's package.json, since the logger's own is ignored for the same reason.
+#
+# The release's value replaces a different one here: these exist to raise a floor, and a stale one
+# would bring the vulnerability back. One a release drops is left in place - it is harmless.
+if [[ "$PKG" == */node_modules/frugal-iot-server && -f ./package.json ]]; then
+  echo "Dependencies:"
+  OVR_SET=$(node -e '
+    const fs = require("fs");
+    const want = JSON.parse(fs.readFileSync(process.argv[1])).overrides || {};
+    const local = JSON.parse(fs.readFileSync("./package.json"));
+    const have = local.overrides || {};
+    const changed = Object.keys(want).filter(k => JSON.stringify(have[k]) !== JSON.stringify(want[k]));
+    if (changed.length) {
+      local.overrides = { ...have, ...want };
+      fs.writeFileSync("./package.json", JSON.stringify(local, null, 2) + "\n");
+    }
+    console.log(changed.map(k => k + " " + JSON.stringify(want[k])).join("\n"));
+  ' "${PKG}/package.json") || OVR_SET="ERROR"
+  if [[ "$OVR_SET" == ERROR ]]; then
+    echo "  ERROR: could not read or update ./package.json - its overrides are unchanged" >&2
+  elif [[ -z "$OVR_SET" ]]; then
+    echo "  kept    ./package.json overrides (already present)"
+  else
+    for l in ${(f)OVR_SET}; do echo "  set     ./package.json override ${l}"; done
+    # The override changes nothing in node_modules until npm installs again
+    if npm install --no-fund --no-audit; then
+      echo "  updated ./node_modules to match"
+    else
+      echo "  ERROR: npm install failed - run it again in $(pwd) before restarting the server" >&2
+    fi
+  fi
+fi
+
 # ---- 4. Is what was installed FROM extras/ still current? ----
 # The files in extras/ are templates for files that live somewhere else - the broker's
-# configuration, its ACL, the systemd unit. Whether this directory's copy matches the release says
-# nothing about any of them, so the question is asked where the file actually is.
+# configuration, its ACL, the systemd units. Whether this directory's copy matches the release says
+# nothing about any of them, so the question is asked of the file actually in use - found by asking
+# the broker and systemd where that is, never by assuming a Pi's layout. Production's broker
+# configuration is several hand-written files in conf.d, its unit is in /usr/lib/systemd/system, and
+# checking "the usual place" there reported everything as absent, which was true and no help.
 #
-# Each of these is checked in the way that file is allowed to differ, because two of them are
-# *supposed* to differ and a plain cmp would cry wolf at every upgrade:
-#  - mosquitto.conf carries a placeholder for the dynamic security plugin, which the installer
-#    replaces with a path that varies by distribution.
-#  - aclfile gets a stanza appended per bridge by addbridge-prod, so the installed file is
-#    expected to be a superset - only a line this release ships and it lacks is a finding.
-STALE=()        # installed, and older than this release
-UNINSTALLED=()  # nothing at the usual place, so there was nothing to compare
-UNREADABLE=()   # there, but not readable without sudo
+# Each is compared in the way it is allowed to differ, and what is reported is the lines that
+# differ, not an instruction to go and diff it:
+#  - the broker's configuration, setting by setting across every file it reads
+#    (broker_conf_compare in scripts/broker-config.zsh)
+#  - aclfile as a subset: addbridge-prod appends a stanza per bridge, so only a line this release
+#    ships and the installed file lacks is a finding
+#  - frugaliot.service with User, WorkingDirectory and ExecStart normalised, because the installer
+#    rewrites those for whoever runs it and wherever the server went
+STALE=()        # "installed|packaged" - and STALE_LINES, what differs, in the same order
+STALE_LINES=()
+UNREADABLE=()   # there, but not readable even with sudo -n
+
+# norm_text <how>: settings only - comments and blank lines gone - with what an installer
+# legitimately rewrites reduced to its name. Reads its standard input.
+norm_text() {
+  local expr='s#^##'
+  [[ $1 == service ]] && expr='s#^(User|WorkingDirectory|ExecStart)=.*#\1=(set by the installer)#'
+  sed -E "$expr" | grep -vE '^[[:space:]]*(#|$)' || true
+}
 
-# check_installed <packaged file> <installed path> <exact|plugin|subset>
+# check_installed <packaged file> <installed path> <exact|service|subset>
+# The installed file is read with broker_conf_cat, which retries with sudo -n: the aclfile is
+# mosquitto:mosquitto 600, so reading it plainly as the service user always fails.
 check_installed() {
-  local from=$1 to=$2 how=$3 a b missing
-  [[ -e "$from" ]] || return 0
-  if [[ ! -e "$to" ]]; then
-    UNINSTALLED+=("${to}|${from}")
-    return 0
-  fi
-  if [[ ! -r "$to" ]]; then
+  local from=$1 to=$2 how=$3 here lines
+  [[ -e "$from" && -e "$to" ]] || return 0
+  if ! here=$(broker_conf_cat "$to"); then
     UNREADABLE+=("$to")
     return 0
   fi
-  # Both sides through the same filter, so only a real difference survives. Note the "#"
-  # delimiter: an alternation inside an "s|...|" would end the expression at its first branch, and
-  # the "|| true" below would then hide the error and report the two files as identical.
-  local norm
   case "$how" in
-    plugin)  norm='s#^plugin .*#plugin -#' ;;
-    # The installer rewrites User, WorkingDirectory and ExecStart for whoever runs it and wherever
-    # the server went, so those three are normalised - comparing them exactly would report every
-    # installation that is not user "pi" in /home/pi/frugal-iot as out of date at every upgrade.
-    service) norm='s#^(User|WorkingDirectory|ExecStart)=.*#\1=-#' ;;
-    *)       norm='' ;;
-  esac
-
-  case "$how" in
-    plugin|service)
-      a=$(sed -E "$norm" "$from" | grep -vE '^[[:space:]]*(#|$)' | sort || true)
-      b=$(sed -E "$norm" "$to"   | grep -vE '^[[:space:]]*(#|$)' | sort || true)
-      # Neither of these files is ever empty once the comments are gone. Both blank means the
-      # filter failed rather than that they agree, and reporting "current" on that would be the
-      # worst answer of the three.
-      if [[ -z "$a" || -z "$b" ]]; then
-        UNREADABLE+=("$to")
-      elif [[ "$a" != "$b" ]]; then
-        STALE+=("${to}|${from}")
-      fi
-      ;;
     subset)
-      missing=$(comm -23 <(grep -vE '^[[:space:]]*(#|$)' "$from" | sort -u || true) \
-                         <(grep -vE '^[[:space:]]*(#|$)' "$to"   | sort -u || true) || true)
-      [[ -z "$missing" ]] || STALE+=("${to}|${from}")
+      lines=$(comm -23 <(norm_text exact < "$from" | sort -u) <(print -r -- "$here" | norm_text exact | sort -u) |
+        sed 's/^/release only: /') || true
       ;;
     *)
-      cmp -s "$from" "$to" || STALE+=("${to}|${from}")
+      # "here" and "release", from diff's < and >; its hunk headers say nothing to a reader
+      lines=$(diff <(print -r -- "$here" | norm_text "$how") <(norm_text "$how" < "$from") |
+        sed -n -e 's/^< /here only:    /p' -e 's/^> /release only: /p') || true
       ;;
   esac
+  if [[ -n "$lines" ]]; then
+    STALE+=("${to}|${from}")
+    STALE_LINES+=("$lines")
+  fi
 }
 
-check_installed "${PKG}/extras/mosquitto.conf"    /etc/mosquitto/conf.d/frugal-iot.conf  plugin
-check_installed "${PKG}/extras/aclfile"           /etc/mosquitto/aclfile                 subset
-check_installed "${PKG}/extras/frugaliot.service" /etc/systemd/system/frugaliot.service  service
-# Installed as they are, so compared exactly. Absent on a machine with no WiFi, which install-pi.sh
-# leaves without a setup network - reported as "nothing to compare", which is what it is.
-check_installed "${PKG}/extras/frugaliot-ap-watchdog"         /usr/local/sbin/frugaliot-ap-watchdog             exact
-check_installed "${PKG}/extras/frugaliot-ap-watchdog.service" /etc/systemd/system/frugaliot-ap-watchdog.service  exact
-check_installed "${PKG}/extras/frugaliot-ap-watchdog.timer"   /etc/systemd/system/frugaliot-ap-watchdog.timer    exact
+# Where systemd keeps a unit, if it has one - /etc/systemd/system on a Pi, /usr/lib/systemd/system
+# on production
+unit_path() {
+  command -v systemctl >/dev/null 2>&1 || return 0
+  systemctl show -p FragmentPath --value "$1" 2>/dev/null || true
+}
+
+# The broker: what it reads, asked of its own configuration
+source "${PKG}/scripts/broker-config.zsh"
+broker_conf_read
+BROKER_FINDINGS=""
+if (( ${#BROKER_CONF_FILES} )); then
+  BROKER_FINDINGS=$(broker_conf_compare "${PKG}/extras/mosquitto.conf")
+  broker_setting acl_file
+  [[ -n "$BROKER_SETTING" ]] && check_installed "${PKG}/extras/aclfile" "$BROKER_SETTING" subset
+fi
+
+SERVICE_UNIT=$(unit_path frugaliot.service)
+[[ -n "$SERVICE_UNIT" ]] && check_installed "${PKG}/extras/frugaliot.service" "$SERVICE_UNIT" service
+# The setup network's watchdog exists only on a Pi with WiFi that install-pi.sh gave one. Anywhere
+# else it is simply not there, and saying so at every upgrade was noise.
+WATCHDOG_UNIT=$(unit_path frugaliot-ap-watchdog.service)
+if [[ -n "$WATCHDOG_UNIT" ]]; then
+  check_installed "${PKG}/extras/frugaliot-ap-watchdog.service" "$WATCHDOG_UNIT" exact
+  check_installed "${PKG}/extras/frugaliot-ap-watchdog.timer" "$(unit_path frugaliot-ap-watchdog.timer)" exact
+  check_installed "${PKG}/extras/frugaliot-ap-watchdog" /usr/local/sbin/frugaliot-ap-watchdog exact
+fi
 
 if (( ${#DIFFER_TO} )); then
   echo
-  echo "These files were left as you have them, but this release ships a different version:"
+  echo "Kept as you have them, because they have lines this release's version does not:"
   for i in {1..${#DIFFER_TO}}; do
     echo "  ${DIFFER_TO[$i]}"
-    echo "    compare with:  diff ${DIFFER_TO[$i]} ${DIFFER_FROM[$i]}"
+    print -r -- "${DIFFER_OURS[$i]}" | head -10 | sed 's/^/      /'
+    (( $(print -r -- "${DIFFER_OURS[$i]}" | wc -l) > 10 )) && echo "      ..."
   done
-  echo "The schema describes the sensor types the software understands. If you added one of your"
-  echo "own that is exactly as it should be; if you did not, this release changed it and the"
-  echo "difference is worth reading."
+  echo "If you added those, nothing needs doing. If you did not, this release changed or removed"
+  echo "them, and taking its version is safe:"
+  for i in {1..${#DIFFER_TO}}; do echo "  cp ${DIFFER_FROM[$i]} ${DIFFER_TO[$i]}"; done
 fi
 
-if (( ${#STALE} )); then
+if [[ -n "$BROKER_FINDINGS" ]]; then
   echo
-  echo "Installed elsewhere from extras/, and older than this release:"
-  for e in $STALE; do
-    echo "  ${e%%|*}"
-    echo "    compare with:  sudo diff ${e%%|*} ${e#*|}"
+  echo "The broker's configuration, compared setting by setting with this release's"
+  echo "extras/mosquitto.conf, across the ${#BROKER_CONF_FILES} file(s) it reads:"
+  BROKER_REAL=0
+  for l in ${(f)BROKER_FINDINGS}; do
+    case ${l%%$'\t'*} in
+      missing)   echo "  missing:     ${l#*$'\t'}"; (( BROKER_REAL++ )) || true ;;
+      differs)   echo "  different:   ${l#*$'\t'}"; (( BROKER_REAL++ )) || true ;;
+      elsewhere) echo "  (fine)       ${l#*$'\t'}" ;;
+    esac
   done
-  echo "Re-running the installer brings these up to date and is the safe way to do it - it"
-  echo "substitutes the plugin path and will not discard ACL rules added per bridge:"
-  echo "  bash node_modules/frugal-iot-server/scripts/install-pi.sh"
+  if (( BROKER_REAL )); then
+    echo "To bring it in line, put the settings above in any file in /etc/mosquitto/conf.d/ - on a Pi"
+    echo "set up by install-pi.sh, re-running it does that - then: sudo systemctl restart mosquitto"
+  fi
 fi
+(( ${#BROKER_CONF_UNREADABLE} )) &&
+  { echo; echo "Broker configuration not readable as $(id -un), so the comparison above is incomplete: ${(j:, :)BROKER_CONF_UNREADABLE}"; }
 
-if (( ${#UNINSTALLED} )); then
+if (( ${#STALE} )); then
   echo
-  echo "Nothing at the usual place, so these were not compared:"
-  for e in $UNINSTALLED; do echo "  ${e%%|*}  (would come from extras/${${e#*|}:t})"; done
-  echo "Either install-pi.sh has not been run here, or this machine's broker configuration was put"
-  echo "together by hand under another name - several small files in /etc/mosquitto/conf.d is a"
-  echo "normal way to have done it. Nothing is out of date; there is simply nothing to compare."
+  echo "Installed elsewhere from extras/, and different from this release:"
+  for i in {1..${#STALE}}; do
+    echo "  ${STALE[$i]%%|*}"
+    print -r -- "${STALE_LINES[$i]}" | sed 's/^/      /'
+  done
+  echo "Bring them in line by hand - change the \"here only\" lines to the \"release only\" ones - or on"
+  echo "a Pi set up by it, by re-running install-pi.sh. The whole difference, if you want it:"
+  for e in $STALE; do echo "  sudo diff ${e%%|*} ${e#*|}"; done
 fi
 
 if (( ${#UNREADABLE} )); then
   echo
-  echo "There, but not readable as $(id -un), so not compared: ${(j:, :)UNREADABLE}"
+  echo "There, but not readable as $(id -un) without a sudo password, so not compared: ${(j:, :)UNREADABLE}"
+  echo "To compare them too: sudo -v, then run this again"
 fi
 
 echo

package/scripts/not-published.txt

@@ -18,6 +18,7 @@ extras/mosquitto_passwords
 .gitignore
 claude.md
 create-favicon.js
+BACKUP_PLAN.md
 jest.config.js
 vitest.config.js
 test/**

package/scripts/openwrt-preflight.sh

@@ -156,8 +156,20 @@ info "install directory ${DIR} is on ${DMNT} (${DFS}), ${DFREE} MB free"
 case "$DFS" in
   vfat|exfat|msdos|ntfs|fuseblk) fail "${DIR} is on ${DFS}, which cannot hold the install. Put it on an ext4 drive: --dir /mnt/<drive>/frugal-iot" ;;
   *) if [ "$DFREE" -lt 400 ]; then
-       fail "only ${DFREE} MB free for ${DIR}. Attach a USB/SSD drive (extroot, or mount it) and use --dir"
-       if ! ls /dev/sd[a-z]* /dev/mmcblk[1-9]* /dev/nvme* >/dev/null 2>&1; then
+       # A drive that is already mounted is the usual answer: the --dir may simply name the wrong mount
+       # point (/mnt/sda1 when the drive is at /mnt/sda). Suggest any real filesystem with room.
+       BIGMNT=$(awk '$3 ~ /^(ext2|ext3|ext4|f2fs|btrfs|xfs)$/ && $2 != "/" && $2 !~ /^\/(overlay|rom)/ {print $2}' /proc/mounts | sort -u | while read -r m; do
+         [ "$(free_mb "$m")" -ge 400 ] 2>/dev/null && { echo "$m"; break; }; done)
+       if [ -n "$BIGMNT" ]; then
+         fail "only ${DFREE} MB free for ${DIR}, which is not on the drive. ${BIGMNT} has room: try --dir ${BIGMNT}/frugal-iot"
+       else
+         fail "only ${DFREE} MB free for ${DIR}. Attach a USB/SSD drive (extroot, or mount it) and use --dir"
+       fi
+       # (ls with several patterns fails if ANY has no match, so test each separately)
+       SEEN_DRIVE=0
+       for d in /dev/sd[a-z]* /dev/mmcblk[1-9]* /dev/nvme*; do [ -e "$d" ] && SEEN_DRIVE=1; done
+       if [ -n "$BIGMNT" ]; then :
+       elif [ "$SEEN_DRIVE" = 0 ]; then
          # Seen on a real router: the drive was plugged in and formatted ext4, and invisible, because
          # the image carried no USB storage driver.
          info "No USB, SD or NVMe drive is visible at all. If one is plugged in, this OpenWrt image has no"