frugal-iot-server 2.0.11 → 2.0.12

package/scripts/backup.zsh

@@ -0,0 +1,361 @@
+#!/usr/bin/env zsh
+#
+# Take a snapshot of this installation's STATE - everything that cannot be rebuilt from the
+# package, a release or the database - as one archive. See BACKUP_PLAN.md for what is in it and why.
+#
+# Usage, from the server's own directory (or anywhere, once the frugaliot service is installed):
+#   npx --no frugal-iot-backup check       # say what would be taken, from where; writes nothing
+#   npx --no frugal-iot-backup             # take one, into /var/backups/frugal-iot
+#   npx --no frugal-iot-backup <dir>       # take one into <dir> (an absolute path), pruning nothing there
+#   npx --no frugal-iot-backup install     # run it nightly, from a systemd timer
+#
+# Positional, never --flags: npm swallows those (see "Shell scripts" in CLAUDE.md).
+#
+# Not in the snapshot:
+#   data/   large and append-only - mirrored by rsync instead (BACKUP_PLAN.md, "Two streams")
+#   ota/    rebuilt at every release
+#   anything mosquitto can rebuild or is running on: its plugin, retained messages, its log
+#
+# A snapshot holds every credential this server has - the db, secrets.yaml, the broker's password
+# file, a TLS key - so it is mode 600, owned by the account the server runs as, in a directory
+# nobody else can enter. Taking one needs root, because several of those are root- or
+# mosquitto-only; run as anyone else it re-runs itself with sudo. "check" does not need root.
+#
+# The broker's files are found by asking its own configuration (scripts/broker-config.zsh), never
+# assumed: production is laid out nothing like a Pi.
+
+set -euo pipefail
+zmodload zsh/datetime
+
+SCRIPT="${0:A}"
+
+usage() {
+  sed -n '6,11p' "$SCRIPT" | sed 's/^# //' >&2
+  exit 2
+}
+
+MODE=backup
+DEST=/var/backups/frugal-iot
+PRUNE=1     # only the timer's own directory - see "prune" below
+case "${1:-}" in
+  "")              ;;
+  check)           MODE=check ;;
+  install)         MODE=install ;;
+  help|-h|--help)  usage ;;
+  /*)              DEST=$1; PRUNE=0 ;;
+  *)               echo "Unexpected argument: $1" >&2; usage ;;
+esac
+(( $# <= 1 )) || usage
+
+# How many to keep. Each tier keeps the newest snapshot in each of its most recent N periods that
+# have one - so with the nightly timer: a week of dailies, a month of weeklies, three months of
+# monthlies, a year of quarterlies, and one a year for ever. A snapshot may count in several tiers.
+# Counting periods that HAVE a snapshot, rather than calendar periods, means a timer that stopped
+# for a while does not get its last good snapshots pruned away when it starts again.
+KEEP_DAILY=7
+KEEP_WEEKLY=4
+KEEP_MONTHLY=3
+KEEP_QUARTERLY=4
+KEEP_YEARLY=0       # 0 = no limit
+
+have() { command -v "$1" >/dev/null 2>&1; }
+die()  { echo "Error: $*" >&2; exit 1; }
+# Owner and mode of a file, spelled for Linux or BSD/macOS stat
+ownmode() { stat -c '%U:%G %a' "$1" 2>/dev/null || stat -f '%Su:%Sg %Lp' "$1" 2>/dev/null || echo "?:? ?"; }
+
+# ---- Where is the installation? ----
+# The current directory if it holds the database, as for every other script here; otherwise wherever
+# the frugaliot service runs - which is what lets the timer run this from anywhere.
+SERVICE_UNIT=""
+if have systemctl; then
+  SERVICE_UNIT=$(systemctl show -p FragmentPath --value frugaliot 2>/dev/null) || true
+fi
+if [[ -f ./frugal-iot.db ]]; then
+  INSTALL_DIR=$PWD
+else
+  INSTALL_DIR=""
+  if have systemctl; then
+    INSTALL_DIR=$(systemctl show -p WorkingDirectory --value frugaliot 2>/dev/null) || true
+  fi
+  [[ -n "$INSTALL_DIR" && -f "$INSTALL_DIR/frugal-iot.db" ]] ||
+    die "no frugal-iot.db here, and no frugaliot service saying where one is - run this from the server's own directory"
+fi
+cd "$INSTALL_DIR"
+have sqlite3 || die "sqlite3 command not found - install it with: sudo apt install sqlite3"
+
+# The account the server runs as owns the database, and will own the snapshots - so the offsite pull
+# can log in as that account and needs nothing more. Not $USER: under sudo or the timer that is root.
+# Quoted: unquoted, the substitution is split into words and the %% applies to each - "mitra 644"
+SERVICE_USER=${"$(ownmode frugal-iot.db)"%%:*}
+# The machine's own name, for telling snapshots apart once they are all on one laptop. Not $HOST,
+# which zsh ties to the real hostname and is not ours to assign (see CLAUDE.md).
+SNAP_HOST=$(hostname -s 2>/dev/null || hostname)
+
+# ---- What would be taken ----
+# Parallel arrays, one entry per file or directory:
+#   ITEM_SRC   absolute path it comes from
+#   ITEM_DST   path inside the archive: install/... for this installation, fs/<absolute path> otherwise
+#   ITEM_ROLE  what it is: install, service, config (a broker configuration file), or for a file the
+#              broker's configuration names, the setting that names it - password_file, keyfile... -
+#              because that is what lets a restore put it wherever ANOTHER machine's broker keeps it
+ITEM_SRC=(); ITEM_DST=(); ITEM_ROLE=()
+typeset -A TAKEN
+add_item() {  # role src dst
+  [[ -n "${TAKEN[$2]:-}" ]] && return 0
+  TAKEN[$2]=1
+  ITEM_ROLE+=("$1"); ITEM_SRC+=("$2"); ITEM_DST+=("$3")
+}
+add_item install "$INSTALL_DIR/frugal-iot.db" install/frugal-iot.db
+add_item install "$INSTALL_DIR/config.yaml"   install/config.yaml
+add_item install "$INSTALL_DIR/config.d"      install/config.d
+[[ -n "$SERVICE_UNIT" && -e "$SERVICE_UNIT" ]] && add_item service "$SERVICE_UNIT" "fs$SERVICE_UNIT"
+
+source "${SCRIPT:h}/broker-config.zsh"
+broker_conf_read
+for f in $BROKER_CONF_FILES; do add_item config "$f" "fs$f"; done
+SKIPPED=()
+while read -r setting named role; do
+  [[ -n "${named:-}" ]] || continue
+  case $role in
+    backup|tls)  add_item "$setting" "$named" "fs$named" ;;
+    # Every account in it is rebuilt from the db - except the admin one the rebuild logs in AS,
+    # whose password is in secrets.yaml. So a restore puts this back alongside secrets.yaml, and
+    # the two agree; restoring secrets.yaml alone onto another broker locks the server out of it.
+    rebuildable) add_item "$setting" "$named" "fs$named" ;;
+    *)           SKIPPED+=("$setting $named ($role)") ;;
+  esac
+done < <(broker_named_files)
+
+# ---- Root, when it is needed ----
+# Installing the timer always needs it. Taking a snapshot needs it whenever anything to be taken is
+# not readable as we are - on any machine with a broker, that is its password file at least.
+as_root() {
+  have sudo || die "this needs root, and there is no sudo here - run it as root"
+  echo "Re-running as root, which can read the broker's files..." >&2
+  exec sudo -- zsh "$SCRIPT" "$@"
+}
+if (( EUID != 0 )); then
+  if [[ $MODE == install ]]; then
+    as_root "$@"
+  elif [[ $MODE == backup ]]; then
+    (( ${#BROKER_CONF_UNREADABLE} )) && as_root "$@"
+    for f in $ITEM_SRC; do
+      [[ ! -e "$f" || -r "$f" ]] || as_root "$@"
+    done
+  fi
+fi
+
+# Can each be read, and does it exist at all? A root-only file is only a problem if we are root and
+# still cannot see it; in "check" as an ordinary user it is exactly what the timer, as root, will take.
+PROBLEMS=()
+item_state() {  # src -> "ok", "needs root" or "MISSING"
+  if [[ -r "$1" ]]; then echo ok
+  elif [[ -e "$1" ]] || { (( EUID != 0 )) && have sudo && sudo -n test -e "$1" 2>/dev/null; }; then echo "needs root"
+  elif (( EUID != 0 )) && have sudo && ! sudo -n true 2>/dev/null; then echo "cannot tell without sudo"
+  else echo MISSING
+  fi
+}
+
+# ---- check ----
+if [[ $MODE == check ]]; then
+  echo "Installation:  $INSTALL_DIR (runs as $SERVICE_USER)"
+  echo "Snapshots go:  $DEST"
+  echo
+  echo "Would take:"
+  for i in {1..${#ITEM_SRC}}; do
+    st=$(item_state "$ITEM_SRC[$i]")
+    if [[ $st == ok ]]; then
+      size=$(du -sh "$ITEM_SRC[$i]" 2>/dev/null | awk '{print $1}') || true
+      st="ok, ${size:-?}, $(ownmode "$ITEM_SRC[$i]")"
+    fi
+    printf '  %-22s %s: %s\n' "$ITEM_ROLE[$i]" "$ITEM_SRC[$i]" "$st"
+    [[ $st == MISSING ]] && PROBLEMS+=("$ITEM_SRC[$i] is named by the configuration but does not exist")
+  done
+  (( ${#BROKER_CONF_FILES} )) || echo "  (no mosquitto configuration at $(broker_conf_main) - nothing of the broker's to take)"
+  if (( ${#BROKER_CONF_UNREADABLE} )); then
+    echo "  NOT READABLE without sudo: ${(j:, :)BROKER_CONF_UNREADABLE} - files it names are missing above"
+  fi
+  if (( ${#SKIPPED} )); then
+    echo
+    echo "Named by the broker's configuration, deliberately not taken:"
+    for s in $SKIPPED; do echo "  $s"; done
+  fi
+  echo
+  echo "Not taken: data/ (mirrored separately) and ota/ (rebuilt each release)."
+  echo
+  if [[ -d "$DEST" ]] || { have sudo && sudo -n test -d "$DEST" 2>/dev/null; }; then
+    SNAPS=$( (ls -1 "$DEST" 2>/dev/null || sudo -n ls -1 "$DEST" 2>/dev/null) | grep -c '^frugal-iot-state-.*\.tar\.gz$') || true
+    echo "Existing snapshots in $DEST: ${SNAPS:-0} ($(ownmode "$DEST"))"
+  else
+    echo "$DEST does not exist yet - the first snapshot creates it"
+  fi
+  df -Pk "${DEST:h}" 2>/dev/null | awk 'NR==2 {printf "Free space there: %d MB\n", $4/1024}' || true
+  if have systemctl; then
+    # is-enabled prints "not-found" AND fails for a unit that does not exist, so test its answer,
+    # not its exit status - or both get printed
+    TIMER_STATE=$(systemctl is-enabled frugal-iot-backup.timer 2>/dev/null) || true
+    [[ -z "$TIMER_STATE" || "$TIMER_STATE" == not-found ]] && TIMER_STATE="not installed - npx --no frugal-iot-backup install"
+    echo "Timer: $TIMER_STATE"
+  fi
+  if (( ${#PROBLEMS} )); then
+    echo
+    for p in $PROBLEMS; do echo "PROBLEM: $p"; done
+    exit 1
+  fi
+  exit 0
+fi
+
+# ---- install: a nightly systemd timer ----
+# Written here rather than shipped in extras/, because the paths in it are this machine's: where the
+# installation is, where zsh is. A changed extras/ file never reaches /etc on its own anyway (see
+# CLAUDE.md), so generating it is also what keeps a re-run current.
+if [[ $MODE == install ]]; then
+  have systemctl || die "no systemd here - run '$SCRIPT' nightly from cron instead"
+  TMPDIR_UNITS=$(mktemp -d)
+  trap 'rm -rf "$TMPDIR_UNITS"' EXIT
+  cat > "$TMPDIR_UNITS/frugal-iot-backup.service" <<EOF
+# Written by frugal-iot-backup install - re-run that rather than editing this.
+[Unit]
+Description=Frugal IoT state snapshot (frugal-iot-backup)
+After=frugaliot.service
+
+[Service]
+Type=oneshot
+WorkingDirectory=${INSTALL_DIR}
+ExecStart=$(command -v zsh) ${SCRIPT}
+# Root, which is the point: it reads the broker's root- and mosquitto-only files
+Nice=10
+IOSchedulingClass=idle
+EOF
+  cat > "$TMPDIR_UNITS/frugal-iot-backup.timer" <<EOF
+# Written by frugal-iot-backup install - re-run that rather than editing this.
+[Unit]
+Description=Nightly Frugal IoT state snapshot
+
+[Timer]
+OnCalendar=*-*-* 03:17:00
+RandomizedDelaySec=20min
+# A Pi switched off at 03:17 takes the missed one when it next boots
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+EOF
+  CHANGED=0
+  for u in frugal-iot-backup.service frugal-iot-backup.timer; do
+    if cmp -s "$TMPDIR_UNITS/$u" "/etc/systemd/system/$u" 2>/dev/null; then
+      echo "  kept    /etc/systemd/system/$u (already current)"
+    else
+      install -o root -g root -m 644 "$TMPDIR_UNITS/$u" "/etc/systemd/system/$u"
+      echo "  wrote   /etc/systemd/system/$u"
+      CHANGED=1
+    fi
+  done
+  (( CHANGED )) && systemctl daemon-reload
+  systemctl enable --now frugal-iot-backup.timer
+  systemctl list-timers frugal-iot-backup.timer --no-pager || true
+  echo
+  echo "Take the first one now, rather than waiting for tonight:"
+  echo "  sudo systemctl start frugal-iot-backup && journalctl -u frugal-iot-backup -n 20 --no-pager"
+  exit 0
+fi
+
+# ---- backup ----
+umask 077
+for i in {1..${#ITEM_SRC}}; do
+  [[ -e "$ITEM_SRC[$i]" ]] || die "$ITEM_SRC[$i] ($ITEM_ROLE[$i]) does not exist - run 'frugal-iot-backup check'"
+done
+(( ${#BROKER_CONF_UNREADABLE} == 0 )) || die "could not read ${(j:, :)BROKER_CONF_UNREADABLE}, even as root"
+
+mkdir -p "$DEST"
+chown "$SERVICE_USER" "$DEST"
+chmod 700 "$DEST"
+
+STAMP=$(strftime '%Y%m%d-%H%M%S' $EPOCHSECONDS)
+NAME="frugal-iot-state-${SNAP_HOST}-${STAMP}.tar.gz"
+# Built beside the destination and renamed into place, so a snapshot that exists is a whole one -
+# a run killed part-way leaves only the hidden staging directory, removed on exit
+STAGE=$(mktemp -d "$DEST/.staging.XXXXXX")
+trap 'rm -rf "$STAGE"' EXIT
+SNAP="$STAGE/snap"
+mkdir -p "$SNAP/install"
+
+for i in {1..${#ITEM_SRC}}; do
+  src=$ITEM_SRC[$i]; dst="$SNAP/$ITEM_DST[$i]"
+  mkdir -p "${dst:h}"
+  if [[ $ITEM_DST[$i] == install/frugal-iot.db ]]; then
+    # Never cp a database the server is writing to: .backup takes a consistent copy while it runs
+    sqlite3 "$src" ".backup '${dst}'"
+    INTEGRITY=$(sqlite3 "$dst" 'PRAGMA integrity_check;') || true
+    [[ "$INTEGRITY" == ok ]] || die "the copied database fails its integrity check: ${INTEGRITY:-no answer}"
+  else
+    cp -Rp -- "$src" "$dst"
+  fi
+done
+
+# What a restore needs to know that the archive does not say by itself: where each thing came from
+# and who owned it - a laptop extracting this as an ordinary user loses the owners.
+{
+  echo "# Frugal IoT state snapshot - see BACKUP_PLAN.md"
+  # 2: the role of a broker file is the setting that named it. frugal-iot-restore refuses others.
+  echo "format	2"
+  echo "created	$(strftime '%Y-%m-%dT%H:%M:%S%z' $EPOCHSECONDS)"
+  echo "host	$(hostname 2>/dev/null)"
+  echo "install_dir	$INSTALL_DIR"
+  echo "service_user	$SERVICE_USER"
+  for p in frugal-iot-server frugal-iot-client frugal-iot-logger; do
+    v=$(node -e "console.log(require('./node_modules/$p/package.json').version)" 2>/dev/null) || true
+    echo "$p	${v:-not installed here}"
+  done
+  v=""
+  have mosquitto && { v=$( (set +o pipefail; mosquitto -h 2>&1 | head -1) ) || true; }
+  echo "mosquitto	${v:-not installed here}"
+  echo "broker_conf	$(broker_conf_main)"
+  echo "tables	$(sqlite3 "$SNAP/install/frugal-iot.db" "SELECT group_concat(name, ' ') FROM sqlite_master WHERE type='table';")"
+  echo "# file	role	owner:group mode	from	in archive"
+  for i in {1..${#ITEM_SRC}}; do
+    echo "file	$ITEM_ROLE[$i]	$(ownmode "$ITEM_SRC[$i]")	$ITEM_SRC[$i]	$ITEM_DST[$i]"
+  done
+  for s in $SKIPPED; do echo "skipped	$s"; done
+} > "$SNAP/MANIFEST"
+
+tar -czf "$STAGE/$NAME" -C "$SNAP" .
+# Read it back before trusting it: a truncated archive on a full disk otherwise looks like a backup
+tar -tzf "$STAGE/$NAME" >/dev/null || die "the archive just written cannot be read back"
+chown "$SERVICE_USER" "$STAGE/$NAME"
+chmod 600 "$STAGE/$NAME"
+mv "$STAGE/$NAME" "$DEST/$NAME"
+echo "wrote $DEST/$NAME ($(du -h "$DEST/$NAME" | awk '{print $1}'), ${#ITEM_SRC} items)"
+
+# ---- prune ----
+# Only in the default directory: one named on the command line is the caller's. frugal-iot-restore
+# keeps its undo points in pre-restore/, and pruning there to one a day deleted the previous undo
+# point - the snapshot a second restore that day was being asked to put back.
+(( PRUNE )) || exit 0
+# Only this machine's snapshots, by name - so a directory that also holds copies pulled from
+# elsewhere, or anything else at all, is left alone. Newest first; names sort by time.
+typeset -A SEEN COUNT
+# tier <name> <period> <limit>: keep this snapshot if it is the first (so newest) seen in its period,
+# and that tier has not yet filled its limit of periods
+tier() {
+  [[ -z "${SEEN[$1:$2]:-}" ]] || return 0
+  (( $3 == 0 || ${COUNT[$1]:-0} < $3 )) || return 0
+  SEEN[$1:$2]=1
+  COUNT[$1]=$(( ${COUNT[$1]:-0} + 1 ))
+  keep=1
+}
+for f in "$DEST"/frugal-iot-state-${SNAP_HOST}-<->-<->.tar.gz(N.On); do
+  stamp=${${f:t}#frugal-iot-state-${SNAP_HOST}-}
+  day=${stamp[1,8]}
+  strftime -r -s secs '%Y%m%d' "$day" 2>/dev/null || continue
+  keep=0
+  tier daily     "$day"                                          $KEEP_DAILY
+  tier weekly    $(( secs / 604800 ))                            $KEEP_WEEKLY
+  tier monthly   "${day[1,6]}"                                   $KEEP_MONTHLY
+  tier quarterly "${day[1,4]}q$(( (${day[5,6]} - 1) / 3 + 1 ))"  $KEEP_QUARTERLY
+  tier yearly    "${day[1,4]}"                                   $KEEP_YEARLY
+  if (( ! keep )); then
+    rm -f -- "$f"
+    echo "pruned ${f:t}"
+  fi
+done

package/scripts/broker-config.zsh

@@ -0,0 +1,185 @@
+# Where this machine's mosquitto keeps its configuration and files - asked of the broker's own
+# configuration, never assumed. Sourced, not run:
+#
+#   source "${0:A:h}/broker-config.zsh"     # ${0:A} resolves the node_modules/.bin symlink
+#   broker_conf_read
+#   broker_setting password_file; PWFILE="$BROKER_SETTING"
+#
+# Machines are not laid out alike. A Pi from install-pi.sh has conf.d/frugal-iot.conf and its
+# password file in /var/lib/mosquitto; production was assembled by hand as several files in conf.d
+# with the password file in /etc/mosquitto; OpenWrt writes its own mosquitto.conf. Three scripts each
+# had a copy of this, each slightly different, and the one thing they all got wrong is below.
+#
+# A conf.d file can be readable by root only - a Pi's bridge configuration holds a password, so it
+# should be. Reading all the files in one grep or cat then gets the readable ones and silently skips
+# that one, which is not empty and so never triggered the sudo retry - and a setting in the skipped
+# file was reported as whatever an earlier file said. So each file is read on its own, with sudo when
+# it cannot be read plainly, and a file that still cannot be read is listed in
+# BROKER_CONF_UNREADABLE so the caller can say the answer is incomplete instead of being wrong.
+#
+# Set before broker_conf_read, if the defaults do not suit:
+#   BROKER_CONF  the main configuration file. Default: the -c of the running service, else
+#                /etc/mosquitto/mosquitto.conf. Also how the tests point this at a fixture.
+#   BROKER_SUDO  how to retry an unreadable file. Default (sudo -n), which never prompts - right for
+#                the diagnostic's quiet mode; a script that may prompt sets (sudo). Unused as root.
+#
+# Results, after broker_conf_read:
+#   BROKER_CONF_FILES       every file the broker reads, in the order it reads them
+#   BROKER_CONF_UNREADABLE  those of them that could not be read even with BROKER_SUDO
+#   BROKER_CONF_TEXT        their contents, comments and blank lines removed, in order
+
+: ${BROKER_CONF:=}
+(( ${+BROKER_SUDO} )) || BROKER_SUDO=(sudo -n)
+BROKER_CONF_FILES=()
+BROKER_CONF_UNREADABLE=()
+BROKER_CONF_TEXT=""
+BROKER_SETTING=""
+
+# The main configuration file. mosquitto has no default of its own - it reads nothing without -c -
+# so the service's command line is the authority, and /etc/mosquitto/mosquitto.conf only the
+# fallback when there is no systemd to ask (OpenWrt) or the service is not installed.
+broker_conf_main() {
+  local exec_start
+  if [[ -n "$BROKER_CONF" ]]; then print -r -- "$BROKER_CONF"; return; fi
+  if command -v systemctl >/dev/null 2>&1; then
+    exec_start=$(systemctl show -p ExecStart mosquitto 2>/dev/null) || true
+    if [[ "$exec_start" =~ ' -c ([^ ;]+)' ]]; then print -r -- "${match[1]}"; return; fi
+  fi
+  print -r -- /etc/mosquitto/mosquitto.conf
+}
+
+# One file's contents, with sudo if it cannot be read plainly. Fails if neither works.
+broker_conf_cat() {
+  local f=$1
+  if [[ -r "$f" ]]; then
+    cat -- "$f" 2>/dev/null
+  elif (( EUID != 0 && ${#BROKER_SUDO} )) && command -v "${BROKER_SUDO[1]}" >/dev/null 2>&1; then
+    "${BROKER_SUDO[@]}" cat -- "$f" 2>/dev/null
+  else
+    return 1
+  fi
+}
+
+broker_conf_read() {
+  local main dir f t
+  local -a dirs
+  BROKER_CONF_FILES=(); BROKER_CONF_UNREADABLE=(); BROKER_CONF_TEXT=""
+  main=$(broker_conf_main)
+  [[ -e "$main" ]] || return 0
+  BROKER_CONF_FILES=("$main")
+  # include_dir is only honoured in the main file, and mosquitto loads a directory's *.conf files in
+  # case-sensitive alphabetical order - hence the C collation. Files not ending .conf are ignored by
+  # the broker, so a stray frugal-iot.conf.bak is ignored here too.
+  t=$(broker_conf_cat "$main") || { BROKER_CONF_UNREADABLE=("$main"); return 0; }
+  BROKER_CONF_TEXT=$(print -r -- "$t" | grep -vE '^[[:space:]]*(#|$)') || true
+  dirs=(${(f)"$(print -r -- "$BROKER_CONF_TEXT" | awk '$1 == "include_dir" {print $2}')"})
+  for dir in $dirs; do
+    BROKER_CONF_FILES+=(${(f)"$(LC_COLLATE=C; print -rl -- $dir/*.conf(N))"})
+  done
+  for f in ${BROKER_CONF_FILES[2,-1]}; do
+    if t=$(broker_conf_cat "$f"); then
+      # "|| true": a file of nothing but comments makes grep fail, which under the callers'
+      # "set -e -o pipefail" would end them silently
+      t=$(print -r -- "$t" | grep -vE '^[[:space:]]*(#|$)') || true
+      BROKER_CONF_TEXT+=$'\n'"$t"
+    else
+      BROKER_CONF_UNREADABLE+=("$f")
+    fi
+  done
+}
+
+# Every configuration line whose first word matches a pattern, in the order the broker reads them:
+#   broker_conf_lines 'connection|address'
+broker_conf_lines() {
+  print -r -- "$BROKER_CONF_TEXT" | grep -E "^[[:space:]]*($1)([[:space:]]|$)" || true
+}
+
+# The value of a setting into BROKER_SETTING - a variable rather than output, so that it can be
+# called without a subshell. The last occurrence wins, as it does in mosquitto, for the settings
+# where that is true (password_file, acl_file, plugin...); for per-listener ones it is only the last
+# listener's, and broker_conf_lines is the thing to use.
+broker_setting() {
+  BROKER_SETTING=$(broker_conf_lines "$1" | tail -1 | awk '{print $2}') || true
+}
+
+# Files the configuration names, one "setting path role" per line, wherever this machine keeps them.
+# The role is what a backup does with it - see BACKUP_PLAN.md:
+#   backup       not recoverable any other way, so taken and put back on restore
+#   tls          taken: a certificate can be re-issued, but production's were placed by hand, and
+#                restoring them saves every bridge an outage. keyfile is a secret like the db.
+#   rebuildable  frugal-iot-rebuild-dynsec recreates its accounts from the db - all but the admin
+#                one it logs in as, whose password is in secrets.yaml, so it is taken all the same
+#   package      part of the mosquitto install, not of this installation
+#   live         the broker's running state or its log - stale the moment it is copied
+broker_named_files() {
+  broker_conf_lines 'password_file|acl_file' | awk '{print $1, $2, "backup"}'
+  broker_conf_lines 'certfile|keyfile|cafile|capath|dhparamfile|crlfile' | awk '{print $1, $2, "tls"}'
+  broker_conf_lines 'plugin_opt_config_file' | awk '{print $1, $2, "rebuildable"}'
+  broker_conf_lines 'plugin' | awk '{print $1, $2, "package"}'
+  broker_conf_lines 'persistence_location' | awk '{print $1, $2, "live"}'
+  broker_conf_lines 'log_dest' | awk '$2 == "file" {print "log_dest", $3, "live"}'
+}
+
+# Compare a shipped configuration (extras/mosquitto.conf) with what this broker actually reads,
+# setting by setting - not file against file. Production's configuration is several hand-written
+# files in conf.d and a Pi's is one frugal-iot.conf, and the same settings in a different file, a
+# different order or with different comments are the same configuration. One finding per line:
+#
+#   missing<TAB>text     the release sets something this broker does not set at all
+#   differs<TAB>text     both set it, to different values
+#   elsewhere<TAB>text   a file the release names at one path and this broker at another - normal,
+#                        and only information: a hand-built broker keeps its files where it likes
+#
+# Nothing printed means nothing to say. Three kinds of setting, because they differ in what counts:
+#  - listener and protocol: a listener is a port, and the protocol line after it belongs to it -
+#    so "listener 1883 0.0.0.0" is the release's "listener 1883", and its protocol is compared there
+#  - a setting whose value is a path (or the installer's PLUGIN_PATH placeholder): only that it is
+#    set; where is this machine's business
+#  - anything else: the value, last one read winning, as in mosquitto
+broker_conf_compare() {
+  local shipped=$1 line d v port
+  local -a w
+  local -A ship_proto here_proto ship_val here_val
+  local -a ship_order
+  local -A seen
+  # Shipped side
+  port=""
+  for line in ${(f)"$(grep -vE '^[[:space:]]*(#|$)' "$shipped" 2>/dev/null)"}; do
+    w=(${=line}); d=$w[1]; v="${w[2,-1]}"
+    case $d in
+      listener) port=$w[2]; ship_proto[$port]=mqtt ;;
+      protocol) [[ -n "$port" ]] && ship_proto[$port]=$w[2] ;;
+      *)        ship_val[$d]=$v; (( ${+seen[$d]} )) || { seen[$d]=1; ship_order+=("$d"); } ;;
+    esac
+  done
+  # This broker's side, in the order it reads its files
+  port=""
+  for line in ${(f)BROKER_CONF_TEXT}; do
+    w=(${=line}); d=${w[1]:-}; v="${w[2,-1]}"
+    [[ -n "$d" ]] || continue
+    case $d in
+      listener) port=$w[2]; here_proto[$port]=mqtt ;;
+      protocol) [[ -n "$port" ]] && here_proto[$port]=$w[2] ;;
+      *)        here_val[$d]=$v ;;
+    esac
+  done
+  for port in ${(ok)ship_proto}; do
+    if (( ! ${+here_proto[$port]} )); then
+      print -r -- "missing	listener $port ($ship_proto[$port]) - not declared"
+    elif [[ $here_proto[$port] != $ship_proto[$port] ]]; then
+      print -r -- "differs	listener $port - this release speaks $ship_proto[$port] on it, this broker $here_proto[$port]"
+    fi
+  done
+  for d in $ship_order; do
+    v=$ship_val[$d]
+    if (( ! ${+here_val[$d]} )); then
+      print -r -- "missing	$d $v - not set (mosquitto's default applies)"
+    elif [[ $v == /* || $v == *PLUGIN_PATH* ]]; then
+      [[ $here_val[$d] == $v || $v == *PLUGIN_PATH* ]] ||
+        print -r -- "elsewhere	$d - here $here_val[$d] (this release's default is $v)"
+    elif [[ $here_val[$d] != $v ]]; then
+      print -r -- "differs	$d - this release sets $v, this broker $here_val[$d]"
+    fi
+  done
+  return 0    # callers run under "set -e"; a comparison that found nothing has not failed
+}