frugal-iot-server 2.0.0 → 2.0.2

package/scripts/diagnostic.zsh

@@ -10,10 +10,40 @@
 #
 # Paste the whole output into a bug report. Passwords are deliberately not printed: the password
 # file is reported by account name only, and organization configs by name only.
-
+#
+# QUIET MODE says nothing at all unless something is wrong, which is what makes it usable as a
+# periodic check rather than only as something to read when you already have a problem:
+#   npx --no frugal-iot-diagnostic quiet        # positional, see below
+#   zsh diagnostic.zsh -q
+#
+# The word "quiet" with no dash is not a typo. npm parses the whole command line itself and passes
+# only the values through, so a flag never reaches a script run by npx - and "-q" is worse than
+# most, because it is npm's OWN abbreviation for --quiet, so npm consumes it and goes quiet itself.
+# "npx --no frugal-iot-diagnostic -- --quiet" also works, if you would rather remember that.
+#
+# Exit status, either mode:
+#   0  nothing found
+#   1  problems found - they are listed, and are the only thing a quiet run prints
+#
 # No "set -e": a failing check should report and carry on, not stop the script.
 setopt no_unset 2>/dev/null
 
+QUIET=0
+for arg in "$@"; do
+  case "$arg" in
+    -q|--quiet|quiet) QUIET=1 ;;
+    -h|--help|help)   sed -n '3,28p' "$0" | sed -e 's/^#$//' -e 's/^# //'; exit 0 ;;
+    *) print -u2 "Unknown argument: $arg (try \"quiet\", or \"help\")"; exit 2 ;;
+  esac
+done
+
+# Keep the real stdout on fd 3 for the summary, which prints in both modes. Under -q everything
+# else - including whatever the commands being run write - goes nowhere, so a quiet run that
+# prints anything is a quiet run that found something. stderr goes too: the noise it carries is
+# things like sudo declining, which the checks below already turn into findings of their own.
+exec 3>&1
+(( QUIET )) && exec 1>/dev/null 2>/dev/null
+
 section() { print -r -- ""; print -r -- "===== $* ====="; }
 item()    { print -r -- "  $*"; }
 have()    { command -v "$1" >/dev/null 2>&1; }
@@ -115,9 +145,33 @@ for f in /etc/mosquitto/conf.d/*(N); do
 done
 [[ -z "$(print -r -- /etc/mosquitto/conf.d/*(N))" ]] && problem "Nothing in /etc/mosquitto/conf.d - the Frugal IoT config was never copied there (step 5)"
 
+# The files the broker actually reads. mosquitto's include_dir only loads names ending in .conf,
+# so look at exactly those - and the last setting wins, hence "tail -1" everywhere below.
+BROKER_CONF_FILES=(/etc/mosquitto/mosquitto.conf(N) /etc/mosquitto/conf.d/*.conf(N))
+
+# Everything the broker reads, as one stream - with sudo if reading it plainly gives nothing. A
+# file under /etc can be readable only by root, and an ordinary user then gets silence rather than
+# an error, which is indistinguishable from "that setting is not configured". Left unguarded, the
+# comparison below reported a broker as missing EVERY setting this release ships, which as a
+# periodic check is worse than no check at all.
+broker_conf_text() {
+  local t
+  (( ${#BROKER_CONF_FILES} )) || return 0
+  t=$(cat "${BROKER_CONF_FILES[@]}" 2>/dev/null) || true
+  if [[ -z "$t" ]] && have sudo; then
+    t=$(sudo -n cat "${BROKER_CONF_FILES[@]}" 2>/dev/null) || true
+  fi
+  print -r -- "$t"
+}
+
+# Where the packaged copy of a shipped file is. What is in ./extras is NOT it: frugal-iot-init
+# copies extras/ in and then never overwrites it, so once a release changes one of these files the
+# local copy is a fossil. Checks below read the package.
+PKG_EXTRAS="node_modules/frugal-iot-server/extras"
+[[ -d "$PKG_EXTRAS" ]] || PKG_EXTRAS="${0:A:h:h}/extras"
+
 # Which password file does the running configuration actually name?
-# mosquitto's include_dir only loads files ending in .conf, so look at exactly those, last one wins
-CONFIGURED_PWFILE=$(grep -hE '^[[:space:]]*password_file[[:space:]]' /etc/mosquitto/mosquitto.conf /etc/mosquitto/conf.d/*.conf(N) 2>/dev/null | tail -1 | awk '{print $2}')
+CONFIGURED_PWFILE=$(broker_conf_text | grep -hE '^[[:space:]]*password_file[[:space:]]' | tail -1 | awk '{print $2}')
 section "Mosquitto password file"
 if [[ -n "$CONFIGURED_PWFILE" ]]; then
   item "configuration names: $CONFIGURED_PWFILE"
@@ -133,12 +187,35 @@ else
   item "no password_file line found in any mosquitto configuration"
   problem "No password_file configured - the broker would allow anonymous access or reject everything"
 fi
-# The copy in this installation is what step 5 tells you to put into /etc - if the two disagree,
-# either the copy never happened or the installed package is older than the instructions.
-if [[ -f extras/mosquitto.conf ]]; then
-  item "this installation's extras/mosquitto.conf names: $(grep -hE '^[[:space:]]*password_file' extras/mosquitto.conf 2>/dev/null | awk '{print $2}')"
-  if [[ -n "$CONFIGURED_PWFILE" ]] && ! diff -q extras/mosquitto.conf /etc/mosquitto/conf.d/frugal-iot.conf >/dev/null 2>&1; then
-    problem "extras/mosquitto.conf here differs from /etc/mosquitto/conf.d/frugal-iot.conf - if you followed step 5 with an older installed package, copy it again after upgrading"
+# What this release ships, against what the broker is actually running.
+#
+# This used to be "diff ./extras/mosquitto.conf /etc/mosquitto/conf.d/frugal-iot.conf", which
+# reported a problem on every machine there has ever been, for three separate reasons: ./extras is
+# a fossil frugal-iot-init will not overwrite, so it differs from the release for ever; the
+# installed copy has the dynamic-security plugin's path substituted, so it differs from the shipped
+# one by design; and "diff -q" against a frugal-iot.conf that does not exist fails too, which is
+# the normal state of a broker whose configuration was assembled by hand as several small files in
+# conf.d. A check that is always wrong is one you stop reading, so this asks the question that
+# actually matters instead: is the broker missing a setting this release ships?
+if [[ -f "${PKG_EXTRAS}/mosquitto.conf" ]]; then
+  item "this release's mosquitto.conf names: $(grep -hE '^[[:space:]]*password_file' "${PKG_EXTRAS}/mosquitto.conf" 2>/dev/null | awk '{print $2}')"
+  if (( ${#BROKER_CONF_FILES} )); then
+    # The plugin line is normalised: the installer substitutes a path that varies by distribution.
+    # Comparing settings rather than files means this works whether the configuration arrived as one
+    # frugal-iot.conf or as several hand-written pieces.
+    MISSING_CONF=$(comm -23 \
+      <(sed -E 's#^plugin .*#plugin -#' "${PKG_EXTRAS}/mosquitto.conf" | grep -vE '^[[:space:]]*(#|$)' | sort -u) \
+      <(broker_conf_text | sed -E 's#^plugin .*#plugin -#' | grep -vE '^[[:space:]]*(#|$)' | sort -u) 2>/dev/null)
+    # password_file and the listeners are allowed to be sited differently on an older installation,
+    # so a difference in those is reported without being called a fault.
+    MISSING_REAL=$(print -r -- "$MISSING_CONF" | grep -vE '^[[:space:]]*(password_file|listener|protocol)[[:space:]]' || true)
+    if [[ -n "$MISSING_CONF" ]]; then
+      item "settings this release ships that the broker does not have:"
+      print -r -- "$MISSING_CONF" | while IFS= read -r l; do [[ -n "$l" ]] && item "    $l"; done
+    else
+      item "the broker has every setting this release ships"
+    fi
+    [[ -n "$MISSING_REAL" ]] && problem "The broker is missing $(print -r -- "$MISSING_REAL" | grep -c .) setting(s) this release ships - see \"Mosquitto password file\" above. Re-run install-pi.sh, or add them by hand if this broker's config.d was written by hand."
   fi
 fi
 item "candidate locations, whether or not configured:"
@@ -270,15 +347,10 @@ section "Broker access control"
 # An acl_file confines each account to its own organization's topics; without one, every account
 # that can log in reaches every topic on the broker. Reported here because a broker installed before
 # the ACL shipped will not have one, and nothing else says so.
-BROKER_CONF_FILES=(/etc/mosquitto/mosquitto.conf /etc/mosquitto/conf.d/*.conf(N))
 BROKER_SETTING=""
 broker_setting() {  # name -> value from the broker's own configuration, sudo only if needed
-  local v
-  v=$(grep -hE "^[[:space:]]*$1[[:space:]]" "${BROKER_CONF_FILES[@]}" 2>/dev/null | tail -1 | awk '{print $2}') || true
-  if [[ -z "$v" ]] && have sudo; then
-    v=$(sudo -n grep -hE "^[[:space:]]*$1[[:space:]]" "${BROKER_CONF_FILES[@]}" 2>/dev/null | tail -1 | awk '{print $2}') || true
-  fi
-  BROKER_SETTING="$v"
+  # broker_conf_text carries the sudo fallback this used to spell out for itself
+  BROKER_SETTING=$(broker_conf_text | grep -hE "^[[:space:]]*$1[[:space:]]" | tail -1 | awk '{print $2}') || true
 }
 broker_setting acl_file; ACLFILE="$BROKER_SETTING"
 if [[ -z "$ACLFILE" ]]; then
@@ -317,11 +389,45 @@ else
     fi
   fi
   # Can the server actually drive it? That needs the admin credential frugal-iot-init recorded.
-  DYNUSER=$(sed -n 's/^dynsec_admin_user:[[:space:]]*//p' config.d/secrets.yaml 2>/dev/null | tr -d '"'"'"'' | head -1)
-  DYNPW=$(sed -n 's/^dynsec_admin_password:[[:space:]]*//p' config.d/secrets.yaml 2>/dev/null | tr -d '"'"'"'' | head -1)
+  #
+  # Read with a sudo fallback, and report the file's ownership, because the two ways this goes wrong
+  # look identical from here: the setting really is absent, or the file belongs to somebody else and
+  # an ordinary reader gets silence. The second is the one that matters, since the SERVER is an
+  # ordinary reader too - it runs as the account in the unit file, and a secrets.yaml it cannot read
+  # leaves it with no credential to hand a browser, whatever is written inside.
+  item "  secrets file: config.d/secrets.yaml $(fileinfo config.d/secrets.yaml)"
+  SERVICE_USER=$(sed -n 's/^User=//p' /etc/systemd/system/frugaliot.service 2>/dev/null | head -1)
+  [[ -z "$SERVICE_USER" ]] && SERVICE_USER=$(sudo -n sed -n 's/^User=//p' /etc/systemd/system/frugaliot.service 2>/dev/null | head -1)
+  secret_value() {   # name -> its value, with sudo if reading it plainly gives nothing
+    local v
+    v=$(sed -n "s/^$1:[[:space:]]*//p" config.d/secrets.yaml 2>/dev/null | tr -d "\"'" | head -1)
+    if [[ -z "$v" ]] && have sudo; then
+      v=$(sudo -n sed -n "s/^$1:[[:space:]]*//p" config.d/secrets.yaml 2>/dev/null | tr -d "\"'" | head -1)
+    fi
+    print -r -- "$v"
+  }
+  if [[ -f config.d/secrets.yaml && -n "$SERVICE_USER" ]] && have stat; then
+    SECRETS_OWNER=$(stat -c '%U' config.d/secrets.yaml 2>/dev/null || true)
+    if [[ -n "$SECRETS_OWNER" && "$SECRETS_OWNER" != "$SERVICE_USER" && "$SECRETS_OWNER" != root ]]; then
+      problem "config.d/secrets.yaml belongs to ${SECRETS_OWNER} but the server runs as ${SERVICE_USER}, so the server cannot read it. Fix with: sudo chown ${SERVICE_USER} config.d/secrets.yaml"
+    elif [[ "$SECRETS_OWNER" == root && "$SERVICE_USER" != root ]]; then
+      problem "config.d/secrets.yaml belongs to root but the server runs as ${SERVICE_USER}, so the server reads nothing from it - every login is told live data is unavailable. Fix with: sudo chown ${SERVICE_USER} config.d/secrets.yaml && sudo systemctl restart frugaliot"
+    fi
+  fi
+  # Without this a login gets no broker credential at all, whatever else is right - which shows up
+  # in the dashboard as "No broker credential for this login".
+  USERSECRET=$(secret_value user_secret)
+  if [[ -z "$USERSECRET" ]]; then
+    item "  user_secret: MISSING - no login can be given a broker credential"
+    problem "config.d/secrets.yaml has no user_secret, so every login is told \"No broker credential for this login\". The server writes one at startup; if it cannot, it says so in its log."
+  else
+    item "  user_secret: present (each login's broker password is derived from it)"
+  fi
+  DYNUSER=$(secret_value dynsec_admin_user)
+  DYNPW=$(secret_value dynsec_admin_password)
   if [[ -z "$DYNUSER" || -z "$DYNPW" ]]; then
-    item "  admin credential: not in config.d/secrets.yaml - the server cannot create accounts"
-    problem "dynamic security is enabled but config.d/secrets.yaml has no dynsec_admin_user/password"
+    item "  admin credential: not readable in config.d/secrets.yaml - the server cannot create accounts"
+    problem "dynamic security is enabled but config.d/secrets.yaml has no readable dynsec_admin_user/password, so no login gets a broker credential"
   elif have mosquitto_ctrl; then
     if DYNOUT=$(mosquitto_ctrl -h localhost -u "$DYNUSER" -P "$DYNPW" dynsec listClients 2>&1 | grep -viE 'without encryption|visible on the network'); then
       item "  admin credential works ($(print -r -- $DYNOUT | tr '\n' ' ' | cut -c1-60))"
@@ -515,8 +621,11 @@ if [[ -f config.d/server.yaml ]]; then
   MORGANSET=$(sed -n 's/^morgan:[[:space:]]*//p' config.d/server.yaml 2>/dev/null | head -1)
   item "server morgan:       ${MORGANSET:-not set, so on - a line logged per HTTP request}"
 fi
-if [[ -f /etc/mosquitto/conf.d/frugal-iot.conf ]]; then
-  if grep -qE '^[[:space:]]*connection_messages[[:space:]]+false' /etc/mosquitto/conf.d/frugal-iot.conf 2>/dev/null; then
+# Across every file the broker reads, not just frugal-iot.conf: a configuration assembled by hand
+# puts this wherever it likes, and looking in one file meant a broker that logs every connect and
+# disconnect was never reported as doing so.
+if (( ${#BROKER_CONF_FILES} )); then
+  if broker_conf_text | grep -qhE '^[[:space:]]*connection_messages[[:space:]]+false'; then
     item "mosquitto connections: not logged"
   else
     item "mosquitto connections: logged - every connect and disconnect is a write"
@@ -539,10 +648,23 @@ if [[ -d data ]]; then
   [[ -n "$OLDEST" ]] && item "  oldest day: ${OLDEST}"
 fi
 
-section "Summary"
+# The summary prints to fd 3, which is the real stdout whether or not the rest was silenced. A
+# quiet run with nothing to report prints nothing at all, which is the point of it.
 if (( ${#PROBLEMS} == 0 )); then
-  item "No problems detected by these checks."
-else
-  for p in $PROBLEMS; do item "PROBLEM: $p"; done
+  if (( ! QUIET )); then
+    print -r -- "" >&3
+    print -r -- "===== Summary =====" >&3
+    print -r -- "  No problems detected by these checks." >&3
+    print -r -- "" >&3
+  fi
+  exit 0
+fi
+print -r -- "" >&3
+print -r -- "===== Summary: ${#PROBLEMS} problem(s) =====" >&3
+for p in $PROBLEMS; do print -r -- "  PROBLEM: $p" >&3; done
+if (( QUIET )); then
+  print -r -- "" >&3
+  print -r -- "  Run it again without \"quiet\" for the whole report." >&3
 fi
-print -r -- ""
+print -r -- "" >&3
+exit 1

package/scripts/init.zsh

@@ -63,10 +63,16 @@ done
 for f in "${PKG}"/config.d/schema/*.yaml(N); do
   copy_if_missing "$f" "./config.d/schema/${f:t}" compare
 done
-# Copied so that mosquitto.conf and frugaliot.service can be edited and installed from here. Worth
-# comparing: these get copied on somewhere else (/etc/...), where an old version lingers unnoticed.
+# Copied so that mosquitto.conf and frugaliot.service can be there to read and install from.
+#
+# NOT compared against the packaged copy. This copy is never overwritten - that is the whole point
+# of copy_if_missing - so once a release changes one of these it differs for ever, and says so at
+# every upgrade whether or not anything is actually out of date. A warning that is always wrong is
+# one you learn to skip past, and then miss on the day it is right. What matters is not whether
+# this copy is current but whether the file actually in use is, so that is what is checked, below,
+# where it really lives.
 for f in "${PKG}"/extras/*(N); do
-  copy_if_missing "$f" "./extras/${f:t}" compare
+  copy_if_missing "$f" "./extras/${f:t}"
 done
 
 # ---- 1b. Secrets this server generates for itself ----
@@ -182,6 +188,74 @@ else
   echo "  kept    ${DB} permissions.project (already present)"
 fi
 
+# ---- 4. Is what was installed FROM extras/ still current? ----
+# The files in extras/ are templates for files that live somewhere else - the broker's
+# configuration, its ACL, the systemd unit. Whether this directory's copy matches the release says
+# nothing about any of them, so the question is asked where the file actually is.
+#
+# Each of these is checked in the way that file is allowed to differ, because two of them are
+# *supposed* to differ and a plain cmp would cry wolf at every upgrade:
+#  - mosquitto.conf carries a placeholder for the dynamic security plugin, which the installer
+#    replaces with a path that varies by distribution.
+#  - aclfile gets a stanza appended per bridge by addbridge-prod, so the installed file is
+#    expected to be a superset - only a line this release ships and it lacks is a finding.
+STALE=()        # installed, and older than this release
+UNINSTALLED=()  # nothing at the usual place, so there was nothing to compare
+UNREADABLE=()   # there, but not readable without sudo
+
+# check_installed <packaged file> <installed path> <exact|plugin|subset>
+check_installed() {
+  local from=$1 to=$2 how=$3 a b missing
+  [[ -e "$from" ]] || return 0
+  if [[ ! -e "$to" ]]; then
+    UNINSTALLED+=("${to}|${from}")
+    return 0
+  fi
+  if [[ ! -r "$to" ]]; then
+    UNREADABLE+=("$to")
+    return 0
+  fi
+  # Both sides through the same filter, so only a real difference survives. Note the "#"
+  # delimiter: an alternation inside an "s|...|" would end the expression at its first branch, and
+  # the "|| true" below would then hide the error and report the two files as identical.
+  local norm
+  case "$how" in
+    plugin)  norm='s#^plugin .*#plugin -#' ;;
+    # The installer rewrites User, WorkingDirectory and ExecStart for whoever runs it and wherever
+    # the server went, so those three are normalised - comparing them exactly would report every
+    # installation that is not user "pi" in /home/pi/frugal-iot as out of date at every upgrade.
+    service) norm='s#^(User|WorkingDirectory|ExecStart)=.*#\1=-#' ;;
+    *)       norm='' ;;
+  esac
+
+  case "$how" in
+    plugin|service)
+      a=$(sed -E "$norm" "$from" | grep -vE '^[[:space:]]*(#|$)' | sort || true)
+      b=$(sed -E "$norm" "$to"   | grep -vE '^[[:space:]]*(#|$)' | sort || true)
+      # Neither of these files is ever empty once the comments are gone. Both blank means the
+      # filter failed rather than that they agree, and reporting "current" on that would be the
+      # worst answer of the three.
+      if [[ -z "$a" || -z "$b" ]]; then
+        UNREADABLE+=("$to")
+      elif [[ "$a" != "$b" ]]; then
+        STALE+=("${to}|${from}")
+      fi
+      ;;
+    subset)
+      missing=$(comm -23 <(grep -vE '^[[:space:]]*(#|$)' "$from" | sort -u || true) \
+                         <(grep -vE '^[[:space:]]*(#|$)' "$to"   | sort -u || true) || true)
+      [[ -z "$missing" ]] || STALE+=("${to}|${from}")
+      ;;
+    *)
+      cmp -s "$from" "$to" || STALE+=("${to}|${from}")
+      ;;
+  esac
+}
+
+check_installed "${PKG}/extras/mosquitto.conf"    /etc/mosquitto/conf.d/frugal-iot.conf  plugin
+check_installed "${PKG}/extras/aclfile"           /etc/mosquitto/aclfile                 subset
+check_installed "${PKG}/extras/frugaliot.service" /etc/systemd/system/frugaliot.service  service
+
 if (( ${#DIFFER_TO} )); then
   echo
   echo "These files were left as you have them, but this release ships a different version:"
@@ -189,9 +263,35 @@ if (( ${#DIFFER_TO} )); then
     echo "  ${DIFFER_TO[$i]}"
     echo "    compare with:  diff ${DIFFER_TO[$i]} ${DIFFER_FROM[$i]}"
   done
-  echo "Usually that just means you edited it, and there is nothing to do. But a release can also"
-  echo "change one of these files - and anything installed elsewhere from it, such as"
-  echo "/etc/mosquitto/conf.d/frugal-iot.conf, keeps the old content until you copy it again."
+  echo "The schema describes the sensor types the software understands. If you added one of your"
+  echo "own that is exactly as it should be; if you did not, this release changed it and the"
+  echo "difference is worth reading."
+fi
+
+if (( ${#STALE} )); then
+  echo
+  echo "Installed elsewhere from extras/, and older than this release:"
+  for e in $STALE; do
+    echo "  ${e%%|*}"
+    echo "    compare with:  sudo diff ${e%%|*} ${e#*|}"
+  done
+  echo "Re-running the installer brings these up to date and is the safe way to do it - it"
+  echo "substitutes the plugin path and will not discard ACL rules added per bridge:"
+  echo "  bash node_modules/frugal-iot-server/scripts/install-pi.sh"
+fi
+
+if (( ${#UNINSTALLED} )); then
+  echo
+  echo "Nothing at the usual place, so these were not compared:"
+  for e in $UNINSTALLED; do echo "  ${e%%|*}  (would come from extras/${${e#*|}:t})"; done
+  echo "Either install-pi.sh has not been run here, or this machine's broker configuration was put"
+  echo "together by hand under another name - several small files in /etc/mosquitto/conf.d is a"
+  echo "normal way to have done it. Nothing is out of date; there is simply nothing to compare."
+fi
+
+if (( ${#UNREADABLE} )); then
+  echo
+  echo "There, but not readable as $(id -un), so not compared: ${(j:, :)UNREADABLE}"
 fi
 
 echo