frugal-iot-server 0.3.4 → 0.3.6

package/scripts/check-schema.js

@@ -0,0 +1,127 @@
+#!/usr/bin/env node
+/*
+ * Warn about things in a sensor schema that are easy to get wrong and expensive to get wrong.
+ *
+ * It only warns - it never changes anything. Each of these needs somebody to decide what the right
+ * value is for that particular sensor, and a script guessing would be worse than a script asking.
+ *
+ * What it looks for:
+ *
+ *  - A topic that does not say whether it is logged. With no "log" setting, any float, int or bool
+ *    with "rw: r" is recorded and anything else is not, which is easy to be surprised by in either
+ *    direction - a sensor silently not recorded, or one recorded that nobody wanted.
+ *
+ *  - A logged topic with no "duplicates" rule. Without one, every reading that arrives is written
+ *    to disk as it arrives. On a server running from an SD card that wears the card out, and it
+ *    fills the disk with readings that mostly repeat the one before.
+ *
+ *  - A "duplicates" rule that cannot do anything, because it sets neither significantvalue nor
+ *    significantdate.
+ *
+ *  - A module topic overriding a field that its topic does not define, which is usually a
+ *    misspelling.
+ *
+ * Usage:
+ *   node scripts/check-schema.js                     # checks ./config.d/schema
+ *   node scripts/check-schema.js <dir> [<dir> ...]   # each dir holding topics.yaml and modules.yaml
+ *
+ * Exits 0 whatever it finds, so it can run from a prerelease script without blocking a release for
+ * something that may well be deliberate. Exits 1 only if it could not read a schema at all.
+ */
+
+import yaml from 'js-yaml';
+import { readFileSync, existsSync } from 'fs';
+import path from 'path';
+
+// The types the logger records when a topic does not say - keep in step with shouldLog() in
+// frugal-iot-logger/index.js
+const TYPES_LOGGED_BY_DEFAULT = ['float', 'int', 'bool'];
+
+function loadYaml(file) {
+  if (!existsSync(file)) return null;
+  return yaml.load(readFileSync(file, 'utf8')) || {};
+}
+
+/*
+ * Check one directory holding topics.yaml and (optionally) modules.yaml.
+ * Returns an array of warning strings.
+ */
+function checkSchemaDir(dir) {
+  const topics = loadYaml(path.join(dir, 'topics.yaml'));
+  if (topics === null) return [`${dir}/topics.yaml does not exist`];
+  const modules = loadYaml(path.join(dir, 'modules.yaml')) || {};
+  const warnings = [];
+
+  for (const [name, topic] of Object.entries(topics)) {
+    if (!topic || (typeof topic !== 'object')) {
+      warnings.push(`topic "${name}" is not a set of settings`);
+      continue;
+    }
+    // Whether it is logged, and whether that was actually stated
+    const wouldBeLogged = TYPES_LOGGED_BY_DEFAULT.includes(topic.type) && (topic.rw === 'r');
+    if (topic.log === undefined) {
+      warnings.push(`topic "${name}" does not say whether it is logged`
+        + ` - with type "${topic.type}" and rw "${topic.rw}" it ${wouldBeLogged ? 'IS' : 'is not'} recorded.`
+        + ` Add "log: ${wouldBeLogged}" if that is what you want.`);
+    }
+    const logged = (topic.log === undefined) ? wouldBeLogged : topic.log;
+    if (logged && !topic.duplicates) {
+      warnings.push(`topic "${name}" is logged but has no "duplicates" rule, so every reading is`
+        + ` written to disk as it arrives. Add significantvalue (how far it has to move) and`
+        + ` significantdate (how long since the last one, in milliseconds).`);
+    }
+    if (topic.duplicates
+        && (topic.duplicates.significantvalue === undefined)
+        && (topic.duplicates.significantdate === undefined)) {
+      warnings.push(`topic "${name}" has a "duplicates" rule setting neither significantvalue nor`
+        + ` significantdate, so it does nothing`);
+    }
+  }
+
+  // A module topic can override any field of the topic it is built from. A field the topic does not
+  // have is usually a spelling mistake, and one that quietly does nothing.
+  for (const [moduleName, module] of Object.entries(modules)) {
+    for (const topic of ((module && module.topics) || [])) {
+      const from = topic.leaf_from || topic.leaf;
+      const base = topics[from];
+      if (!base) {
+        warnings.push(`module "${moduleName}" has topic "${topic.leaf}" built from "${from}",`
+          + ` which is not in topics.yaml`);
+        continue;
+      }
+      for (const field of Object.keys(topic)) {
+        if (['leaf', 'leaf_from', 'name'].includes(field)) continue; // Always local to the module
+        if (base[field] === undefined) {
+          warnings.push(`module "${moduleName}" topic "${topic.leaf}" sets "${field}", which`
+            + ` "${from}" in topics.yaml does not have - a misspelling?`);
+        }
+      }
+    }
+  }
+  return warnings;
+}
+
+const dirs = (process.argv.length > 2) ? process.argv.slice(2) : ['config.d/schema'];
+let unreadable = 0;
+let total = 0;
+
+for (const dir of dirs) {
+  const warnings = checkSchemaDir(dir);
+  if ((warnings.length === 1) && warnings[0].endsWith('does not exist')) {
+    console.error(`${dir}: ${warnings[0]}`);
+    unreadable++;
+    continue;
+  }
+  if (!warnings.length) {
+    console.log(`${dir}: nothing to report`);
+    continue;
+  }
+  console.log(`${dir}: ${warnings.length} thing(s) worth a look`);
+  for (const w of warnings) console.log(`  ${w}`);
+  total += warnings.length;
+}
+
+if (total) {
+  console.log(`\n${total} warning(s) - these are for you to decide about, nothing has been changed.`);
+}
+process.exit(unreadable ? 1 : 0);

package/scripts/copy-schema-to-examples.zsh

@@ -0,0 +1,55 @@
+#!/usr/bin/env zsh
+#
+# Copy this server's sensor schema into the examples that ship with frugal-iot-logger, so that they
+# do not drift away from the schema actually in use.
+#
+# Called by the prerelease script of both packages - whichever one you are about to publish, the
+# examples end up current. It is here rather than in frugal-iot-logger because this is where the
+# schema is maintained.
+#
+# Usage:
+#   scripts/copy-schema-to-examples.zsh [<frugal-iot-logger directory>]
+#
+# With no argument it looks for frugal-iot-logger beside this checkout, which is how these are
+# developed. Says what it did and changes nothing else; committing is up to you.
+
+set -euo pipefail
+
+HERE="${0:A:h:h}"                     # The frugal-iot-server checkout this script is in
+SCHEMA="${HERE}/config.d/schema"
+LOGGER="${1:-${HERE:h}/frugal-iot-logger}"
+
+if [[ ! -d "$LOGGER" ]]; then
+  echo "  No frugal-iot-logger checkout at ${LOGGER}, so nothing copied."
+  echo "  The examples in that package keep their own copy of the schema and will drift."
+  echo "  Clone it beside this one and run this again if you want them updated."
+  exit 0                              # Not an error - you may simply not have it checked out
+fi
+
+DIRS=("${LOGGER}"/examples/*/config.d/schema(N/))
+if (( ! ${#DIRS} )); then
+  echo "  No examples/*/config.d/schema directories under ${LOGGER}"
+  exit 0
+fi
+
+COPIED=0
+for d in $DIRS; do
+  for f in topics.yaml modules.yaml; do
+    if [[ ! -f "${SCHEMA}/${f}" ]]; then
+      echo "  ${SCHEMA}/${f} does not exist - skipped"
+      continue
+    fi
+    if cmp -s "${SCHEMA}/${f}" "${d}/${f}"; then
+      echo "  same     ${d#${LOGGER}/}/${f}"
+    else
+      cp "${SCHEMA}/${f}" "${d}/${f}"
+      echo "  updated  ${d#${LOGGER}/}/${f}"
+      COPIED=$((COPIED+1))
+    fi
+  done
+done
+
+if (( COPIED )); then
+  echo
+  echo "  ${COPIED} file(s) changed in ${LOGGER} - commit them there before publishing the logger."
+fi

package/scripts/diagnostic.zsh

@@ -0,0 +1,382 @@
+#!/usr/bin/env zsh
+#
+# Report the state of a Frugal IoT installation, for working out why something in INSTALLATION.md
+# did not do what it said it would. It only looks - it changes nothing.
+#
+# Run it from the directory the server is installed in (the one holding frugal-iot.db):
+#   zsh diagnostic.zsh
+# or, once installed from npm:
+#   npx --no frugal-iot-diagnostic
+#
+# Paste the whole output into a bug report. Passwords are deliberately not printed: the password
+# file is reported by account name only, and organization configs by name only.
+
+# No "set -e": a failing check should report and carry on, not stop the script.
+setopt no_unset 2>/dev/null
+
+section() { print -r -- ""; print -r -- "===== $* ====="; }
+item()    { print -r -- "  $*"; }
+have()    { command -v "$1" >/dev/null 2>&1; }
+# Ownership and mode of a file, spelled for either Linux or BSD/macOS stat
+fileinfo() {
+  local f=$1
+  if [[ ! -e "$f" ]]; then print -r -- "missing"; return; fi
+  stat -c '%U:%G %a %s bytes' "$f" 2>/dev/null || stat -f '%Su:%Sg %Lp %z bytes' "$f" 2>/dev/null || print -r -- "(cannot stat)"
+}
+
+PROBLEMS=()
+problem() { PROBLEMS+=("$1"); }
+
+section "When and where"
+item "date:        $(date 2>/dev/null)"
+item "host:        $(hostname 2>/dev/null)"
+item "user:        $(id -un 2>/dev/null) (groups: $(id -Gn 2>/dev/null))"
+item "directory:   $PWD"
+
+section "The machine"
+have uname && item "kernel:      $(uname -srm)"
+[[ -r /etc/os-release ]] && item "os:          $(grep PRETTY_NAME /etc/os-release | cut -d'"' -f2)"
+[[ -r /proc/device-tree/model ]] && item "board:       $(tr -d '\0' < /proc/device-tree/model)"
+have free && item "memory:      $(free -h | awk '/^Mem:/ {print $3 " used of " $2 ", " $7 " available"}')"
+have df && item "disk (/):    $(df -h / | awk 'NR==2 {print $3 " used of " $2 ", " $4 " free"}')"
+if have timedatectl; then
+  item "clock:       $(timedatectl show -p NTPSynchronized --value 2>/dev/null | sed 's/^yes$/synchronized with a time server/; s/^no$/NOT synchronized - timestamps may be wrong/')"
+fi
+
+section "Software versions"
+for c in node npm sqlite3 zsh mosquitto mosquitto_passwd mosquitto_sub; do
+  if have $c; then
+    case $c in
+      node|npm)            item "$c: $($c -v 2>&1 | head -1)" ;;
+      sqlite3)             item "$c: $(sqlite3 --version 2>&1 | awk '{print $1}')" ;;
+      zsh)                 item "$c: $ZSH_VERSION" ;;
+      mosquitto)           item "$c: $(mosquitto -h 2>&1 | head -1)" ;;
+      *)                   item "$c: present" ;;
+    esac
+  else
+    item "$c: NOT INSTALLED"
+    [[ $c == (node|npm|sqlite3) ]] && problem "$c is not installed - see INSTALLATION.md step 3"
+  fi
+done
+
+section "This installation"
+for f in config.yaml config.d frugal-iot.db data ota node_modules; do
+  if [[ -e $f ]]; then item "$f: present"; else item "$f: MISSING"; fi
+done
+[[ ! -e frugal-iot.db ]] && problem "No frugal-iot.db here - either the wrong directory, or 'npx --no frugal-iot-init' has not been run"
+if [[ -d node_modules/frugal-iot-server ]]; then
+  item "frugal-iot-server: $(node -e 'console.log(require("./node_modules/frugal-iot-server/package.json").version)' 2>/dev/null)"
+  item "frugal-iot-client: $(node -e 'console.log(require("./node_modules/frugal-iot-client/package.json").version)' 2>/dev/null)"
+  item "frugal-iot-logger: $(node -e 'console.log(require("./node_modules/frugal-iot-logger/package.json").version)' 2>/dev/null)"
+  item "commands:    $(ls node_modules/.bin 2>/dev/null | grep frugal | tr '\n' ' ')"
+else
+  item "frugal-iot-server is not installed in this directory"
+fi
+# sqlite3 is the one dependency with compiled code in it. Where no ready-made binary exists for the
+# platform (32-bit ARM, so any Pi Zero W or Pi 1) npm compiles it during install - and if that fails,
+# npm can still report success, leaving a package directory with nothing usable inside it. Nothing
+# else here looks wrong when that happens, and the server simply refuses to start.
+if [[ -d node_modules/sqlite3 ]]; then
+  if node -e "require('sqlite3')" >/dev/null 2>&1; then
+    item "sqlite3 native module: builds and loads OK"
+  else
+    item "sqlite3 native module: PRESENT BUT NOT BUILT"
+    problem "node_modules/sqlite3 has no compiled binding, so the server cannot start. Build it with:
+      npm rebuild sqlite3 --foreground-scripts
+    (about 40 minutes on a Pi Zero W; needs build-essential python3-dev python3-setuptools)"
+  fi
+fi
+if [[ -f config.d/mqtt.yaml ]]; then item "broker configured as: $(grep -h '^broker:' config.d/mqtt.yaml 2>/dev/null)"; fi
+if [[ -d config.d/organizations ]]; then
+  # Names only - these files contain the broker password.
+  # (N) makes a non-matching glob expand to nothing rather than erroring - and the list has to be
+  # tested before use, because "ls" with no arguments would list the current directory instead.
+  ORGS=(config.d/organizations/*.yaml(N:t:r))
+  if (( ${#ORGS} )); then
+    item "organizations: ${ORGS}"
+  else
+    item "organizations: none defined yet"
+  fi
+fi
+if [[ -f frugal-iot.db ]] && have sqlite3; then
+  item "accounts:    $(sqlite3 frugal-iot.db 'SELECT group_concat(username, ", ") FROM users;' 2>/dev/null)"
+fi
+
+section "Mosquitto configuration"
+if [[ -f /etc/mosquitto/mosquitto.conf ]]; then
+  item "/etc/mosquitto/mosquitto.conf: $(fileinfo /etc/mosquitto/mosquitto.conf)"
+else
+  item "/etc/mosquitto/mosquitto.conf: missing - is mosquitto installed?"
+fi
+item "files in /etc/mosquitto/conf.d:"
+for f in /etc/mosquitto/conf.d/*(N); do
+  item "  ${f}: $(fileinfo $f)"
+  grep -nE '^[[:space:]]*(listener|protocol|password_file|allow_anonymous)' $f 2>/dev/null | sed 's/^/      /'
+done
+[[ -z "$(print -r -- /etc/mosquitto/conf.d/*(N))" ]] && problem "Nothing in /etc/mosquitto/conf.d - the Frugal IoT config was never copied there (step 5)"
+
+# Which password file does the running configuration actually name?
+# mosquitto's include_dir only loads files ending in .conf, so look at exactly those, last one wins
+CONFIGURED_PWFILE=$(grep -hE '^[[:space:]]*password_file[[:space:]]' /etc/mosquitto/mosquitto.conf /etc/mosquitto/conf.d/*.conf(N) 2>/dev/null | tail -1 | awk '{print $2}')
+section "Mosquitto password file"
+if [[ -n "$CONFIGURED_PWFILE" ]]; then
+  item "configuration names: $CONFIGURED_PWFILE"
+  item "  that file:         $(fileinfo $CONFIGURED_PWFILE)"
+  if [[ ! -e "$CONFIGURED_PWFILE" ]]; then
+    problem "Mosquitto is configured to use $CONFIGURED_PWFILE but that file does not exist - it will refuse to start"
+  else
+    # Account names only, never the hashes
+    item "  accounts in it:    $(cut -d: -f1 "$CONFIGURED_PWFILE" 2>/dev/null | tr '\n' ' ')"
+    [[ ! -s "$CONFIGURED_PWFILE" ]] && item "  (the file is empty - no accounts yet)"
+  fi
+else
+  item "no password_file line found in any mosquitto configuration"
+  problem "No password_file configured - the broker would allow anonymous access or reject everything"
+fi
+# The copy in this installation is what step 5 tells you to put into /etc - if the two disagree,
+# either the copy never happened or the installed package is older than the instructions.
+if [[ -f extras/mosquitto.conf ]]; then
+  item "this installation's extras/mosquitto.conf names: $(grep -hE '^[[:space:]]*password_file' extras/mosquitto.conf 2>/dev/null | awk '{print $2}')"
+  if [[ -n "$CONFIGURED_PWFILE" ]] && ! diff -q extras/mosquitto.conf /etc/mosquitto/conf.d/frugal-iot.conf >/dev/null 2>&1; then
+    problem "extras/mosquitto.conf here differs from /etc/mosquitto/conf.d/frugal-iot.conf - if you followed step 5 with an older installed package, copy it again after upgrading"
+  fi
+fi
+item "candidate locations, whether or not configured:"
+for f in /var/lib/mosquitto/passwords /etc/mosquitto/mosquitto_passwords; do
+  item "  ${f}: $(fileinfo $f)"
+done
+item "directories they live in:"
+for d in /var/lib/mosquitto /etc/mosquitto; do
+  item "  ${d}: $(fileinfo $d)"
+done
+
+section "Mosquitto service"
+if have systemctl; then
+  item "enabled:     $(systemctl is-enabled mosquitto 2>&1)"
+  item "active:      $(systemctl is-active mosquitto 2>&1)"
+  [[ "$(systemctl is-active mosquitto 2>/dev/null)" != "active" ]] && problem "Mosquitto is not running - see its own log, quoted above under 'Mosquitto service'"
+  print -r -- "  --- last 12 journal lines (systemd's view: usually only an exit code) ---"
+  journalctl -u mosquitto -n 12 --no-pager 2>&1 | sed 's/^/      /'
+else
+  item "systemctl not available on this machine"
+fi
+# Debian's mosquitto logs to a file rather than the journal, so this is where the real reason is.
+MOSQUITTO_LOG=$(grep -hE '^[[:space:]]*log_dest[[:space:]]+file' /etc/mosquitto/mosquitto.conf /etc/mosquitto/conf.d/*(N) 2>/dev/null | tail -1 | awk '{print $3}')
+[[ -z "$MOSQUITTO_LOG" ]] && MOSQUITTO_LOG=/var/log/mosquitto/mosquitto.log
+item "its own log file: ${MOSQUITTO_LOG} ($(fileinfo $MOSQUITTO_LOG))"
+if [[ -r "$MOSQUITTO_LOG" ]]; then
+  print -r -- "  --- last 12 lines of that log (this is where startup errors appear) ---"
+  tail -12 "$MOSQUITTO_LOG" 2>&1 | sed 's/^/      /'
+elif [[ -e "$MOSQUITTO_LOG" ]]; then
+  # The log belongs to the mosquitto user, so reading it needs privilege. Try without prompting -
+  # on Raspberry Pi OS the first user has passwordless sudo, so this usually just works.
+  if have sudo && MOSQUITTO_LOG_TAIL=$(sudo -n tail -12 "$MOSQUITTO_LOG" 2>/dev/null); then
+    print -r -- "  --- last 12 lines of that log, read with sudo (startup errors appear here) ---"
+    print -r -- "$MOSQUITTO_LOG_TAIL" | sed 's/^/      /'
+  else
+    item "  it belongs to $(fileinfo $MOSQUITTO_LOG | awk '{print $1}') so it cannot be read as $(id -un)."
+    item "  Re-run this whole script with sudo to include it:  sudo zsh $0"
+    problem "Could not read ${MOSQUITTO_LOG}, which is where Mosquitto explains itself - re-run with sudo"
+  fi
+fi
+
+section "Listening ports"
+PORTS=""
+if have ss; then
+  PORTS=$(ss -tln 2>/dev/null | grep -E ':(1883|9012|8080)\b')
+elif have netstat; then
+  PORTS=$(netstat -an 2>/dev/null | grep -E '[.:](1883|9012|8080) ')
+else
+  item "neither ss nor netstat available"
+fi
+if [[ -n "$PORTS" ]]; then
+  print -r -- "$PORTS" | sed 's/^/  /'
+else
+  item "none of 1883 (mqtt), 9012 (websockets), 8080 (web) are listening"
+fi
+
+section "Frugal IoT server service"
+if have systemctl; then
+  if systemctl list-unit-files 2>/dev/null | grep -q '^frugaliot.service'; then
+    item "enabled:     $(systemctl is-enabled frugaliot 2>&1)"
+    item "active:      $(systemctl is-active frugaliot 2>&1)"
+    print -r -- "  --- last 20 journal lines ---"
+    journalctl -u frugaliot -n 20 --no-pager 2>&1 | sed 's/^/      /'
+  else
+    item "no frugaliot service installed yet (step 8 not reached)"
+  fi
+else
+  item "systemctl not available on this machine"
+fi
+
+section "Name resolution (steps 3, 4)"
+# The broker URL in config.d/mqtt.yaml has to resolve from every machine that uses it - this one,
+# and whatever browser or node talks to it.
+BROKER_URL=$(grep -h '^broker:' config.d/mqtt.yaml 2>/dev/null | awk '{print $2}')
+# Strip scheme, then any path (wss://host/wss), then any port - leaving just the hostname
+BROKER_HOST=${${${BROKER_URL#*://}%%/*}%%:*}
+if [[ -n "$BROKER_HOST" ]]; then
+  item "broker host: $BROKER_HOST"
+  if have getent && BROKER_IP=$(getent hosts "$BROKER_HOST" 2>/dev/null | head -1 | awk '{print $1}') && [[ -n "$BROKER_IP" ]]; then
+    item "  resolves to $BROKER_IP from this machine"
+  elif have ping && ping -c1 -W2 "$BROKER_HOST" >/dev/null 2>&1; then
+    item "  answers ping from this machine"
+  else
+    item "  DOES NOT RESOLVE from this machine"
+    problem "The broker host '$BROKER_HOST' does not resolve here - the server's own logger cannot connect either"
+  fi
+  [[ "$BROKER_HOST" == *.local ]] && item "  note: .local names do not resolve on most Android phones - use an IP address there"
+fi
+MYADDRS=$( (have ip && ip -4 -o addr show scope global | awk '{print $2"="$4}') 2>/dev/null | tr '\n' ' ')
+item "this machine's addresses: ${MYADDRS:-(could not determine)}"
+
+section "Broker authentication (steps 5, 6)"
+if have mosquitto_sub; then
+  # A wrong password must be refused - that is the check in step 5
+  WRONGOUT=$(mosquitto_sub -h localhost -u nobody -P wrong -t '#' -W 2 2>&1)
+  if [[ "$WRONGOUT" == *"not authorised"* ]]; then
+    item "wrong password: correctly refused"
+  elif [[ "$WRONGOUT" == *"Connection refused"* || "$WRONGOUT" == *"Error"* ]]; then
+    item "wrong password: broker did not answer - $WRONGOUT"
+    problem "The broker is not answering on localhost:1883"
+  else
+    item "wrong password: ACCEPTED - the broker is not requiring credentials"
+    problem "A wrong password was not refused - check password_file is set in the mosquitto config"
+  fi
+  # Then each configured organization should be able to connect with its own credentials.
+  # The password is read from the config and used, never printed.
+  for f in config.d/organizations/*.yaml(N); do
+    ORG=${f:t:r}
+    ORG_PW=$(sed -n 's/^mqtt_password:[[:space:]]*//p' "$f" 2>/dev/null | head -1 | tr -d '"'"'"'')
+    if [[ -z "$ORG_PW" ]]; then
+      item "organization $ORG: no mqtt_password in $f"
+      continue
+    fi
+    ORGOUT=$(mosquitto_sub -h localhost -u "$ORG" -P "$ORG_PW" -t '#' -W 2 2>&1)
+    if [[ "$ORGOUT" == *"not authorised"* ]]; then
+      item "organization $ORG: REFUSED by the broker"
+      problem "Organization '$ORG' cannot log in to the broker - its password in $f does not match the broker's password file (step 6)"
+    elif [[ "$ORGOUT" == *"Connection refused"* ]]; then
+      item "organization $ORG: broker not answering"
+    else
+      item "organization $ORG: authenticates, and saw $(print -r -- "$ORGOUT" | grep -c . ) message(s) in a 2 second sample"
+    fi
+  done
+else
+  item "mosquitto_sub not installed - cannot test broker logins (sudo apt install mosquitto-clients)"
+fi
+
+section "Web server (step 7)"
+WEBPORT=$(grep -h '^port:' config.d/server.yaml 2>/dev/null | awk '{print $2}')
+[[ -z "$WEBPORT" ]] && WEBPORT=8080
+if have curl; then
+  HOMECODE=$(curl -s -o /dev/null -w '%{http_code}' -m 5 http://localhost:${WEBPORT}/ 2>/dev/null)
+  if [[ "$HOMECODE" == "000" ]]; then
+    item "nothing is answering on port ${WEBPORT} - the server is not running"
+    item "  (expected if you have not reached step 7 yet; otherwise start it, or see the frugaliot service below)"
+  else
+    item "GET /            -> ${HOMECODE} (expect 200)"
+    item "GET /config.json -> $(curl -s -o /dev/null -w '%{http_code}' -m 5 http://localhost:${WEBPORT}/config.json 2>/dev/null) (expect 401 when not logged in)"
+    [[ "$HOMECODE" != "200" ]] && problem "The web server answered ${HOMECODE} rather than 200 on port ${WEBPORT}"
+  fi
+else
+  item "curl not installed - cannot test the web server"
+fi
+
+section "Logged data (step 9)"
+if [[ -d data ]]; then
+  DATADIRS=(data/*(N/))
+  if (( ${#DATADIRS} )); then
+    for d in $DATADIRS; do
+      item "${d}: $(find $d -type f 2>/dev/null | wc -l | tr -d ' ') files, newest $(ls -t $d/**/*(N.om[1]) 2>/dev/null | head -1)"
+    done
+  else
+    item "data/ is empty - no readings logged yet, which is expected until a node reports (step 9)"
+  fi
+fi
+
+section "Wear on the SD card"
+# An SD card wears out from being written to, and a server logging sensor readings writes constantly
+# unless told not to. This reports how much is actually being written and whether the settings that
+# reduce it are in force. There is no threshold to compare against - run it twice a few days apart
+# and look at the rate.
+if [[ -r /proc/diskstats && -r /proc/uptime ]]; then
+  # Field 10 of each line is sectors written, and a sector is 512 bytes. Take the whole card rather
+  # than a partition (mmcblk0, not mmcblk0p2) so the boot partition and swap are included too.
+  ROOTDEV=$( (have findmnt && findmnt -no SOURCE / 2>/dev/null) || print -r -- "" )
+  ROOTDISK=${${ROOTDEV:t}%p[0-9]*}      # /dev/mmcblk0p2 -> mmcblk0
+  [[ -z "$ROOTDISK" ]] && ROOTDISK=mmcblk0
+  WRITTEN=$(awk -v d="$ROOTDISK" '$3 == d {print $10}' /proc/diskstats 2>/dev/null)
+  UPSECS=$(awk '{print int($1)}' /proc/uptime 2>/dev/null)
+  if [[ -n "$WRITTEN" && -n "$UPSECS" && "$UPSECS" -gt 0 ]]; then
+    item "device:      /dev/${ROOTDISK}"
+    item "written:     $(awk -v s="$WRITTEN" 'BEGIN {printf "%.1f MB", s*512/1048576}') since boot, over $(awk -v u="$UPSECS" 'BEGIN {printf "%.1f", u/3600}') hours"
+    item "  that is:   $(awk -v s="$WRITTEN" -v u="$UPSECS" 'BEGIN {printf "%.1f MB/hour", s*512/1048576/(u/3600)}')"
+  else
+    item "could not read write counters for /dev/${ROOTDISK} from /proc/diskstats"
+  fi
+else
+  item "no /proc/diskstats on this machine - write counters are Linux only"
+fi
+
+# Swapping to an SD card wears it fast. On a Pi 4 there should be little or none; a Pi Zero W may
+# need some. See INSTALLATION.md, "Wear and tear on the SD card".
+if have free; then
+  item "swap:        $(free -h | awk '/^Swap:/ {print $3 " used of " $2}')"
+fi
+if [[ -r /proc/sys/vm/swappiness ]]; then
+  SWAPPINESS=$(< /proc/sys/vm/swappiness)
+  item "swappiness:  ${SWAPPINESS} (1 means swap only when there is no alternative)"
+fi
+have swapon && item "swap areas:  $(swapon --show=NAME,TYPE,SIZE,USED --noheadings 2>/dev/null | tr '\n' ';' || print -r -- '(none)')"
+
+# The journal is the other thing that writes on every event, if it is kept on disk at all
+if [[ -d /var/log/journal ]]; then
+  item "journal:     stored on disk in /var/log/journal - every logged line is a write"
+  have journalctl && item "  using:     $(journalctl --disk-usage 2>/dev/null | sed 's/^Archived and active journals take up //')"
+else
+  item "journal:     kept in RAM only (no /var/log/journal), so it costs no writes"
+fi
+
+# The settings that decide how much gets logged in the first place
+if [[ -f config.d/logger.yaml ]]; then
+  VERBOSE=$(sed -n 's/^verbose:[[:space:]]*//p' config.d/logger.yaml 2>/dev/null | head -1)
+  FLUSHSECS=$(sed -n 's/^flushseconds:[[:space:]]*//p' config.d/logger.yaml 2>/dev/null | head -1)
+  item "logger verbose:      ${VERBOSE:-not set, so on - a line logged per message received}"
+  item "logger flushseconds: ${FLUSHSECS:-not set, so readings are written as they arrive}"
+fi
+if [[ -f config.d/server.yaml ]]; then
+  MORGANSET=$(sed -n 's/^morgan:[[:space:]]*//p' config.d/server.yaml 2>/dev/null | head -1)
+  item "server morgan:       ${MORGANSET:-not set, so on - a line logged per HTTP request}"
+fi
+if [[ -f /etc/mosquitto/conf.d/frugal-iot.conf ]]; then
+  if grep -qE '^[[:space:]]*connection_messages[[:space:]]+false' /etc/mosquitto/conf.d/frugal-iot.conf 2>/dev/null; then
+    item "mosquitto connections: not logged"
+  else
+    item "mosquitto connections: logged - every connect and disconnect is a write"
+  fi
+fi
+if [[ -e "$MOSQUITTO_LOG" ]]; then
+  item "mosquitto log size:  $(fileinfo $MOSQUITTO_LOG | awk '{print $3, $4}')"
+else
+  item "mosquitto log size:  no log file at ${MOSQUITTO_LOG}"
+fi
+
+# How much the readings themselves are taking up, and whether the old ones have been compressed
+if [[ -d data ]]; then
+  CSVCOUNT=$(find data -name '*.csv' 2>/dev/null | wc -l | tr -d ' ')
+  GZCOUNT=$(find data -name '*.csv.gz' 2>/dev/null | wc -l | tr -d ' ')
+  item "readings:    ${CSVCOUNT} csv files and ${GZCOUNT} compressed, $(du -sh data 2>/dev/null | awk '{print $1}') in total"
+  # Filenames are the date, so the earliest name is the oldest day held. Both suffixes stripped
+  # separately, because BSD sed (macOS) has no "\?" for an optional group.
+  OLDEST=$(find data \( -name '*.csv' -o -name '*.csv.gz' \) 2>/dev/null | sed 's|.*/||; s|\.gz$||; s|\.csv$||' | sort | head -1)
+  [[ -n "$OLDEST" ]] && item "  oldest day: ${OLDEST}"
+fi
+
+section "Summary"
+if (( ${#PROBLEMS} == 0 )); then
+  item "No problems detected by these checks."
+else
+  for p in $PROBLEMS; do item "PROBLEM: $p"; done
+fi
+print -r -- ""

package/scripts/init.zsh

@@ -5,7 +5,7 @@
 # in node_modules (or which sits in a git clone).
 #
 # Usage:
-#   npx frugal-iot-init          # after: npm install frugal-iot-server
+#   npx --no frugal-iot-init          # after: npm install frugal-iot-server
 #   scripts/init.zsh             # in a git clone, where the config files are already present
 #
 # Safe to run again: existing files are left exactly as they are, only missing ones are added,
@@ -28,11 +28,24 @@ if [[ ! -f "${PKG}/frugal-iot-createdb.sql" ]]; then
 fi
 
 # ---- 1. Configuration files, copied only if not already here ----
-COPIED=()   # which files this run actually created, so existing ones are never rewritten below
+COPIED=()      # which files this run actually created, so existing ones are never rewritten below
+DIFFER_TO=()   # files kept that no longer match what this release ships, and where to compare them
+DIFFER_FROM=()
+# copy_if_missing <packaged file> <local file> [compare]
+# "compare" asks for a warning when the file exists but differs from the one this release ships.
+# Only pass it for files nobody is expected to edit - config.yaml, mqtt.yaml and server.yaml are
+# this installation's own settings, so of course they differ, and warning about them every time
+# would train you to ignore the warning that matters.
 copy_if_missing() {
-  local from=$1 to=$2
+  local from=$1 to=$2 compare=${3:-}
   if [[ -e "$to" ]]; then
-    echo "  kept    ${to} (already present)"
+    if [[ "$compare" == compare && -e "$from" ]] && ! cmp -s "$from" "$to"; then
+      DIFFER_TO+=("$to")
+      DIFFER_FROM+=("$from")
+      echo "  kept    ${to} (DIFFERS from this release - see below)"
+    else
+      echo "  kept    ${to} (already present)"
+    fi
   elif [[ -e "$from" ]]; then
     mkdir -p "${to:h}"
     cp "$from" "$to"
@@ -46,12 +59,14 @@ copy_if_missing "${PKG}/config.yaml" "./config.yaml"
 for f in logger.yaml mqtt.yaml server.yaml; do
   copy_if_missing "${PKG}/config.d/${f}" "./config.d/${f}"
 done
+# The schema describes the sensor types the software understands, so a release changing it matters
 for f in "${PKG}"/config.d/schema/*.yaml(N); do
-  copy_if_missing "$f" "./config.d/schema/${f:t}"
+  copy_if_missing "$f" "./config.d/schema/${f:t}" compare
 done
-# Copied so that mosquitto.conf and frugaliot.service can be edited and installed from here
+# Copied so that mosquitto.conf and frugaliot.service can be edited and installed from here. Worth
+# comparing: these get copied on somewhere else (/etc/...), where an old version lingers unnoticed.
 for f in "${PKG}"/extras/*(N); do
-  copy_if_missing "$f" "./extras/${f:t}"
+  copy_if_missing "$f" "./extras/${f:t}" compare
 done
 
 # ---- 1a. Point server.yaml at the web client this instance actually has ----
@@ -104,8 +119,20 @@ else
   echo "  created ${DB}"
 fi
 
+if (( ${#DIFFER_TO} )); then
+  echo
+  echo "These files were left as you have them, but this release ships a different version:"
+  for i in {1..${#DIFFER_TO}}; do
+    echo "  ${DIFFER_TO[$i]}"
+    echo "    compare with:  diff ${DIFFER_TO[$i]} ${DIFFER_FROM[$i]}"
+  done
+  echo "Usually that just means you edited it, and there is nothing to do. But a release can also"
+  echo "change one of these files - and anything installed elsewhere from it, such as"
+  echo "/etc/mosquitto/conf.d/frugal-iot.conf, keeps the old content until you copy it again."
+fi
+
 echo
 echo "Ready. Next:"
-echo "  npx frugal-iot-setpassword superuser <password>            # so you can log in as the administrator"
-echo "  npx frugal-iot-addorganization <org-id> <name> <email> <phone> <broker-password>"
-echo "  npx frugal-iot-server                                      # start it"
+echo "  npx --no frugal-iot-setpassword superuser <password>            # so you can log in as the administrator"
+echo "  npx --no frugal-iot-addorganization <org-id> <name> <email> <phone> <broker-password>"
+echo "  npx --no frugal-iot-server                                      # start it"