frugal-iot-server 0.3.7 → 0.3.9

package/scripts/check-schema.js

@@ -21,12 +21,25 @@
  *  - A module topic overriding a field that its topic does not define, which is usually a
  *    misspelling.
  *
+ *  - A float or exponential topic with no "width", or one too small to hold its own min..max. Width
+ *    is what decides how many decimals a reading is shown to; without it the UI guesses.
+ *
+ *  - A numeric topic with no "units". Often right - plenty of readings are dimensionless - but
+ *    worth a look, because the UI has nothing to put after the number.
+ *
+ * And, separately, things that are never deliberate and so count as errors:
+ *
+ *  - A devices.yaml entry naming a module, or a leaf within a module, that does not exist. That
+ *    silently loses a row from a card, with nothing to say why.
+ *
  * Usage:
  *   node scripts/check-schema.js                     # checks ./config.d/schema
  *   node scripts/check-schema.js <dir> [<dir> ...]   # each dir holding topics.yaml and modules.yaml
+ *   node scripts/check-schema.js --resolve <otakey>  # which devices.yaml entry that OTA key uses
  *
- * Exits 0 whatever it finds, so it can run from a prerelease script without blocking a release for
- * something that may well be deliberate. Exits 1 only if it could not read a schema at all.
+ * Exits 0 for warnings, so it can run from a prerelease script without blocking a release for
+ * something that may well be deliberate. Exits 1 for an error, or if it could not read a schema.
+ * Both callers guard it with "|| true", so neither blocks on either.
  */
 
 import yaml from 'js-yaml';
@@ -36,6 +49,8 @@ import path from 'path';
 // The types the logger records when a topic does not say - keep in step with shouldLog() in
 // frugal-iot-logger/index.js
 const TYPES_LOGGED_BY_DEFAULT = ['float', 'int', 'bool'];
+// The types with decimals to decide, and so the ones that need a width
+const NUMERIC_TYPES = ['float', 'exponential'];
 
 function loadYaml(file) {
   if (!existsSync(file)) return null;
@@ -48,9 +63,11 @@ function loadYaml(file) {
  */
 function checkSchemaDir(dir) {
   const topics = loadYaml(path.join(dir, 'topics.yaml'));
-  if (topics === null) return [`${dir}/topics.yaml does not exist`];
+  if (topics === null) return { warnings: [`${dir}/topics.yaml does not exist`], errors: [], missing: true };
   const modules = loadYaml(path.join(dir, 'modules.yaml')) || {};
+  const devices = loadYaml(path.join(dir, 'devices.yaml')) || {};
   const warnings = [];
+  const errors = [];
 
   for (const [name, topic] of Object.entries(topics)) {
     if (!topic || (typeof topic !== 'object')) {
@@ -76,6 +93,28 @@ function checkSchemaDir(dir) {
       warnings.push(`topic "${name}" has a "duplicates" rule setting neither significantvalue nor`
         + ` significantdate, so it does nothing`);
     }
+    // Width decides the decimals a reading is shown to, derived as width - intWidth - 1 where
+    // intWidth comes from min..max. Only a float has decimals to decide.
+    const hasDecimals = NUMERIC_TYPES.includes(topic.type);
+    if (hasDecimals && (topic.width === undefined)) {
+      warnings.push(`topic "${name}" is a ${topic.type} with no "width", so the UI has to guess how`
+        + ` many decimals to show. Width counts every character including the sign and the point.`);
+    }
+    if (!hasDecimals && (topic.width !== undefined)) {
+      warnings.push(`topic "${name}" is a ${topic.type}, which has no decimals, so "width" does nothing`);
+    }
+    if (hasDecimals && (topic.width !== undefined)) {
+      const intWidth = Math.max(String(Math.trunc(topic.min ?? 0)).length,
+                                String(Math.trunc(topic.max ?? 0)).length);
+      if (topic.width < intWidth) {
+        warnings.push(`topic "${name}" has width ${topic.width} but its range needs ${intWidth}`
+          + ` characters before any decimal point, so values will overflow it`);
+      }
+    }
+    if (hasDecimals && (topic.units === undefined)) {
+      warnings.push(`topic "${name}" has no "units" - fine if it is dimensionless, but the UI has`
+        + ` nothing to put after the number`);
+    }
   }
 
   // A module topic can override any field of the topic it is built from. A field the topic does not
@@ -98,30 +137,102 @@ function checkSchemaDir(dir) {
       }
     }
   }
-  return warnings;
+  // A device entry picks the rows on a card, by twig (module/leaf) or by bare control module id.
+  // Naming something that does not exist loses a row silently, so these are errors not warnings.
+  for (const [key, device] of Object.entries(devices)) {
+    for (const listName of ['front', 'summary']) {
+      for (const entry of ((device && device[listName]) || [])) {
+        const [moduleName, leaf] = entry.split('/');
+        const module = modules[moduleName];
+        if (!module) {
+          errors.push(`devices "${key}" ${listName} names module "${moduleName}", which is not in modules.yaml`);
+        } else if (leaf !== undefined) {
+          const leaves = ((module.topics) || []).map((t) => t.leaf);
+          if (!leaves.includes(leaf)) {
+            errors.push(`devices "${key}" ${listName} names "${entry}", but module "${moduleName}"`
+              + ` has ${leaves.length ? leaves.join(', ') : 'no leaves'}`);
+          }
+        } else if (!moduleName.startsWith('control')) {
+          warnings.push(`devices "${key}" ${listName} names module "${moduleName}" with no leaf.`
+            + ` Only a control module renders as a whole - did you mean a "${moduleName}/leaf" twig?`);
+        }
+      }
+    }
+    // Longest match wins, so this is not a fault - but it is worth knowing which entry a key lands on
+    const shadowed = Object.keys(devices).filter((k) => (k !== key) && k.startsWith(key));
+    if (shadowed.length) {
+      warnings.push(`devices "${key}" is also a prefix of ${shadowed.join(', ')}; the longest match`
+        + ` wins, so those take precedence for their own keys`);
+    }
+  }
+
+  // summary is opt-out - a module contributes unless it says otherwise
+  for (const [moduleName, module] of Object.entries(modules)) {
+    if (module && (module.summary === true)) {
+      warnings.push(`module "${moduleName}" sets "summary: true", which is already the default -`
+        + ` only "summary: false" does anything`);
+    }
+  }
+
+  return { warnings, errors };
+}
+
+// Which devices.yaml entry an OTA key uses: exact device id, then exact key, then longest prefix.
+function resolveDeviceEntry(devices, otakey) {
+  if (devices[otakey]) return [otakey, 'an exact match'];
+  const prefixes = Object.keys(devices).filter((k) => otakey.startsWith(k));
+  if (!prefixes.length) return [null, 'no match, so the defaults apply'];
+  const longest = prefixes.sort((a, b) => b.length - a.length)[0];
+  return [longest, prefixes.length > 1 ? `the longest of ${prefixes.length} matching prefixes` : 'a prefix'];
 }
 
-const dirs = (process.argv.length > 2) ? process.argv.slice(2) : ['config.d/schema'];
+const argv = process.argv.slice(2);
+const resolveAt = argv.indexOf('--resolve');
+if (resolveAt !== -1) {
+  const otakey = argv[resolveAt + 1];
+  const dir = argv.find((a, i) => !a.startsWith('--') && (i !== resolveAt + 1)) || 'config.d/schema';
+  if (!otakey) {
+    console.error('--resolve needs an OTA key, e.g. --resolve sht30_c3_pico');
+    process.exit(1);
+  }
+  const devices = loadYaml(path.join(dir, 'devices.yaml')) || {};
+  const [entry, how] = resolveDeviceEntry(devices, otakey);
+  console.log(`${otakey} uses ${entry ? `the "${entry}" entry` : 'no entry'} (${how})`);
+  process.exit(0);
+}
+
+const dirs = argv.length ? argv : ['config.d/schema'];
 let unreadable = 0;
-let total = 0;
+let totalWarnings = 0;
+let totalErrors = 0;
 
 for (const dir of dirs) {
-  const warnings = checkSchemaDir(dir);
-  if ((warnings.length === 1) && warnings[0].endsWith('does not exist')) {
+  const { warnings, errors, missing } = checkSchemaDir(dir);
+  if (missing) {
     console.error(`${dir}: ${warnings[0]}`);
     unreadable++;
     continue;
   }
-  if (!warnings.length) {
+  if (!warnings.length && !errors.length) {
     console.log(`${dir}: nothing to report`);
     continue;
   }
-  console.log(`${dir}: ${warnings.length} thing(s) worth a look`);
-  for (const w of warnings) console.log(`  ${w}`);
-  total += warnings.length;
+  if (warnings.length) {
+    console.log(`${dir}: ${warnings.length} thing(s) worth a look`);
+    for (const w of warnings) console.log(`  ${w}`);
+  }
+  if (errors.length) {
+    console.log(`${dir}: ${errors.length} thing(s) that cannot be right`);
+    for (const e of errors) console.log(`  ERROR ${e}`);
+  }
+  totalWarnings += warnings.length;
+  totalErrors += errors.length;
 }
 
-if (total) {
-  console.log(`\n${total} warning(s) - these are for you to decide about, nothing has been changed.`);
+if (totalWarnings) {
+  console.log(`\n${totalWarnings} warning(s) - these are for you to decide about, nothing has been changed.`);
+}
+if (totalErrors) {
+  console.log(`${totalErrors} error(s) - a card will silently lose a row until these are fixed.`);
 }
-process.exit(unreadable ? 1 : 0);
+process.exit((unreadable || totalErrors) ? 1 : 0);

package/scripts/clearretained.js

@@ -0,0 +1,185 @@
+#!/usr/bin/env node
+/*
+ * List, and optionally delete, retained MQTT messages.
+ *
+ * Nodes publish nearly everything retained, so the broker keeps the last value of every topic and
+ * hands it to each new subscriber - which is how a dashboard can show a reading straight away
+ * instead of waiting for the next one. The cost is that a topic published by mistake outlives the
+ * mistake: rename a module, misspell a field, test a node with the wrong id, and that topic stays
+ * on the broker for ever, appearing on every dashboard. Restarting the broker does not clear it and
+ * neither does fixing the node. The only way to remove one is to publish an empty message to that
+ * exact topic, which is what this does.
+ *
+ * Run from the server's own directory, the one holding frugal-iot.db, so that the organization's
+ * broker password can be read from config.d/organizations/.
+ *
+ * Usage:
+ *   npx --no frugal-iot-clearretained '<topic-pattern>'          # list what is retained
+ *   npx --no frugal-iot-clearretained '<topic-pattern>' delete   # delete it
+ *
+ * Quote the pattern, or the shell expands the wildcards before this ever sees them. In MQTT
+ * patterns "+" matches exactly one level and "#" matches the rest:
+ *   'myfarm/#'                          everything the organization holds - look before deleting
+ *   'myfarm/lotus/esp8266-fb94bb/#'     one node
+ *   'myfarm/lotus/+/sht/temperture/#'   a misspelled field, on every node in the project
+ *
+ * "delete" is a bare word rather than "--delete" because npm parses the whole command line of an
+ * "npx" invocation itself: it treats any --flag it does not recognise as an unknown config setting
+ * and passes only the values on, so "--delete" would silently never reach this script and it would
+ * quietly list instead of deleting. "--delete" is still accepted for running it directly.
+ *
+ * This talks to the broker through the "mqtt" library rather than mosquitto_sub and mosquitto_pub,
+ * so it works anywhere the server itself runs. On the production server those commands cannot be
+ * installed at all: its broker came from the mosquitto PPA built for an older Ubuntu, and the
+ * mosquitto-clients in the distribution demand an exactly-matching libmosquitto1, so apt would have
+ * to downgrade the library out from under the running broker.
+ */
+
+import { readFileSync } from 'node:fs';
+import yaml from 'js-yaml';
+import mqtt from 'mqtt';
+
+const SCRIPT = 'frugal-iot-clearretained';
+
+function usage(message) {
+  if (message) console.error(`Error: ${message}`);
+  console.error(`Usage: ${SCRIPT} '<topic-pattern>' [delete] [--host H] [--port N] [--user U] [--password P] [--wait S]`);
+  console.error(`Example: ${SCRIPT} 'myfarm/lotus/+/sht/temperture/#'`);
+  console.error(`         ${SCRIPT} 'myfarm/lotus/+/sht/temperture/#' delete`);
+  process.exit(1);
+}
+
+// ---- arguments ----
+const opts = { host: 'localhost', port: 1883, user: null, password: null, wait: 3 };
+let pattern = null;
+let doDelete = false;
+
+const argv = process.argv.slice(2);
+for (let i = 0; i < argv.length; i++) {
+  const a = argv[i];
+  switch (a) {
+    case '--delete': doDelete = true; break;
+    case '--host': opts.host = argv[++i]; break;
+    case '--port': opts.port = Number(argv[++i]); break;
+    case '--user': opts.user = argv[++i]; break;
+    case '--password': opts.password = argv[++i]; break;
+    case '--wait': opts.wait = Number(argv[++i]); break;
+    default:
+      if (a.startsWith('-')) usage(`unknown option '${a}'`);
+      else if (pattern === null) pattern = a;
+      else if (a === 'delete') doDelete = true;
+      else usage(`unexpected argument '${a}'`);
+  }
+}
+if (!pattern) usage('no topic pattern given');
+
+// A bare "#" would be every topic on the broker, across every organization. Almost always a
+// mistake, and the one case where a slip is unrecoverable, so it is not accepted at all.
+if (pattern === '#' || pattern === '+/#') {
+  console.error(`Error: refusing the pattern '${pattern}' - that is every topic on the broker.`);
+  console.error("Name the organization at least, as in 'myfarm/#'.");
+  process.exit(1);
+}
+
+// ---- credentials ----
+// The broker requires them, and the organization's are in its config file - the same place the
+// server reads them from. The first element of the topic pattern is the organization.
+if (!opts.user) {
+  opts.user = pattern.split('/')[0];
+  const configFile = `config.d/organizations/${opts.user}.yaml`;
+  let parsed;
+  try {
+    parsed = yaml.load(readFileSync(configFile, 'utf8'));
+  } catch (e) {
+    console.error(`Error: cannot read ${configFile} (${e.code || e.message}), so the broker password`);
+    console.error(`for '${opts.user}' cannot be found. Run this from the server's own directory, or`);
+    console.error('give credentials with --user and --password.');
+    process.exit(1);
+  }
+  opts.password = parsed && parsed.mqtt_password;
+  if (!opts.password) {
+    console.error(`Error: no mqtt_password in ${configFile}`);
+    process.exit(1);
+  }
+}
+
+// ---- collect what is retained ----
+console.log(`Looking for retained messages matching '${pattern}' on ${opts.host}:${opts.port} as ${opts.user} ...`);
+
+const client = mqtt.connect(`mqtt://${opts.host}:${opts.port}`, {
+  username: opts.user,
+  password: opts.password,
+  connectTimeout: 5000,
+  // A fresh session every time: we want the retained messages delivered on subscribe, not a
+  // continuation of some earlier one.
+  clean: true,
+});
+
+const topics = new Set();
+
+client.on('error', (err) => {
+  // ECONNREFUSED arrives with an empty message, so fall back to the code - otherwise this prints
+  // "Error:" and nothing, which says neither "wrong password" nor "nothing listening there".
+  console.error(`Error: could not talk to the broker: ${err.message || err.code || err}`);
+  client.end(true);
+  process.exit(1);
+});
+
+client.on('message', (topic, _payload, packet) => {
+  // Only messages the broker had already kept. A node reporting normally while we listen arrives
+  // with the retain flag clear, and clearing that would be removing something nobody asked us to.
+  if (packet.retain) topics.add(topic);
+});
+
+client.on('connect', () => {
+  client.subscribe(pattern, { qos: 0 }, (err) => {
+    if (err) {
+      console.error(`Error: could not subscribe to '${pattern}': ${err.message}`);
+      client.end(true);
+      process.exit(1);
+    }
+    // Retained messages arrive immediately on subscribing, so this only has to cover the round
+    // trip; a busy broker on a slow Pi may want more, hence --wait.
+    setTimeout(finish, opts.wait * 1000);
+  });
+});
+
+async function finish() {
+  const list = [...topics].sort();
+  if (list.length === 0) {
+    console.log('Nothing retained matches that pattern.');
+    client.end();
+    return;
+  }
+  for (const t of list) console.log(`  ${t}`);
+  console.log(`${list.length} retained topic(s).`);
+
+  if (!doDelete) {
+    console.log('');
+    console.log(`Nothing has been changed. To delete these, run the same command again with "delete"`);
+    console.log('as the last word.');
+    client.end();
+    return;
+  }
+
+  // Deleting one means publishing a zero-length retained message to it, which is how MQTT spells
+  // "forget this". At QoS 1 so the broker acknowledges each rather than dropping it under load.
+  console.log(`Deleting ${list.length} retained topic(s) ...`);
+  let failed = 0;
+  for (const t of list) {
+    try {
+      await new Promise((resolve, reject) => {
+        client.publish(t, '', { retain: true, qos: 1 }, (err) => (err ? reject(err) : resolve()));
+      });
+      console.log(`  cleared ${t}`);
+    } catch (e) {
+      console.error(`  FAILED  ${t}: ${e.message}`);
+      failed++;
+    }
+  }
+  console.log('');
+  console.log('Done. Run the same command without "delete" - it should report nothing.');
+  console.log('If a topic comes back, a node is still publishing it: fix or restart the node, then re-run.');
+  // A bridged broker forwards these deletions to the other end, so clearing here clears there too.
+  client.end(false, {}, () => process.exit(failed ? 1 : 0));
+}

package/scripts/copy-schema-to-examples.zsh

@@ -32,13 +32,17 @@ if (( ! ${#DIRS} )); then
   exit 0
 fi
 
+# Every yaml in the schema directory, rather than a list to keep in step - devices.yaml was added
+# and left behind once already.
+FILES=("${SCHEMA}"/*.yaml(N:t))
+if (( ! ${#FILES} )); then
+  echo "  No yaml files in ${SCHEMA}"
+  exit 0
+fi
+
 COPIED=0
 for d in $DIRS; do
-  for f in topics.yaml modules.yaml; do
-    if [[ ! -f "${SCHEMA}/${f}" ]]; then
-      echo "  ${SCHEMA}/${f} does not exist - skipped"
-      continue
-    fi
+  for f in $FILES; do
     if cmp -s "${SCHEMA}/${f}" "${d}/${f}"; then
       echo "  same     ${d#${LOGGER}/}/${f}"
     else

package/scripts/diagnostic.zsh

@@ -266,6 +266,53 @@ else
   item "mosquitto_sub not installed - cannot test broker logins (sudo apt install mosquitto-clients)"
 fi
 
+section "Bridge to a production server (optional)"
+# A bridge relays this Pi's readings to a production server - see
+# extras/mosquitto-bridge.conf.example. It is configured by a "connection" line in one of
+# mosquitto's own config files, so most installations have none, which is not a problem.
+BRIDGECONFS=(/etc/mosquitto/mosquitto.conf(N) /etc/mosquitto/conf.d/*.conf(N))
+BRIDGENAMES=""
+(( ${#BRIDGECONFS} )) && BRIDGENAMES=$(grep -hE '^[[:space:]]*connection[[:space:]]' $BRIDGECONFS 2>/dev/null | awk '{print $2}')
+if [[ -z "$BRIDGENAMES" ]]; then
+  item "no bridge configured - readings stay on this Pi"
+else
+  for b in ${(f)BRIDGENAMES}; do item "configured bridge: $b"; done
+  # What it is set to relay and where. remote_password is deliberately not among these.
+  item "its settings:"
+  grep -hE '^[[:space:]]*(address|remote_username|remote_clientid|topic|cleansession|restart_timeout|notifications)[[:space:]]' $BRIDGECONFS 2>/dev/null | sed 's/^/      /'
+  # Whether it is actually up, rather than what the configuration hoped for. With "notifications"
+  # left on, the bridge publishes a retained 1 or 0 here each time it connects or drops.
+  if have mosquitto_sub; then
+    # Reading $SYS needs a broker login, so borrow the first organization that has one
+    STATEORG=""; STATEPW=""
+    for f in config.d/organizations/*.yaml(N); do
+      STATEPW=$(sed -n 's/^mqtt_password:[[:space:]]*//p' "$f" 2>/dev/null | head -1 | tr -d '"'"'"'')
+      if [[ -n "$STATEPW" ]]; then STATEORG=${f:t:r}; break; fi
+    done
+    if [[ -z "$STATEORG" ]]; then
+      item "live state: no organization credentials here to read it with"
+    else
+      STATEOUT=$(mosquitto_sub -h localhost -u "$STATEORG" -P "$STATEPW" -v -t '$SYS/broker/connection/+/state' -W 2 2>&1 | grep -E '/state ')
+      if [[ -z "$STATEOUT" ]]; then
+        item "live state: the broker reported nothing - the bridge has never connected since it started"
+        problem "A bridge is configured but has not reported its state - check 'address' is reachable and see the mosquitto log above"
+      else
+        for line in ${(f)STATEOUT}; do
+          BNAME=${${(s:/:)${line%% *}}[4]}
+          if [[ "${line##* }" == "1" ]]; then
+            item "live state: ${BNAME} is CONNECTED to production"
+          else
+            item "live state: ${BNAME} is DOWN - this Pi is recording locally only"
+            problem "Bridge '${BNAME}' is not connected to production - readings since it dropped will not appear there"
+          fi
+        done
+      fi
+    fi
+  else
+    item "mosquitto_sub not installed - cannot check whether the bridge is up"
+  fi
+fi
+
 section "Web server (step 7)"
 WEBPORT=$(grep -h '^port:' config.d/server.yaml 2>/dev/null | awk '{print $2}')
 [[ -z "$WEBPORT" ]] && WEBPORT=8080

package/scripts/init.zsh

@@ -56,7 +56,7 @@ copy_if_missing() {
 
 echo "Configuration:"
 copy_if_missing "${PKG}/config.yaml" "./config.yaml"
-for f in logger.yaml mqtt.yaml server.yaml; do
+for f in logger.yaml mqtt.yaml server.yaml email.yaml; do
   copy_if_missing "${PKG}/config.d/${f}" "./config.d/${f}"
 done
 # The schema describes the sensor types the software understands, so a release changing it matters