frugal-iot-server 0.3.7 → 0.3.8

package/config.d/schema/topics.yaml

@@ -9,6 +9,11 @@
 # name  Capitalized English (and add translation below in 'languages')
 # max   For gauges, slider
 # min   For gauges, slider
+# width Total characters of the rendered number, including any sign and the decimal point and
+#   excluding units - the same meaning the node gives it. Decimals are derived, not declared:
+#   intWidth = max(len(int(min)), len(int(max))) including a '-', then decimals = width-intWidth-1.
+#   Deriving from min/max rather than from the value keeps the decimal count steady as the value
+#   moves. Only float and exponential topics need it; an int has no decimals to decide.
 # color can be a name or a #RRGGBB
 # display One of bar,gauge,text,slider,inputbox
 # slot  if this topic fits into a specific slot in the enclosing module (currently just frugal_iot module)
@@ -21,7 +26,8 @@
 #   no setting at all, any float, int or bool with "rw: r" is recorded, which is easy to be
 #   surprised by. scripts/check-schema.js warns about a topic that does not say.
 # duplicates  when to bother recording a reading: significantvalue is how much it has to have moved
-#   since the last one recorded, significantdate how long (in milliseconds) since that one, and a
+#   since the last one recorded - either an absolute amount e.g. 0.5, or as a percentage of the last
+#   reading e.g. "2%" - significantdate how long (in milliseconds) since that one, and a
 #   reading that meets neither is dropped. Every topic with "log: true" wants a rule here - without
 #   one, every reading is written to the SD card as it arrives, which wears the card out.
 # Note battery gets special cased
@@ -30,9 +36,11 @@ altitude:
   leaf:     altitude
   name:     Altitude
   type:     float
+  units:    m
   display:  text
   min:      -500
   max:      9000
+  width:    6
   color:    black
   log:      true
   duplicates:
@@ -111,6 +119,7 @@ bus:
   display:  text
   min:      0
   max:      32
+  width:    5
   color:    green
   rw:       r
   log:      true
@@ -142,6 +151,7 @@ controlfloat:
   #[leaf, name] should be overridden
   min:      0
   max:      100
+  width:    5
   type:     float
   log:      false
   display:  text
@@ -198,6 +208,7 @@ course:
   display:  text
   min:      0
   max:      360
+  width:    5
   color:    black
   log:      true
   duplicates:
@@ -213,6 +224,7 @@ current:
   display:  text
   min:      -3200
   max:      3200
+  width:    7
   color:    red
   rw:       r
   log:      true
@@ -237,6 +249,7 @@ do:
   display:  bar
   min:      0
   max:      20
+  width:    5
   color:    blue
   rw:       r
   log:      true
@@ -267,6 +280,7 @@ gas:
   display:  text
   min:      0
   max:      500
+  width:    5
   color:    green
   rw:       r
   log:      true
@@ -282,6 +296,7 @@ hdop:
   display:  text
   min:      0
   max:      50
+  width:    5
   color:    black
   log:      false
   duplicates:
@@ -296,6 +311,7 @@ humidity:
   display:  bar
   min:      0
   max:      100
+  width:    5
   color:    blue
   log:      true
   duplicates:
@@ -342,6 +358,7 @@ latitude:
   display:  text
   min:      -90
   max:      +90
+  width:    9
   color:    black
   log:      true
   duplicates:
@@ -357,6 +374,7 @@ load:
   display:  text
   min:      0
   max:      32
+  width:    5
   color:    green
   rw:       r
   log:      true
@@ -372,6 +390,7 @@ loadcell:
   display:  text
   min:      0
   max:      65000
+  width:    7
   color:    yellow
   log:      true
   duplicates:
@@ -387,6 +406,7 @@ longitude:
   display:  text
   min:      -180
   max:      +180
+  width:    10
   color:    black
   log:      true
   duplicates:
@@ -402,12 +422,13 @@ lux:
   display:  bar
   min:      0
   max:      65000
+  width:    5
   color:    yellow
   log:      true
   rw:       r
   duplicates:
     significantdate: 900000
-    significantvalue: 100
+    significantvalue: 5%
   graphable:  true
   units:    lx
 manual:
@@ -442,6 +463,7 @@ mm:
   display:  text
   min:      0
   max:      500
+  width:    5
   color:    blue
   rw:       r
   log:      true
@@ -499,6 +521,7 @@ power:
   display:  text
   min:      0
   max:      100000
+  width:    8
   color:    orange
   rw:       r
   log:      true
@@ -512,8 +535,10 @@ pressure:
   name:     Pressure
   type:     float
   display:  bar
-  min:      0
-  max:      99
+  min:      300
+  max:      1100
+  width:    6
+  units:    hPa
   color:    blue
   rw:       r
   log:      true
@@ -528,6 +553,7 @@ rain:
   display:  text
   min:      0
   max:      500
+  width:    5
   color:    blue
   rw:       r
   log:      true
@@ -543,6 +569,7 @@ satellites:
   display:  text
   min:      0
   max:      32
+  width:    2
   color:    black
   log:      false
   duplicates:
@@ -550,7 +577,7 @@ satellites:
     significantvalue: 1
   rw:       r
   graphable:  false
-  units:    m/s
+  units:    count
 select:
   leaf:     select
   name:     Select
@@ -569,6 +596,7 @@ shunt:
   display:  text
   min:      -320
   max:      320
+  width:    6
   color:    blue
   rw:       r
   log:      true
@@ -584,6 +612,7 @@ speed:
   display:  text
   min:      0
   max:      999
+  width:    5
   color:    black
   log:      true
   duplicates:
@@ -607,7 +636,7 @@ soil:
     significantvalue: 1
   graphable:  true
   calibrate:  true
-  units:    "%RH"
+  units:    "%"
 temperature:
   leaf:     temperature
   name:     Temperature
@@ -616,6 +645,7 @@ temperature:
   rw:       r
   min:      0
   max:      50
+  width:    4
   color:    red
   wireable: false
   log:      true
@@ -629,6 +659,7 @@ timeoff:
   name:     Time Off (s)
   min:      0
   max:      3600
+  width:    4
   type:     float
   log:      false
   display:  text
@@ -641,6 +672,7 @@ timeon:
   name:     Time On (s)
   min:      0
   max:      3600
+  width:    4
   type:     float
   log:      false
   display:  text
@@ -670,6 +702,7 @@ ultrasonic:
   display:  bar
   min:      0
   max:      7500
+  width:    4
   color:    blue
   rw:       r
   log:      true

package/extras/mosquitto-bridge.conf.example

@@ -0,0 +1,130 @@
+# Bridging this Pi's broker to a production Frugal IoT server
+#
+# Optional. Without this file the Pi is self-contained: nodes publish to the broker here, the
+# logger records the readings here, and the dashboard here shows them. Adding it also relays
+# everything to a production server, so the same nodes appear on a dashboard elsewhere and can be
+# controlled from it, while the Pi carries on working on its own whenever the link is down.
+#
+# This is a Mosquitto "bridge" - the broker's own feature for connecting to another broker. The
+# server, the logger and the nodes know nothing about it and need no configuration change.
+#
+# To use it:
+#   sudo cp extras/mosquitto-bridge.conf.example /etc/mosquitto/conf.d/frugal-iot-bridge.conf
+#   sudo nano /etc/mosquitto/conf.d/frugal-iot-bridge.conf     # replace every <...> below
+#   sudo systemctl restart mosquitto
+#
+# It has to be a restart, not a reload: Mosquitto does not pick up bridges on a reload signal.
+# That briefly disconnects every node, which they recover from on their own.
+#
+# The file goes in /etc/mosquitto/conf.d/ rather than in this installation, because it holds a
+# password for the production broker. Everything under config.d/ here is served to any logged-in
+# browser by /config.json, so a credential put there would not stay private.
+
+# ============ The connection ============
+
+# Names the bridge. Anything, but it appears in the log, so make it recognisable.
+connection to-prod
+
+# The production broker's plain-MQTT-over-TLS address. Not the wss:// URL the browsers and the
+# server use - a Mosquitto bridge speaks MQTT or MQTT-over-TLS and cannot use WebSockets, so
+# production needs a listener on 8883 for this. Port 1883 would work on a trusted LAN, but sends
+# the password below in clear, so over the internet use 8883 and keep the TLS lines further down.
+address <prod-host>:8883
+
+# The account this Pi uses on the production broker. Give each Pi its own, rather than sharing the
+# organization's account: the organization password is handed to every browser that logs in, so it
+# is not a secret, and a per-Pi account can be restricted to just this site's topics and revoked
+# on its own.
+remote_username <bridge-account>
+remote_password <bridge-password>
+
+# The client id this Pi presents to production, and the name production uses when it reports the
+# state of this bridge (see "notifications" below). One per Pi, or two Pis will disconnect each
+# other in a loop as each takes over the other's session.
+remote_clientid <bridge-account>
+
+# Verify production's certificate against the system CA store, which is what a Let's Encrypt
+# certificate on production needs - no certificate has to be copied onto this Pi. Delete both
+# lines, and use port 1883 above, only on a network you trust end to end.
+bridge_cafile /etc/ssl/certs/ca-certificates.crt
+bridge_insecure false
+
+# How long to wait before retrying, in seconds, doubling from the first number up to the second.
+# A Pi on a weak or intermittent link keeps retrying for as long as it takes; the cap stops a
+# flapping connection from hammering production.
+restart_timeout 5 60
+
+# Keep production's copy of our subscriptions across a dropped connection. This is the default,
+# and it matters: with "true" instead, Mosquitto re-subscribes on every reconnect and production
+# re-sends every retained message each time, which on a flaky link means a burst of duplicated
+# readings after every blip. Nodes publish nearly everything retained, so that burst is large.
+cleansession false
+
+# Publish a retained "1" or "0" to $SYS/broker/connection/<remote_clientid>/state on both brokers
+# whenever the link comes up or goes down. This is how "npx --no frugal-iot-diagnostic" reports
+# whether the bridge is connected, and it also lets production see which sites are reachable.
+# The default is true; it is written out here because it is useful rather than incidental.
+notifications true
+
+# Uncomment if production refuses the bridge's connection or logs a denied publish to $SYS. Once
+# production has an acl_file, writing to $SYS may not be permitted, and this stops trying: the
+# state is then published only to the broker here, which is all the diagnostic script reads. What
+# is lost is production's view of whether this site is reachable.
+#notifications_local_only true
+
+# Not set, and worth knowing why: local_username / local_password. The broker here requires
+# authentication, but a bridge's local side is internal to the broker and does not go through it.
+# Verified on mosquitto 2.0.21 - an anonymous publish is refused while the bridge still forwards.
+
+# ============ What gets relayed ============
+#
+# Replace <org> in all four lines with the organization id - the first element of every topic, and
+# the name of the file in config.d/organizations/. One set of four lines per organization on this
+# Pi. To relay only some projects, replace "<org>/" with "<org>/<project>/" and add "+/" where a
+# level is now missing, or repeat the block per project.
+#
+# Frugal IoT topics come in five shapes, and each line below picks out one or two of them. "+"
+# matches exactly one level, so counting levels is what separates them:
+#
+#   <org>/<project>                                        the "quick discover" announcement
+#   <org>/<project>/<node>/<module>/<field>                a reading
+#   <org>/<project>/<node>/<module>/<field>/<parameter>    a reading's min, max, colour, wiring
+#   <org>/<project>/<node>/set/<module>/<field>            a control
+#   <org>/<project>/<node>/set/<module>/<field>/<parameter>
+#
+# Readings, at five levels. These are the only topics the logger writes to disk, so this is the
+# one line where a topic must not be matched twice - each extra copy would be another row in the
+# CSV files at the far end.
+#
+# The "0" is the QoS, and it is doing real work. At 0, anything published while production is
+# unreachable is simply dropped: production's history has a gap for the outage and this Pi's is
+# complete. At 1 the broker would instead queue it all up and deliver it on reconnect, where
+# production stamps each reading with the time it arrived - so an outage would come back as a
+# whole day's readings all claiming to have happened at the moment the link returned, which is
+# worse than a gap because it looks like real data. Leave this at 0.
+topic <org>/+/+/+/+ out 0
+
+# The quick discover announcement, so production learns which nodes live here. Sent periodically
+# and not retained, so nothing accumulates while the link is down.
+topic <org>/+ out 0
+
+# Parameters - the min, max, colour and wiring a node announces once at startup. The dashboard
+# needs them to draw a node properly, and the logger never records them.
+topic <org>/+/+/+/+/+ out 0
+
+# Controls, in both directions and at QoS 1. Both directions, because a control can be set from
+# either dashboard and both brokers need to know. QoS 1 because here queueing is exactly what is
+# wanted: a control set on production while this Pi is offline is held and delivered when it comes
+# back, which is the same reason nodes publish these retained.
+#
+# At six levels this overlaps the parameter line above, so some controls reach production twice.
+# That is deliberate - it is the harmless place to put the overlap, since a repeated control
+# carries the same value and the logger discards anything with "set" in it. There is no way to
+# write "six levels but not a control", as MQTT patterns cannot exclude.
+topic <org>/+/+/set/# both 1
+
+# Loop protection is Mosquitto's "try_private", on by default, which tells production that this is
+# a bridge and not an ordinary client so that it never echoes our own messages back at us. Both
+# ends must be Mosquitto for that to hold - it was measured working between 2.0.21 and itself.
+# TODO revisit if production ever moves to a different broker, or if a third broker is ever
+# bridged into the same topics, as neither case is covered by that protection.

package/frugal-iot-server.js

@@ -255,6 +255,13 @@ function send_people_list(req, res) {
   });
 }
 
+// Sessions older than this hold that user's permissions from before the change - see the middleware
+// after passport.authenticate('session')
+const permissionsChangedAt = new Map(); // user id -> when it last changed
+function notePermissionsChanged(id) {
+  permissionsChangedAt.set(Number(id), Date.now());
+}
+
 function add_permission(id, capability, org, cb) {
   if ((id === undefined)
     || (capability === undefined) || (capability.length < 2)
@@ -267,6 +274,7 @@ function add_permission(id, capability, org, cb) {
       (cb) => db.get('SELECT COUNT(id) FROM permissions WHERE id = ? AND capability = ? AND org = ?', [id, capability, org], cb),
       (n_perms, cb) => { if (n_perms["COUNT(id)"] != 0) { cb(new Error("Duplicate permission")); } else { cb(null); }},
       (cb) => db.run(sqlAddPermission, [id, capability, org], cb),
+      (cb) => { notePermissionsChanged(id); cb(null); },
     ], cb);
   }
 }
@@ -278,6 +286,7 @@ function permissions_delete(id, capability, org, cb) {
   } else {
     waterfall([
       (cb) => db.get('DELETE FROM permissions WHERE id = ? AND capability = ? AND org = ?', [id,capability,org], cb),
+      (cb) => { notePermissionsChanged(id); cb(null); },
     ], cb);
   }
 }
@@ -806,6 +815,7 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
               email: user.email,
               phone: user.phone,
               permissions: user.permissions,
+              loginAt: Date.now(), // So a permission change can tell which sessions predate it
             });
           });
         });
@@ -822,6 +832,30 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
         // Check if have a session, and if so store in req.user, uses function defined in deserializeUser above
         app.use(passport.authenticate('session')); // Add user to req.user
 
+        // End any session that predates a change to that user's permissions.
+        //
+        // The session carries the permissions read at login, and can_READ, can_ADMIN and the rest
+        // read them from there - so without this a change takes effect only when the user next logs
+        // in. Granting is merely confusing; revoking is worse, because someone whose ADMIN has been
+        // taken away keeps it until their session ends.
+        //
+        // Rather than re-reading the table on every request to catch something this rare, note when
+        // a user's permissions changed and end the sessions older than that. They log in again,
+        // which is a small price for a change that happens a handful of times.
+        //
+        // In memory on purpose: express-session's default store keeps the sessions in this process
+        // too, so the two are lost together on a restart and neither can outlive the other.
+        app.use((req, res, next) => {
+          const changedAt = req.user && permissionsChangedAt.get(Number(req.user.id));
+          if (changedAt && (!req.user.loginAt || (changedAt > req.user.loginAt))) {
+            console.log("Permissions changed for", req.user.id, "- ending the session made before it");
+            // Carry on unauthenticated: a page redirects to login, /config.json answers 401 and the
+            // client redirects itself
+            return req.logout((err) => (err ? next(err) : next()));
+          }
+          next();
+        });
+
         // ===== API Integration: Farm IoT Interoperability Standard =====
         // Its tables are in frugal-iot-createdb.sql, already run by execSqlStart above, so there is
         // nothing to initialize here (lib/database.js still exports initializeSchema for the tests).
@@ -864,11 +898,15 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
           (req,res,next) => {
             console.log("Trying to login with redirect to",req.body.url);
             // This is ugly, but I cannot see how to pass the URL to passport.authenticate options
+            // Default the destination rather than trusting every caller to supply one: with no
+            // successRedirect, passport calls next() on success and the request 404s as
+            // "Cannot POST /login" - a confusing way to report a missing form field.
+            const returnTo = req.body.url || '/dashboard';
             passport.authenticate('local', {
               session: true,
               //failWithError: true,
-              failureRedirect: `${loginUrl}?register=false&message=Incorrect+username+or+password&url=${req.body.url}`,
-              successRedirect: req.body.url,
+              failureRedirect: `${loginUrl}?register=false&message=Incorrect+username+or+password&url=${returnTo}`,
+              successRedirect: returnTo,
               // In failure case will also be messages in the session which need clearing out TODO-N89
               //failureRedirect: `${loginUrl}?register=false&message=Incorrect+username+or+password&url=${req.body.url}`,
             })(req, res, next);
@@ -1021,6 +1059,16 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
           },
           send_projects_list,
         );
+        // Log out and go back to the login page. There was no way to do this at all before, which
+        // matters more now that a permissions change ends the session and people log in again.
+        app.get('/logout', (req, res, next) => {
+          // With a url to come back to: without one, POST /login has no successRedirect, so on a
+          // *successful* login passport calls next(), nothing else handles POST /login, and express
+          // answers "Cannot POST /login" - which looks like a broken login rather than a missing
+          // parameter.
+          req.logout((err) => (err ? next(err) : res.redirect(`${loginUrl}?register=false&url=/dashboard`)));
+        });
+
         //  /dashboard is served statically, to logged in users //TODO-N89 restrict orgs to those have permissions for (maybe handled via /config.org )
         const routerDashboard = express.Router();
         app.use('/dashboard', routerDashboard);

package/lib/housekeeping.js

@@ -19,7 +19,12 @@
  * appending to them.
  */
 
-import { createReadStream, createWriteStream, readdir, rename, stat, statfs, unlink, readFile, writeFile } from 'fs';
+import { createReadStream, createWriteStream, readdir, rename, stat, unlink, readFile, writeFile } from 'fs';
+// statfs arrived in node 18.15, and this package says it needs node 18. Taken off the namespace
+// rather than imported by name, because a named import of something a builtin does not have fails
+// when the module loads - which would stop the server outright on an 18.x older than that, instead
+// of just doing without the one check that needs it (see freePercent below).
+import fs from 'fs';
 import { createGzip, gunzip, gzip } from 'zlib';
 import path from 'path';
 import { each, eachSeries, waterfall } from 'async';
@@ -147,8 +152,8 @@ function deleteFiles(files, callback) {
  * How much of the filesystem holding "dir" is free, as a percentage, or null if it cannot be told.
  */
 function freePercent(dir, callback) {
-  if (typeof statfs !== 'function') { callback(null, null); return; } // Older node
-  statfs(dir, (err, st) => {
+  if (typeof fs.statfs !== 'function') { callback(null, null); return; } // node older than 18.15
+  fs.statfs(dir, (err, st) => {
     if (err || !st || !st.blocks) { callback(null, null); return; }
     callback(null, (st.bavail / st.blocks) * 100);
   });
@@ -189,7 +194,8 @@ export function housekeep(datadir, config, callback) {
         // Delete oldest days until there is enough room. The shortfall is worked out in bytes so
         // that the number of days removed matches what is actually needed.
         console.log(`Housekeeping: ${free.toFixed(1)}% of the disk free, below the ${freeWanted}% wanted - removing the oldest readings`);
-        statfs(datadir, (err3, st) => {
+        // Reached only when freePercent returned a number, so statfs exists on this node
+        fs.statfs(datadir, (err3, st) => {
           let shortfall = (err3 || !st) ? 0 : ((freeWanted / 100) * st.blocks - st.bavail) * st.bsize;
           let toRemove = [];
           let planned = 0;

package/package.json

@@ -12,8 +12,8 @@
     "cookie-parser": "^1.4.7",
     "express": "^5.2.1",
     "express-session": "^1.19.0",
-    "frugal-iot-client": "^1.3.13",
-    "frugal-iot-logger": "^1.1.18",
+    "frugal-iot-client": "^1.3.14",
+    "frugal-iot-logger": "^1.1.20",
     "hash-wasm": "^4.12.0",
     "js-yaml": "^4.2.0",
     "morgan": "^1.11.0",
@@ -27,7 +27,10 @@
     "frugal-iot-init": "scripts/init.zsh",
     "frugal-iot-addorganization": "scripts/addorganization.zsh",
     "frugal-iot-setpassword": "scripts/setpassword.zsh",
-    "frugal-iot-diagnostic": "scripts/diagnostic.zsh"
+    "frugal-iot-diagnostic": "scripts/diagnostic.zsh",
+    "frugal-iot-clearretained": "scripts/clearretained.zsh",
+    "frugal-iot-addbridge-prod": "scripts/addbridge-prod.zsh",
+    "frugal-iot-addbridge-pi": "scripts/addbridge-pi.zsh"
   },
   "description": "Frugal IoT client and server",
   "//files": "An allowlist, not an ignore list - anything not named here is never published, so a stray file (a database backup, a credential) cannot leak by default. Check with: npm pack --dry-run",
@@ -44,6 +47,7 @@
     "private/",
     "scripts/",
     "extras/mosquitto.conf",
+    "extras/mosquitto-bridge.conf.example",
     "extras/frugaliot.service",
     "API.md",
     "INSTALLATION.md"
@@ -68,8 +72,9 @@
     "test:watch": "vitest",
     "test:coverage": "vitest run --coverage",
     "check-schema": "node scripts/check-schema.js",
-    "prerelease": "zsh scripts/prerelease.zsh"
+    "prerelease": "zsh scripts/prerelease.zsh",
+    "check-files": "node scripts/check-published-files.js"
   },
   "type": "module",
-  "version": "0.3.7"
+  "version": "0.3.8"
 }

package/public/index.html

@@ -167,6 +167,14 @@
 <div class="nav-section">
     <h2>Other examples</h2>
     <div class="box-grid">
+        <!-- The display the Dashboard used before the cards. Kept reachable while people get used
+             to the new one; it will go once nothing needs it. -->
+        <div class="nav-box">
+            <div class="nav-box-background" style="background-image: url('/images/icon-dashboard.png')"></div>
+            <div class="nav-box-content">
+                <a href="/dashboard/index-old.html">Previous dashboard</a>
+            </div>
+        </div>
         <div class="nav-box">
             <div class="nav-box-background" style="background-image: url('/images/icon-dashboard.png')"></div>
             <div class="nav-box-content">

package/public/service-worker.js

@@ -1,6 +1,7 @@
-// Change this version number if want existing installs of PWA to refresh their cache
-// TODO-28 - probably make this the same as the app version in frugal-iot-client
-const CACHE_NAME = 'frugal-iot-cache-1.3.13';
+// Changing this version is what makes an already installed PWA fetch the app again instead of
+// serving what it cached. It follows the version of frugal-iot-client this release installs, and
+// "npm run prerelease" sets it - so there is normally no reason to edit it by hand.
+const CACHE_NAME = 'frugal-iot-cache-1.3.14';
 const urlsToCache = [
     '/',
     '/index.html',