frugal-iot-server 2.0.5 → 2.0.7

package/INSTALLATION.md

@@ -308,9 +308,9 @@ node -v
 
 What each is for:
 
-* `nodejs` — the server needs **Node 18 or later**. Current Raspberry Pi OS (Debian 13, trixie)
-  provides 20.19.2, which is fine, on 32-bit as well as 64-bit. Older images shipped Node 18, also
-  fine. If `node -v` reports anything below 18, install a current version from
+* `nodejs` — the server needs **Node 20.17 or later**. Current Raspberry Pi OS (Debian 13, trixie)
+  provides 20.19.2, which is fine, on 32-bit as well as 64-bit. Older images shipped Node 18, which
+  is too old. If `node -v` reports anything below 20.17, install a current version from
   [NodeSource](https://github.com/nodesource/distributions) instead.
 * `npm` — installs the server; it is a separate package from `nodejs` on Debian.
 * `sqlite3` — the database the server keeps its accounts in.

package/README.md

@@ -8,7 +8,7 @@ thing in one go once you can ssh in.
 
 On a unix box that is already running ...
 
-Check you have node & npm installed `node -v` - version 18 or later.
+Check you have node & npm installed `node -v` - version 20.17 or later.
 If not then you'll need nodejs from [nodejs.org](https://nodejs.org) or your package manager.
 You also need `sqlite3` and `zsh` for the setup commands below.
 

package/frugal-iot-server.js

@@ -110,9 +110,9 @@ import { createAPIRouter, createAPIErrorHandler } from './lib/api-routes.js';
 import { buildConfigFor, hasPermissions } from './lib/config-for-user.js';
 import { ensureSecrets, addEnrolmentSecret, removeEnrolmentSecret } from './lib/secrets.js';
 import { syncUser, syncUserById, syncLoggers, syncNode, dropNode, syncBridge } from './lib/dynsec-server.js';
-import { enrol, ENROL, enrolmentSecretsFor, makeRateLimiter, forgetNode,
+import { enrol, ENROL, enrolmentSecretsFor, makeRateLimiter, forgetNode, formerProjects,
          makeAttemptLog, setGrant, clearGrant, readGrants } from './lib/enrol.js';
-import { deleteRetained } from './lib/retained.js';
+import { deleteRetained, clearRetainedUnder } from './lib/retained.js';
 import { replicaFor, bridgeForToken, noteBridgePull, startReplica } from './lib/replica.js';
 import { addMissingColumns } from './lib/schema-columns.js';
 import { createLoggerClient } from './lib/logger-client.js';
@@ -780,6 +780,38 @@ function addLoggedNodesToConfig() {
     });
   });
 }
+/*
+ * A node has enrolled in a new project, so its old address is retired: enrolment, broker account,
+ * retained messages, and what the logger and /config.json remember of it.
+ *
+ * Done here, once it has arrived, rather than when the move is asked for. A sleeping node only gets
+ * the move when it next wakes, and it needs its old credential to receive it - removed earlier, the
+ * node would be refused, enrol again in the OLD project, and never hear of the move.
+ */
+function retireFormerHome(org, project, nodeid, cb) {
+  waterfall([
+    (cb1) => forgetNode(db, org, project, nodeid, (err) => cb1(err)),
+    // Best effort, as for /node_reset - frugal-iot-diagnostic reports anything left behind.
+    (cb1) => dropNode(config, org, project, nodeid, () => cb1(null)),
+    (cb1) => clearRetainedUnder(config, org, `${org}/${project}/${nodeid}`, (err, result) => {
+      if (err) { console.error("Retained messages for", `${org}/${project}/${nodeid}`, "not cleared:", err.message); }
+      cb1(null, result);
+    }),
+    (result, cb1) => {
+      // After the retained messages, or the logger would hear them again and remember the node.
+      mqttLogger.forgetNode(org, project, nodeid);
+      const nodes = config.organizations?.[org]?.projects?.[project]?.nodes;
+      if (nodes) { delete nodes[nodeid]; }
+      console.log("Node", nodeid, "moved out of", `${org}/${project}`, "-",
+        result ? `${result.deleted} retained topic(s) cleared` : "retained topics left");
+      cb1(null);
+    },
+  ], (err) => {
+    if (err) { console.error("Could not retire", `${org}/${project}/${nodeid}:`, err.message); }
+    cb(null);
+  });
+}
+
 // See lib/config-for-user.js - kept there so it can be tested without starting a server.
 function unsafeCopyConfigFor(user) {
   return buildConfigFor(config, user);
@@ -906,7 +938,13 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
         req.body, (err, cred) => {
           if (!err) {
             console.log("Enrolled", cred.username, req.body.lora ? "(LoRa-capable)" : "");
-            return res.status(200).json(cred);
+            res.status(200).json(cred);
+            const { org, project, nodeid } = req.body;
+            formerProjects(db, org, project, nodeid, (ferr, olds) => {
+              if (ferr) { return console.error("Could not look for", nodeid, "elsewhere:", ferr.message); }
+              each(olds, (old, cb) => retireFormerHome(org, old, nodeid, cb), () => {});
+            });
+            return;
           }
           const status = {
             [ENROL.BAD_REQUEST]: 400,

package/lib/enrol.js

@@ -389,3 +389,10 @@ export function forgetNode(db, org, project, nodeid, cb) {
   db.run('DELETE FROM nodes WHERE org = ? AND project = ? AND nodeid = ?',
     [org, project, nodeid], function (err) { cb(err, this && this.changes); });
 }
+
+// Where else this node is enrolled. A node id is one device, so any other project is somewhere it
+// used to be - see retireFormerHome in frugal-iot-server.js.
+export function formerProjects(db, org, project, nodeid, cb) {
+  db.all('SELECT project FROM nodes WHERE org = ? AND nodeid = ? AND project != ?',
+    [org, nodeid, project], (err, rows) => cb(err, (rows || []).map((r) => r.project)));
+}

package/lib/retained.js

@@ -108,3 +108,61 @@ function refusal(status, message) {
 }
 function err500(message) { return refusal(500, message); }
 function refuse(cb, status, message) { return cb(refusal(status, message)); }
+
+// Reading needs a different account from deleting: the admin account may publish anywhere in the
+// organization but subscribe nowhere. The logger's reads the whole organization.
+function readCredentialFor(config, org) {
+  const o = (config && config.organizations && config.organizations[org]) || {};
+  if (o.logger_userid) return { username: o.logger_userid, password: o.logger_password };
+  if (o.mqtt_password) return { username: org, password: o.mqtt_password };
+  return null;
+}
+
+// Retained topics arrive straight after the subscription is granted; this is how long to wait for
+// the last of them, as frugal-iot-clearretained does.
+const COLLECT_MS = 3000;
+
+/*
+ * Delete every retained message under `prefix` - one node's whole subtree, when it has moved.
+ *
+ * Takes a prefix, not a pattern, so it cannot be widened to more than the caller named: the
+ * wildcard is added here.
+ */
+export function clearRetainedUnder(config, org, prefix, cb) {
+  if (!NAME.test(String(org || ''))) return refuse(cb, 400, 'Not an organization name');
+  if (!String(prefix || '').startsWith(`${org}/`) || /[+#]/.test(prefix) || prefix.split('/').length < 3) {
+    return refuse(cb, 400, `${prefix} is not a node within ${org}`);
+  }
+  const broker = config && config.mqtt && config.mqtt.broker;
+  if (!broker) return refuse(cb, 500, 'No mqtt.broker in the configuration');
+  const cred = readCredentialFor(config, org);
+  if (!cred) return refuse(cb, 500, `No broker credential to read ${org}`);
+
+  const client = mqtt.connect(broker, {
+    username: cred.username, password: cred.password,
+    connectTimeout: 5000, reconnectPeriod: 0, clean: true,
+  });
+  const topics = new Set();
+  let settled = false;
+  const finish = (err) => {
+    if (settled) return;
+    settled = true;
+    clearTimeout(timer);
+    client.end(true, () => {
+      if (err) return cb(err);
+      if (!topics.size) return cb(null, { deleted: 0, failed: 0 });
+      deleteRetained(config, org, [...topics], cb);
+    });
+  };
+  const timer = setTimeout(() => finish(err500('The broker did not answer')), 30000);
+  client.on('error', (err) => finish(refusal(502, `Could not reach the broker: ${err.message}`)));
+  client.on('message', (topic, message, packet) => {
+    if (packet && packet.retain && message.length) topics.add(topic);
+  });
+  client.on('connect', () => {
+    client.subscribe(`${prefix}/#`, { qos: 0 }, (err) => {
+      if (err) return finish(refusal(502, `Could not subscribe to ${prefix}/#: ${err.message}`));
+      setTimeout(() => finish(null), COLLECT_MS);
+    });
+  });
+}

package/package.json

@@ -12,17 +12,17 @@
     "cookie-parser": "^1.4.7",
     "express": "^5.2.1",
     "express-session": "^1.19.0",
-    "frugal-iot-client": "^2.0.5",
-    "frugal-iot-logger": "^2.0.5",
+    "frugal-iot-client": "^2.0.7",
+    "frugal-iot-logger": "^2.0.7",
     "hash-wasm": "^4.12.0",
     "js-yaml": "^4.2.0",
     "morgan": "^1.11.0",
     "mqtt": "^5.15.2",
     "multer": "^2.1.1",
-    "nodemailer": "^9.0.6",
+    "nodemailer": "^10.1.2",
     "passport": "^0.7.0",
     "passport-local": "^1.0.0",
-    "sqlite3": "^5.1.7"
+    "sqlite3": "^6.0.1"
   },
   "bin": {
     "frugal-iot-server": "frugal-iot-server.js",
@@ -63,10 +63,8 @@
     "INSTALLATION.md"
   ],
   "devDependencies": {
-    "@types/jest": "^30.0.0",
-    "jest": "^30.4.2",
     "supertest": "^7.2.2",
-    "vitest": "^1.0.0"
+    "vitest": "^4.1.12"
   },
   "homepage": "https://github.com/mitra42/frugal-iot-server",
   "keywords": [
@@ -89,5 +87,8 @@
     "check-files": "node scripts/check-published-files.js"
   },
   "type": "module",
-  "version": "2.0.5"
+  "version": "2.0.7",
+  "overrides": {
+    "uuid": "^11.1.1"
+  }
 }

package/public/service-worker.js

@@ -6,7 +6,7 @@
 // it, an installed browser keeps serving the files it already cached - and since the list now covers
 // the dashboard's own code, that means old JS behind freshly fetched HTML. Any edit to the list
 // needs this bumped, or it does not reach anyone who already has the old one.
-const CACHE_NAME = 'frugal-iot-cache-2.0.5';
+const CACHE_NAME = 'frugal-iot-cache-2.0.7';
 /*
  * Everything the app needs, so that a phone on a poor or expensive link fetches it once.
  *

package/scripts/install-pi.sh

@@ -571,10 +571,11 @@ PACKAGES=(nodejs npm sqlite3 zsh curl)
 sudo_ env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "${PACKAGES[@]}"
 NODE_VERSION="$(node -v)"
 info "node ${NODE_VERSION}, npm $(npm -v), sqlite3 $(sqlite3 --version | awk '{print $1}')"
-# The server needs node 18 or later
+# The server needs node 20.17 or later (sqlite3 6 needs 20.17, nodemailer 10 needs 20)
 NODE_MAJOR="$(echo "${NODE_VERSION#v}" | cut -d. -f1)"
-if (( NODE_MAJOR < 18 )); then
-  echo "node ${NODE_VERSION} is too old - the server needs 18 or later." >&2
+NODE_MINOR="$(echo "${NODE_VERSION#v}" | cut -d. -f2)"
+if (( NODE_MAJOR < 20 || (NODE_MAJOR == 20 && NODE_MINOR < 17) )); then
+  echo "node ${NODE_VERSION} is too old - the server needs 20.17 or later." >&2
   echo "Install a current version from https://github.com/nodesource/distributions and run this again." >&2
   exit 1
 fi

package/public/index-embedded.html

@@ -1,54 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <title>Frugal IoT  Demo</title>
-    <link href="./frugaliot.css" rel="stylesheet" type="text/css">
-    <!-- this maps files that webcomponents uses, -->
-    <script type="importmap">
-        {
-        "imports": {
-            "@kurkle/color": "https://frugaliot.naturalinnovation.org/node_modules/@kurkle/color/dist/color.esm.js",
-            "chart.js": "https://frugaliot.naturalinnovation.org/node_modules/chart.js/dist/Chart.js",
-            "dial-gauge": "/node_modules/dial-gauge/dial-gauge.js",
-            "luxon": "https://frugaliot.naturalinnovation.org/node_modules/luxon/src/luxon.js",
-            "chartjs-adapter-luxon": "https://frugaliot.naturalinnovation.org/node_modules/chartjs-adapter-luxon/dist/chartjs-adapter-luxon.esm.js",
-            "csv-parse": "https://frugaliot.naturalinnovation.org/node_modules/csv-parse/dist/esm/index.js"
-            }
-        }
-    </script>
-    <!-- load webcomponents.js either from local server, or from frugaliot.naturalinnovation.org -->
-    <script src="./webcomponents.js" type="module"></script>
-    <!--<script src="./webcomponents.js" type="module"></script>-->
-</head>
-<body>
-<h1>This is an example of embedding a widget in a page</h1>
-<div style="display:none">
-    <!-- every embedded page needs a mqtt-client, it sets up the MQTT connection to the broker -->
-    <mqtt-client server="wss://frugaliot.naturalinnovation.org/wss"></mqtt-client>
-</div>
-<div style="float:right;border:2px,solid,blue;width:50%">
-    <!-- this is a simple display widget - in this case a meter bar -->
-    <mqtt-bar max="45" min="0" topic="dev/lotus/esp8266-fb94bb/sht/temperature" label="Temperature" type="float" color="red"></mqtt-bar>
-    <!--<mqtt-bar max="45" min="0" topic="dev/lotus/esp32-70c376/soil" label="Soil" type="float" color="brown"></mqtt-bar>-->
-</div>
-<div style="float:left;border:2px,solid,blue;width:10%">
-    <!-- this is a simple control widget - in this case toggles an LED on a device at my office -->
-    <mqtt-toggle topic="dev/lotus/esp8266-fb94bb/ledbuiltin/on" label="LED" type="bool" color="blue"></mqtt-toggle>
-</div>
-<p>
-    This is a demo of a custom web page with a bar widget; a graph; and a control.
-    These all go to the same device which may or may not always be turned on!
-</p>
-<p>
-    <!-- And a pretty simple graph - in this case its displaying the same value from above, but it can
-    <!-- display any topic -->
-    <!-- Note - I think there is an issue with back button if loaded from a different server, will fix -->
-    <mqtt-graph>
-        <mqtt-graphdataset topic="dev/lotus/esp8266-fb94bb/sht/temperature" color="red" label="Office Temperature" min=0 max=50 type="float"></mqtt-graphdataset>
-        <mqtt-graphdataset topic="dev/lotus/esp8266-fb94bb/sht/humidity" color="blue" label="Office Humidity" min=0 max=100 type="float"></mqtt-graphdataset>
-        <mqtt-graphdataset topic="dev/lotus/esp8266-fb94bb/ledbuiltin/on" color="orange" label="LED" type="bool"></mqtt-graphdataset>
-    </mqtt-graph>
-</p>
-</body>
-</html>