frugal-iot-server 2.0.6 → 2.0.8

package/package.json

@@ -12,17 +12,17 @@
     "cookie-parser": "^1.4.7",
     "express": "^5.2.1",
     "express-session": "^1.19.0",
-    "frugal-iot-client": "^2.0.6",
-    "frugal-iot-logger": "^2.0.6",
+    "frugal-iot-client": "^2.0.8",
+    "frugal-iot-logger": "^2.0.8",
     "hash-wasm": "^4.12.0",
     "js-yaml": "^4.2.0",
     "morgan": "^1.11.0",
     "mqtt": "^5.15.2",
     "multer": "^2.1.1",
-    "nodemailer": "^9.0.6",
+    "nodemailer": "^10.1.2",
     "passport": "^0.7.0",
     "passport-local": "^1.0.0",
-    "sqlite3": "^5.1.7"
+    "sqlite3": "^6.0.1"
   },
   "bin": {
     "frugal-iot-server": "frugal-iot-server.js",
@@ -35,7 +35,9 @@
     "frugal-iot-addbridge-pi": "scripts/addbridge-pi.zsh",
     "frugal-iot-rebuild-dynsec": "scripts/rebuild-dynsec.js",
     "frugal-iot-dynsec-init": "scripts/dynsec-init.js",
-    "frugal-iot-resetnode": "scripts/resetnode.js"
+    "frugal-iot-resetnode": "scripts/resetnode.js",
+    "frugal-iot-grafana-token": "scripts/grafana-token.js",
+    "frugal-iot-backfill-opensearch": "scripts/backfill-opensearch.js"
   },
   "description": "Frugal IoT client and server",
   "//files": "An allowlist, not an ignore list - anything not named here is never published, so a stray file (a database backup, a credential) cannot leak by default. Check with: npm pack --dry-run",
@@ -46,12 +48,14 @@
     "config.d/email.yaml",
     "config.d/logger.yaml",
     "config.d/mqtt.yaml",
+    "config.d/opensearch.yaml",
     "config.d/server.yaml",
     "config.d/schema/",
     "lib/",
     "public/",
     "private/",
     "scripts/",
+    "extras/grafana/",
     "extras/mosquitto.conf",
     "extras/aclfile",
     "extras/mosquitto-bridge.conf.example",
@@ -60,13 +64,13 @@
     "extras/frugaliot-ap-watchdog.service",
     "extras/frugaliot-ap-watchdog.timer",
     "API.md",
+    "GRAFANA.md",
+    "INSTALL-OPENSEARCH.md",
     "INSTALLATION.md"
   ],
   "devDependencies": {
-    "@types/jest": "^30.0.0",
-    "jest": "^30.4.2",
     "supertest": "^7.2.2",
-    "vitest": "^1.0.0"
+    "vitest": "^4.1.12"
   },
   "homepage": "https://github.com/mitra42/frugal-iot-server",
   "keywords": [
@@ -89,5 +93,8 @@
     "check-files": "node scripts/check-published-files.js"
   },
   "type": "module",
-  "version": "2.0.6"
+  "version": "2.0.8",
+  "overrides": {
+    "uuid": "^11.1.1"
+  }
 }

package/public/service-worker.js

@@ -6,7 +6,7 @@
 // it, an installed browser keeps serving the files it already cached - and since the list now covers
 // the dashboard's own code, that means old JS behind freshly fetched HTML. Any edit to the list
 // needs this bumped, or it does not reach anyone who already has the old one.
-const CACHE_NAME = 'frugal-iot-cache-2.0.6';
+const CACHE_NAME = 'frugal-iot-cache-2.0.8';
 /*
  * Everything the app needs, so that a phone on a poor or expensive link fetches it once.
  *

package/scripts/backfill-opensearch.js

@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+/*
+ * Load an organization's recorded readings into OpenSearch - the history from before its OpenSearch
+ * forwarder was switched on, or a gap from while OpenSearch was down.
+ *
+ *   npx --no frugal-iot-backfill-opensearch <org>                       everything
+ *   npx --no frugal-iot-backfill-opensearch <org> 2026-09-01            only files from that day on
+ *   npx --no frugal-iot-backfill-opensearch <org> dry-run               count, and send nothing
+ *
+ * Run from the server's own directory. It uses the organization's OpenSearch forwarder as set up on
+ * the dashboard's Forwarders card (a switched-off one is not used), and its "Only these topics" list.
+ * Safe to repeat, and to run while the server is: a reading's id comes from its topic and time, so one
+ * already in OpenSearch is replaced rather than doubled.
+ *
+ * Positional arguments, no flags: npm swallows any --flag it does not recognise. See the npx note in
+ * claude.md.
+ */
+
+import sqlite3 from 'sqlite3';
+import { MqttLogger, backfillOpenSearch } from 'frugal-iot-logger';
+import { initializeSchema } from '../lib/database.js';
+import { enabledConfigs } from '../lib/forwarders.js';
+
+const [org, ...rest] = process.argv.slice(2);
+const dryRun = rest.includes('dry-run');
+const since = rest.find((a) => /^\d{4}-\d{2}-\d{2}$/.test(a));
+if (!org || rest.some((a) => a !== 'dry-run' && a !== since)) {
+  console.error('Usage: npx --no frugal-iot-backfill-opensearch <org> [YYYY-MM-DD] [dry-run]');
+  process.exit(1);
+}
+
+const realLog = console.log;
+console.log = () => {};                       // the config reader logs a line per file
+new MqttLogger().readYamlConfig('.', (err, config) => {
+  console.log = realLog;
+  if (err) { console.error(`Could not read the configuration: ${err.message}`); process.exit(1); }
+  if (!(config.organizations || {})[org]) { console.error(`No organization "${org}"`); process.exit(1); }
+  const db = new sqlite3.Database('./frugal-iot.db', async (dberr) => {
+    if (dberr) { console.error(`Could not open ./frugal-iot.db: ${dberr.message}`); process.exit(1); }
+    try {
+      await initializeSchema(db); // makes the table if this database is from before it existed
+      const fromDb = (await enabledConfigs(db, org, config.opensearch)).filter((c) => c.type === 'opensearch');
+      // Still honoured while YAML forwarders are supported, so a server not yet moved over works
+      const fromFile = (config.organizations[org].forwarders || []).filter((c) => c && c.type === 'opensearch');
+      const forwarders = [...fromDb, ...fromFile];
+      if (forwarders.length === 0) {
+        console.error(`${org} has no OpenSearch forwarder - add one on the dashboard's Forwarders card first`);
+        process.exitCode = 1;
+        return;
+      }
+      for (const fconfig of forwarders) {
+        const { sent, skippedFiles } = await backfillOpenSearch({
+          config: fconfig, org, dataDir: config.server.datadir, since, dryRun,
+          progress: (n) => process.stdout.write(`\r${n} readings`),
+        });
+        console.log(`\n${dryRun ? 'Would send' : 'Sent'} ${sent} readings to ${fconfig.node}` +
+          (skippedFiles ? `; ${skippedFiles} files skipped by its topics list` : ''));
+      }
+    } catch (e) {
+      console.error(e.message);
+      process.exitCode = 1;
+    } finally {
+      db.close();
+    }
+  });
+});

package/scripts/grafana-token.js

@@ -0,0 +1,62 @@
+#!/usr/bin/env node
+/*
+ * Issue, list or revoke the tokens Grafana uses to read an organization's history (GRAFANA.md).
+ *
+ *   npx --no frugal-iot-grafana-token add <org> <username> <name>   prints the token - once, it is not kept
+ *   npx --no frugal-iot-grafana-token list <org>
+ *   npx --no frugal-iot-grafana-token revoke <org> <name>
+ *
+ * Run from the server's own directory. Adding a name that exists replaces its token, which is how
+ * one is rotated. A token is read-only, for that one organization, and acts as <username>: it sees
+ * what that user may READ, and stops working when they lose access.
+ *
+ * Positional arguments, no flags: npm swallows any --flag it does not recognise. See the npx note
+ * in claude.md.
+ */
+
+import sqlite3 from 'sqlite3';
+import { initializeSchema } from '../lib/database.js';
+import { addMissingColumns } from '../lib/schema-columns.js';
+import { issueToken, listTokens, revokeToken } from '../lib/grafana.js';
+
+const [action, org, ...rest] = process.argv.slice(2);
+const [username, name] = action === 'add' ? rest : [undefined, rest[0]];
+const usage = () => {
+  console.error('Usage: npx --no frugal-iot-grafana-token add <org> <username> <name>');
+  console.error('       npx --no frugal-iot-grafana-token list <org>');
+  console.error('       npx --no frugal-iot-grafana-token revoke <org> <name>');
+  process.exit(1);
+};
+if (!['add', 'list', 'revoke'].includes(action) || !org || (action === 'add' && !username) || (action !== 'list' && !name)) usage();
+if (name && !/^[A-Za-z0-9][A-Za-z0-9 _.-]{0,39}$/.test(name)) {
+  console.error('The name must be 1-40 characters: letters, digits, space, _ . -');
+  process.exit(1);
+}
+
+const db = new sqlite3.Database('./frugal-iot.db', async (err) => {
+  if (err) { console.error(`Could not open ./frugal-iot.db: ${err.message}`); process.exit(1); }
+  try {
+    await initializeSchema(db); // adds the table to a database from before it existed; changes nothing else
+    await new Promise((res, rej) => addMissingColumns(db, (e) => (e ? rej(e) : res())));
+    if (action === 'add') {
+      const user = await new Promise((res, rej) => db.get('SELECT id FROM users WHERE username = ?', [username], (e, r) => (e ? rej(e) : res(r))));
+      if (!user) throw new Error(`No user "${username}"`);
+      const token = await issueToken(db, org, name, user.id);
+      console.log(`Token for ${org} "${name}" (acting as ${username}) - shown once, put it in Grafana now:\n\n${token}\n`);
+    } else if (action === 'list') {
+      const rows = await listTokens(db, org);
+      if (rows.length === 0) console.log(`No Grafana tokens for ${org}`);
+      for (const r of rows) {
+        console.log(`${r.name}\tissued ${new Date(r.created_at).toISOString()}\tlast used ${r.last_used ? new Date(r.last_used).toISOString() : 'never'}`);
+      }
+    } else {
+      const done = await revokeToken(db, org, name);
+      console.log(done ? `Revoked "${name}" for ${org}` : `No token "${name}" for ${org}`);
+      if (!done) process.exitCode = 1;
+    }
+  } catch (e) {
+    console.error(e.message);
+    process.exitCode = 1;
+  }
+  db.close();
+});

package/scripts/install-pi.sh

@@ -571,10 +571,11 @@ PACKAGES=(nodejs npm sqlite3 zsh curl)
 sudo_ env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "${PACKAGES[@]}"
 NODE_VERSION="$(node -v)"
 info "node ${NODE_VERSION}, npm $(npm -v), sqlite3 $(sqlite3 --version | awk '{print $1}')"
-# The server needs node 18 or later
+# The server needs node 20.17 or later (sqlite3 6 needs 20.17, nodemailer 10 needs 20)
 NODE_MAJOR="$(echo "${NODE_VERSION#v}" | cut -d. -f1)"
-if (( NODE_MAJOR < 18 )); then
-  echo "node ${NODE_VERSION} is too old - the server needs 18 or later." >&2
+NODE_MINOR="$(echo "${NODE_VERSION#v}" | cut -d. -f2)"
+if (( NODE_MAJOR < 20 || (NODE_MAJOR == 20 && NODE_MINOR < 17) )); then
+  echo "node ${NODE_VERSION} is too old - the server needs 20.17 or later." >&2
   echo "Install a current version from https://github.com/nodesource/distributions and run this again." >&2
   exit 1
 fi

package/public/index-embedded.html

@@ -1,54 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <title>Frugal IoT  Demo</title>
-    <link href="./frugaliot.css" rel="stylesheet" type="text/css">
-    <!-- this maps files that webcomponents uses, -->
-    <script type="importmap">
-        {
-        "imports": {
-            "@kurkle/color": "https://frugaliot.naturalinnovation.org/node_modules/@kurkle/color/dist/color.esm.js",
-            "chart.js": "https://frugaliot.naturalinnovation.org/node_modules/chart.js/dist/Chart.js",
-            "dial-gauge": "/node_modules/dial-gauge/dial-gauge.js",
-            "luxon": "https://frugaliot.naturalinnovation.org/node_modules/luxon/src/luxon.js",
-            "chartjs-adapter-luxon": "https://frugaliot.naturalinnovation.org/node_modules/chartjs-adapter-luxon/dist/chartjs-adapter-luxon.esm.js",
-            "csv-parse": "https://frugaliot.naturalinnovation.org/node_modules/csv-parse/dist/esm/index.js"
-            }
-        }
-    </script>
-    <!-- load webcomponents.js either from local server, or from frugaliot.naturalinnovation.org -->
-    <script src="./webcomponents.js" type="module"></script>
-    <!--<script src="./webcomponents.js" type="module"></script>-->
-</head>
-<body>
-<h1>This is an example of embedding a widget in a page</h1>
-<div style="display:none">
-    <!-- every embedded page needs a mqtt-client, it sets up the MQTT connection to the broker -->
-    <mqtt-client server="wss://frugaliot.naturalinnovation.org/wss"></mqtt-client>
-</div>
-<div style="float:right;border:2px,solid,blue;width:50%">
-    <!-- this is a simple display widget - in this case a meter bar -->
-    <mqtt-bar max="45" min="0" topic="dev/lotus/esp8266-fb94bb/sht/temperature" label="Temperature" type="float" color="red"></mqtt-bar>
-    <!--<mqtt-bar max="45" min="0" topic="dev/lotus/esp32-70c376/soil" label="Soil" type="float" color="brown"></mqtt-bar>-->
-</div>
-<div style="float:left;border:2px,solid,blue;width:10%">
-    <!-- this is a simple control widget - in this case toggles an LED on a device at my office -->
-    <mqtt-toggle topic="dev/lotus/esp8266-fb94bb/ledbuiltin/on" label="LED" type="bool" color="blue"></mqtt-toggle>
-</div>
-<p>
-    This is a demo of a custom web page with a bar widget; a graph; and a control.
-    These all go to the same device which may or may not always be turned on!
-</p>
-<p>
-    <!-- And a pretty simple graph - in this case its displaying the same value from above, but it can
-    <!-- display any topic -->
-    <!-- Note - I think there is an issue with back button if loaded from a different server, will fix -->
-    <mqtt-graph>
-        <mqtt-graphdataset topic="dev/lotus/esp8266-fb94bb/sht/temperature" color="red" label="Office Temperature" min=0 max=50 type="float"></mqtt-graphdataset>
-        <mqtt-graphdataset topic="dev/lotus/esp8266-fb94bb/sht/humidity" color="blue" label="Office Humidity" min=0 max=100 type="float"></mqtt-graphdataset>
-        <mqtt-graphdataset topic="dev/lotus/esp8266-fb94bb/ledbuiltin/on" color="orange" label="LED" type="bool"></mqtt-graphdataset>
-    </mqtt-graph>
-</p>
-</body>
-</html>