frugal-iot-server 2.0.6 → 2.0.8

package/extras/grafana/frugaliot-readings.json

@@ -0,0 +1,166 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_FRUGAL_IOT",
+      "label": "Frugal IoT data source",
+      "description": "The Infinity data source holding this organization's token",
+      "type": "datasource",
+      "pluginId": "yesoreyeram-infinity-datasource",
+      "pluginName": "Infinity"
+    },
+    {
+      "name": "VAR_SERVER",
+      "type": "constant",
+      "label": "Server URL",
+      "description": "Where the Frugal IoT server is, no trailing slash",
+      "value": "https://frugaliot.naturalinnovation.org"
+    },
+    {
+      "name": "VAR_ORG",
+      "type": "constant",
+      "label": "Organization",
+      "description": "The organization the token is for",
+      "value": "dev"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "datasource",
+      "id": "yesoreyeram-infinity-datasource",
+      "name": "Infinity",
+      "version": "4.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    }
+  ],
+  "title": "Frugal IoT readings",
+  "uid": "frugaliot-readings",
+  "tags": [
+    "frugal-iot"
+  ],
+  "schemaVersion": 39,
+  "time": {
+    "from": "now-7d",
+    "to": "now"
+  },
+  "templating": {
+    "list": [
+      {
+        "name": "server",
+        "type": "constant",
+        "query": "${VAR_SERVER}",
+        "hide": 2,
+        "skipUrlSync": true
+      },
+      {
+        "name": "org",
+        "type": "constant",
+        "query": "${VAR_ORG}",
+        "hide": 2,
+        "skipUrlSync": true
+      },
+      {
+        "name": "topic",
+        "type": "query",
+        "label": "Topic",
+        "datasource": {
+          "type": "yesoreyeram-infinity-datasource",
+          "uid": "${DS_FRUGAL_IOT}"
+        },
+        "description": "Every topic this organization has readings for",
+        "query": {
+          "queryType": "infinity",
+          "refId": "variable",
+          "query": "",
+          "infinityQuery": {
+            "refId": "variable",
+            "type": "json",
+            "source": "url",
+            "format": "table",
+            "parser": "backend",
+            "url": "${server}/grafana/${org}/topics",
+            "url_options": {
+              "method": "GET"
+            },
+            "root_selector": "",
+            "columns": [
+              {
+                "selector": "topic",
+                "text": "topic",
+                "type": "string"
+              }
+            ]
+          }
+        },
+        "refresh": 1,
+        "sort": 1,
+        "includeAll": false,
+        "multi": false
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "timeseries",
+      "title": "$topic",
+      "datasource": {
+        "type": "yesoreyeram-infinity-datasource",
+        "uid": "${DS_FRUGAL_IOT}"
+      },
+      "gridPos": {
+        "h": 14,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "legend": {
+          "showLegend": false
+        }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "spanNulls": true
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "yesoreyeram-infinity-datasource",
+            "uid": "${DS_FRUGAL_IOT}"
+          },
+          "type": "json",
+          "source": "url",
+          "format": "timeseries",
+          "parser": "backend",
+          "url": "${server}/grafana/${org}/data?topic=${topic}&from=${__from}&to=${__to}&maxPoints=1000",
+          "url_options": {
+            "method": "GET"
+          },
+          "root_selector": "",
+          "columns": [
+            {
+              "selector": "time",
+              "text": "time",
+              "type": "timestamp"
+            },
+            {
+              "selector": "value",
+              "text": "value",
+              "type": "number"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

package/frugal-iot-createdb.sql

@@ -88,6 +88,44 @@ CREATE TABLE IF NOT EXISTS `bridges` (
   UNIQUE(`org`, `site`)
 );
 
+-- Where an organization's readings are forwarded to (OpenSearch, Google Sheets), added from the
+-- dashboard's admin tab - see lib/forwarders.js. They are in addition to any "forwarders:" in the
+-- organization's YAML file, which stays as it is and is shown there read-only.
+--
+-- `config` is JSON in the same form as a YAML entry, minus "type". It includes passwords and API
+-- keys, in the clear, as the YAML file does: this database is not served and is backed up with the
+-- rest of the server, and the dashboard is only ever told THAT a secret is set, never its value.
+CREATE TABLE IF NOT EXISTS `forwarders` (
+  `id` INTEGER PRIMARY KEY AUTOINCREMENT,
+  `org` TEXT NOT NULL,
+  `type` TEXT NOT NULL,
+  `name` TEXT NOT NULL,
+  `enabled` INTEGER NOT NULL DEFAULT 1,
+  `config` TEXT NOT NULL,
+  `created_at` INTEGER NOT NULL,
+  `updated_at` INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS `forwarders_org` ON `forwarders` (`org`);
+
+-- A credential for Grafana (or anything else that reads history over HTTP), see lib/grafana.js and
+-- GRAFANA.md. Read-only, and scoped to one organization: it can fetch that organization's readings
+-- through /grafana/:org/ and nothing else.
+--
+-- Only a SHA-256 of the token is kept, so a copy of this database does not hand out working
+-- credentials - the token itself is shown once, when it is issued. It is 32 random bytes, which is
+-- why a plain hash is enough and no salt or slow hash is needed. One token per name, so one
+-- Grafana can be revoked without disturbing another; issuing a name again replaces its token.
+CREATE TABLE IF NOT EXISTS `grafana_tokens` (
+  `org` TEXT NOT NULL,
+  `name` TEXT NOT NULL,
+  `token_hash` TEXT NOT NULL,
+  `created_at` INTEGER NOT NULL,
+  `last_used` INTEGER, -- Approximate - only rewritten if more than an hour old, to spare the SD card
+  `user_id` INTEGER, -- whose permissions the token acts with; NULL (from before this) is not honoured
+  UNIQUE(`org`, `name`)
+);
+CREATE INDEX IF NOT EXISTS `grafana_tokens_hash` ON `grafana_tokens` (`token_hash`);
+
 -- An admin's decision about one node, for the Enrolled column on the dashboard's Nodes card
 -- (SECURITY.md S12). Two states are worth storing:
 --

package/frugal-iot-server.js

@@ -107,6 +107,9 @@ import { MqttLogger } from "frugal-iot-logger";  // https://github.com/mitra42/f
 
 // API Integration - Farm IoT Interoperability Standard
 import { createAPIRouter, createAPIErrorHandler } from './lib/api-routes.js';
+import { createGrafanaRouters } from './lib/grafana.js';
+import { createForwardersRouter, enabledConfigs } from './lib/forwarders.js';
+import { createAccessRouter } from './lib/opensearch-access.js';
 import { buildConfigFor, hasPermissions } from './lib/config-for-user.js';
 import { ensureSecrets, addEnrolmentSecret, removeEnrolmentSecret } from './lib/secrets.js';
 import { syncUser, syncUserById, syncLoggers, syncNode, dropNode, syncBridge } from './lib/dynsec-server.js';
@@ -665,6 +668,15 @@ function can_OTAUPDATE(req, res, next) {
     res.sendStatus(401); // Just fail - shouldnt happen and anyway lost the file by now
   }
 }
+// Listing and downloading firmware is for anyone who can push it or flash it; deleting stays OTAUPDATE.
+function can_OTAUPDATE_or_OTAFLASH(req, res, next) {
+  if (req.isAuthenticated() && (hasPermissions(req.user, req.params.org, "OTAUPDATE")
+      || hasPermissions(req.user, req.params.org, "OTAFLASH"))) {
+    next();
+  } else {
+    res.sendStatus(401);
+  }
+}
 // Exported for reuse by lib/api-routes.js - reads org from req.params.org or res.locals.org, so callers must set one of those first.
 export function can_READ(req, res, next) {
   const org = req.params.org || res.locals.org;
@@ -699,6 +711,15 @@ export function can_ADMIN(req, res, next) {
     res.sendStatus(401);
   }
 }
+// Whoever can flash a board can enrol it, so reading the enrolment secret needs no more than OTAFLASH.
+function can_ADMIN_or_OTAFLASH(req, res, next) {
+  if (req.isAuthenticated() && (hasPermissions(req.user, req.params.org, "ADMIN")
+      || hasPermissions(req.user, req.params.org, "OTAFLASH"))) {
+    next();
+  } else {
+    res.sendStatus(401);
+  }
+}
 // Exported for reuse by lib/api-routes.js - for resources (like api_platforms) that aren't scoped to a
 // single org, valid if the user has ADMIN on ANY org, rather than a specific one.
 export function can_ADMIN_SOME(req, res, next) {
@@ -1132,7 +1153,13 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
           secret: config.secrets.session_secret,
           resave: false,
           saveUninitialized: false,
-          cookie: { secure: 'auto' }  // TODO-N89 cant be secure: true while testing on HTTP
+          // "lax": the browser does not send the session cookie on a request made by a page on ANOTHER site
+          // (a form posted from there, a hidden image, a frame), only when the user themselves navigates
+          // here, e.g. by following a link. That is what stops another site acting as a logged-in
+          // administrator. Not "strict", which would also drop it on a link from an email or chat, so
+          // the first click on one would arrive logged out. Chrome and Edge already treat a cookie with
+          // no SameSite as lax; this makes it so for every browser.
+          cookie: { secure: 'auto', sameSite: 'lax' }  // TODO-N89 cant be secure: true while testing on HTTP
         }));
         // This defines he function that will be used to turn user data returned from database into object for the session
         passport.serializeUser(function(user, cb) {
@@ -1214,6 +1241,16 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
         app.use('/api', apiRouter);
         console.log("Mounted API routes at /api");
 
+        // History for Grafana, by bearer token - see lib/grafana.js and GRAFANA.md
+        const grafana = createGrafanaRouters(db, config.server.datadir, loggerClient, { loggedInOrFail, can_READ });
+        app.use('/grafana', grafana.data);
+        app.use('/grafana_token', grafana.admin);
+
+        // Where each organization's readings are forwarded, edited from the admin tab - lib/forwarders.js
+        app.use('/forwarders', createForwardersRouter(db, mqttLogger, () => config.organizations || {}, { loggedInOrFail, can_ADMIN }, { localOpenSearch: () => config.opensearch }));
+        app.use('/opensearch_access', createAccessRouter(() => config.organizations || {}, { loggedInOrFail, can_READ }, () => config.opensearch));
+        console.log("Mounted Grafana routes at /grafana");
+
         // Add API error handler
         app.use(createAPIErrorHandler()); // createAPIErrorHandler returns a function to use as error handler
         console.log("Added API error handler");
@@ -1383,7 +1420,7 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
         );
         app.get('/ota_list/:org',
           loggedInOrFail,
-          can_OTAUPDATE,
+          can_OTAUPDATE_or_OTAFLASH,
           (req,res) => {
             // TODO-N89 list all OTA files for an organization
             get_ota_dirs(req.params.org, (err, dirs) => {
@@ -1427,7 +1464,7 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
         // and answers 304 rather than sending bytes.
         app.get('/ota_get/:org/*remainingpath',
           loggedInOrFail,
-          can_OTAUPDATE,
+          can_OTAUPDATE_or_OTAFLASH,
           (req,res) => {
             let filepath = safeJoin(config.server.otadir, req.params.org, ...[].concat(req.params.remainingpath), 'firmware.bin');
             if (!filepath) {
@@ -1534,7 +1571,7 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
          */
         app.get('/enrolment_secret/:org',
           loggedInOrFail,
-          can_ADMIN,  // Gets org from URL - an admin sees only their own organization's
+          can_ADMIN_or_OTAFLASH,  // Gets org from URL - sees only their own organization's; rotating stays ADMIN
           (req, res) => {
             const org = req.params.org;
             // Worth a line: this is the one route that hands a secret to a browser.
@@ -1983,7 +2020,12 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
             o.logger_userid = cred.username;
             o.logger_password = cred.password;
           });
-          mqttLogger.start();
+          // Forwarders kept in the database go to the logger before it starts, so they are running when
+          // the broker connects. One that cannot be read is reported and the logger starts without it.
+          Promise.all(Object.keys(config.organizations || {}).map(async (org) => {
+            try { await mqttLogger.setForwarders(org, await enabledConfigs(db, org, config.opensearch)); }
+            catch (ferr) { console.error(`Could not load the forwarders for ${org}:`, ferr.message); }
+          })).then(() => mqttLogger.start());
         });
 
         // On a bridged Pi, keep production's logins and permissions in step (SECURITY.md S11).

package/lib/admin-guards.js

@@ -0,0 +1,26 @@
+/*
+ * Checks shared by the routes an organization's administrator uses to hand out access (Grafana
+ * tokens) and to send readings elsewhere (forwarders). Both act on the WHOLE organization, so they ask
+ * for more than the ordinary ADMIN check does.
+ */
+
+// A form POST needs no CORS preflight, so a page on another site can send one using an
+// administrator's logged-in session; JSON it cannot send without the browser asking this server first,
+// and the server does not agree. This server also parses form bodies for every route (login needs it),
+// so what these routes must do is refuse them. Not a replacement for a SameSite session cookie.
+export function jsonOnly(req, res, next) {
+  if (req.is('application/json')) return next();
+  const message = 'Send JSON (Content-Type: application/json)';
+  res.status(415).json({ error: message, message });
+}
+
+// ADMIN for the whole organization, not just for one project of it. can_ADMIN answers "anywhere in
+// this organization", so an administrator of a single project passes it - and a token or a forwarder
+// reaches every project's readings.
+export function orgWideAdmin(req, res, next) {
+  const ok = req.user && Array.isArray(req.user.permissions)
+    && req.user.permissions.some((p) => p.capability === 'ADMIN' && p.org === req.params.org && !p.project);
+  if (ok) return next();
+  const message = 'This needs ADMIN for the whole organization, not for one project of it';
+  res.status(403).json({ error: message, message });
+}

package/lib/config-for-user.js

@@ -26,6 +26,9 @@ export const configSectionsNeverServed = [
   // the client reads any of it: whether password reset is available is decided server-side by
   // mailConfigured(). Found by the "nothing secret-looking" test below, on its first run.
   'email',
+  // config.d/opensearch.yaml - the address and login of this server's own OpenSearch, including the
+  // account that can create logins on it
+  'opensearch',
 ];
 
 // Fields within an organization's own config that are not sent.
@@ -43,6 +46,13 @@ export const orgFieldsNeverServed = [
   // pick it up. It can read the whole organization, so it is at least as sensitive as the shared
   // password was.
   'logger_password',
+  // Where this organization's readings are forwarded to (OpenSearch, Google Sheets, Firebase), which
+  // is credentials: an OpenSearch password or API key, the Google Sheets webhook URL (anyone holding
+  // it can write to that sheet) and the path to a Firebase service account. 
+  // The older per-type keys are withheld as well, for a config not yet moved to "forwarders:".
+  'forwarders',
+  'gsheets',
+  'firebase',
 ];
 
 /*

package/lib/destinations.js

@@ -0,0 +1,121 @@
+/*
+ * Where this server may be told to connect.
+ *
+ * An organization's administrator chooses the address a forwarder sends to, and it is this server
+ * that connects, from inside whatever network it sits in. So an address is judged by what it IS, not
+ * by how it is spelled: a list of bad names loses to the many ways of writing the same address
+ * (IPv4 inside IPv6, a trailing dot, a name that resolves somewhere private).
+ *
+ *   private, link-local (where cloud metadata lives), shared, reserved, multicast - refused
+ *   loopback - allowed, but only on a port the caller names (OpenSearch's 9200), not the server's own
+ *              other services; not for Google Sheets at all
+ *   public - allowed
+ *
+ * A name is resolved, and refused if ANY of its addresses is refused. This is checked when a forwarder
+ * is saved. A name that resolves somewhere else later (DNS rebinding) is not caught by it, and the
+ * reply to a forwarder's request is never shown to the browser - so this limits what an administrator
+ * can reach and probe, but is not a replacement for the machine's own firewall.
+ */
+
+import net from 'net';
+import dns from 'dns';
+
+// Names that mean "the cloud's metadata service", whatever they resolve to
+const METADATA_NAMES = new Set(['metadata', 'metadata.google.internal', 'instance-data', 'instance-data.ec2.internal']);
+
+// 16 bytes of an IPv6 address, or null. Handles "::" and a trailing dotted IPv4.
+function ipv6Bytes(ip) {
+  let s = ip;
+  const v4 = /(\d+\.\d+\.\d+\.\d+)$/.exec(s);
+  if (v4) {
+    const o = v4[1].split('.').map(Number);
+    s = s.slice(0, -v4[1].length) + ((o[0] << 8) | o[1]).toString(16) + ':' + ((o[2] << 8) | o[3]).toString(16);
+  }
+  const halves = s.split('::');
+  if (halves.length > 2) return null;
+  const head = halves[0] ? halves[0].split(':') : [];
+  const tail = halves.length === 2 && halves[1] ? halves[1].split(':') : [];
+  const missing = 8 - head.length - tail.length;
+  if (halves.length === 2 ? missing < 1 : missing !== 0) return null;
+  const groups = [...head, ...Array(halves.length === 2 ? missing : 0).fill('0'), ...tail];
+  if (groups.length !== 8 || !groups.every((g) => /^[0-9a-f]{1,4}$/i.test(g))) return null;
+  return groups.flatMap((g) => [parseInt(g, 16) >> 8, parseInt(g, 16) & 255]);
+}
+
+// 'public', 'loopback', or 'refused' for an IPv4 or IPv6 address (no brackets)
+export function classifyAddress(address) {
+  let a = String(address).replace(/^\[|\]$/g, '');
+  let o;
+  if (net.isIPv4(a)) {
+    o = a.split('.').map(Number);
+  } else if (net.isIPv6(a)) {
+    const b = ipv6Bytes(a.split('%')[0]);
+    if (!b) return 'refused';
+    const mapped = b.slice(0, 10).every((x) => x === 0) && b[10] === 255 && b[11] === 255; // ::ffff:a.b.c.d
+    if (mapped) { o = b.slice(12); }
+    else {
+      if (b.every((x) => x === 0)) return 'refused';                                       // ::
+      if (b.slice(0, 15).every((x) => x === 0) && b[15] === 1) return 'loopback';          // ::1
+      if ((b[0] & 0xfe) === 0xfc) return 'refused';                                        // fc00::/7 unique local
+      if (b[0] === 0xfe && (b[1] & 0xc0) === 0x80) return 'refused';                       // fe80::/10 link-local
+      if (b[0] === 0xff) return 'refused';                                                 // multicast
+      if (b[0] === 0 && b[1] === 0x64 && b[2] === 0xff && b[3] === 0x9b) return 'refused'; // 64:ff9b::/96 NAT64
+      if (b[0] === 0x20 && b[1] === 0x01 && b[2] === 0x0d && b[3] === 0xb8) return 'refused'; // documentation
+      return 'public';
+    }
+  } else {
+    return 'refused'; // not an address at all
+  }
+  const [a0, a1] = o;
+  if (a0 === 127) return 'loopback';
+  if (a0 === 0 || a0 === 10) return 'refused';
+  if (a0 === 100 && a1 >= 64 && a1 <= 127) return 'refused';              // 100.64/10 shared
+  if (a0 === 169 && a1 === 254) return 'refused';                         // link-local, cloud metadata
+  if (a0 === 172 && a1 >= 16 && a1 <= 31) return 'refused';
+  if (a0 === 192 && a1 === 168) return 'refused';
+  if (a0 === 192 && a1 === 0 && o[2] === 0) return 'refused';            // 192.0.0.0/24
+  if (a0 === 198 && (a1 === 18 || a1 === 19)) return 'refused';          // benchmarking
+  if (a0 >= 224) return 'refused';                                        // multicast and reserved
+  return 'public';
+}
+
+// The host of a URL as it should be judged: lower case, no brackets, no trailing dot
+export function hostOf(urlString) {
+  try { return new URL(urlString).hostname.replace(/^\[|\]$/g, '').replace(/\.+$/, '').toLowerCase(); }
+  catch { return ''; }
+}
+
+// Is this URL's host an address on this machine? "localhost" or a loopback address, however written.
+export function isLoopbackUrl(urlString) {
+  const h = hostOf(urlString);
+  return h === 'localhost' || (net.isIP(h) !== 0 && classifyAddress(h) === 'loopback');
+}
+
+// null if the URL may be connected to, otherwise why not.
+//   loopbackPorts   ports on this machine that may be used, e.g. [9200]. Empty: nothing on this machine.
+//   lookup          dns.promises.lookup, replaceable for tests
+export async function checkDestination(urlString, { loopbackPorts = [], lookup = dns.promises.lookup } = {}) {
+  let u;
+  try { u = new URL(urlString); } catch { return 'is not a valid URL'; }
+  const host = hostOf(urlString);
+  if (!host) return 'has no host';
+  if (METADATA_NAMES.has(host)) return 'points somewhere that is not allowed';
+  const port = Number(u.port || (u.protocol === 'https:' ? 443 : 80));
+  let addresses;
+  if (net.isIP(host)) { addresses = [host]; }
+  else {
+    try { addresses = (await lookup(host, { all: true })).map((r) => r.address); }
+    catch { return 'is a host that could not be found'; }
+    if (!addresses.length) return 'is a host that could not be found';
+  }
+  for (const address of addresses) {
+    const kind = classifyAddress(address);
+    if (kind === 'refused') return 'points somewhere that is not allowed';
+    if (kind === 'loopback' && !loopbackPorts.includes(port)) {
+      return loopbackPorts.length
+        ? `may only use this machine on port ${loopbackPorts.join(' or ')}`
+        : 'may not point at this machine';
+    }
+  }
+  return null;
+}