frugal-iot-server 0.3.9 → 2.0.0

package/frugal-iot-server.js

@@ -21,8 +21,8 @@
  O /config.json Return configuration info - depends on user's org
  O /data  Back files from logger for graphing
  A /dashboard serves dashboard via frugal-iot-client
- * /debug repurposed for development
- * /echo  Send back headers etc
+ X /debug repurposed for development - commented out, see the note by the route
+ X /echo  Send back headers etc - commented out, see the note by the route
  * /login (get) served under default handler - which might go away TODO-N89 make sure not hidden under dashboards Authentication
  * /login (post) login a user, & redirect (to dashboard typically)
  * /node_modules  Javascript libraries (from frugal-iot-client)
@@ -107,6 +107,13 @@ import { MqttLogger } from "frugal-iot-logger";  // https://github.com/mitra42/f
 
 // API Integration - Farm IoT Interoperability Standard
 import { createAPIRouter, createAPIErrorHandler } from './lib/api-routes.js';
+import { buildConfigFor, hasPermissions } from './lib/config-for-user.js';
+import { ensureSecrets, addEnrolmentSecret, removeEnrolmentSecret } from './lib/secrets.js';
+import { syncUser, syncUserById, syncLoggers, syncNode, dropNode } from './lib/dynsec-server.js';
+import { enrol, ENROL, enrolmentSecretsFor, makeRateLimiter, forgetNode,
+         makeAttemptLog, setGrant, clearGrant, readGrants } from './lib/enrol.js';
+import { deleteRetained } from './lib/retained.js';
+import { replicaFor, bridgeForToken, noteBridgePull, startReplica } from './lib/replica.js';
 import { createLoggerClient } from './lib/logger-client.js';
 import { createPushManager } from './lib/farm-platform-push.js';
 import { APIError } from './lib/api-errors.js';
@@ -173,14 +180,19 @@ const responseHeaders = {
 };
 // ============ Helper functions ============
 // Note attribs is the otakey e.g. sht30_d1_mini
+// The four parameters come from the URL of a route with no login on it (devices call it), so they
+// are as untrusted as anything gets here. safeJoin drops any candidate that would leave otadir -
+// before this, `..` in a parameter could read a file called firmware.bin from anywhere the process
+// could reach. A dropped candidate is simply not looked for, which reads the same to the caller as
+// a file that is not there.
 function findMostSpecificFile(topdir, org, project, node, attribs, cb) {
   let possfiles = [
-    `${project}/${node}`, // Unlikely - if specify node, should be at the org level
-    `+/${node}`,
-    `${project}/${attribs}`,
-    `+/${attribs}`
+    [project, node], // Unlikely - if specify node, should be at the org level
+    ['+', node],
+    [project, attribs],
+    ['+', attribs]
     //TODO-C14 might want to accept other variants on Arduino like sht30.ini.bin
-    ].map(x => `${topdir}/${org}/${x}/firmware.bin`);
+    ].map(x => safeJoin(topdir, org, ...x, 'firmware.bin')).filter((x) => x);
   detectSeries(possfiles, (path, cb1) => {
       access(path, constants.R_OK, (err) => { cb1(null, !err); })},
     cb);
@@ -217,11 +229,20 @@ function startServer() {
   });
 }
 function isUnsafe(arr) {
-  return arr.some( x => x && x.includes("/"))
+  return arr.some( x => x && (x.includes("/") || x === ".." ))
 }
-// Dont let client supplied filepath go up directorey tree
-function sanitize(filepath) {
-  return filepath.replace(/[.][.]\//g, '/');
+// Build a path under topdir out of untrusted components, or return null if the result would not be
+// inside topdir. Resolve first and then ask where the answer landed: the reverse - looking for ".."
+// in the components and removing it - is what the two functions this replaced did, and both were
+// defeatable. A single-pass replace of "../" turns "....//" back into "../" rather than removing it,
+// and Express has already decoded "%2e%2e" and "..%2f" by the time a route sees them, so there is
+// nothing distinctive left to search for. Where the resolved path is cannot be argued with.
+// Callers must treat null as "refuse the request" - it is not a path.
+function safeJoin(topdir, ...parts) {
+  if (parts.some((x) => (x === undefined) || (x === null) || (x === ''))) return null;
+  const root = path.resolve(topdir);
+  const wanted = path.resolve(root, path.join(...parts.map(String)));
+  return ((wanted === root) || wanted.startsWith(root + path.sep)) ? wanted : null;
 }
 
 function adminUrl(req, message, lang) {
@@ -243,7 +264,7 @@ function clientErrorHandler(err, req, res, next) {
   }
 }
 const sqlPeoplePermList = `
-  SELECT u.id, u.name, p.capability
+  SELECT u.id, u.name, p.capability, p.project
   FROM users u
   INNER JOIN permissions p ON u.id = p.id AND p.org = ?
 ;`;
@@ -252,7 +273,7 @@ const sqlPeopleList = `
     FROM users u
 ;`;
 const sqlAddPermission = `
-  INSERT INTO permissions (id, capability, org) VALUES (?, ?, ?)
+  INSERT INTO permissions (id, capability, org, project) VALUES (?, ?, ?, ?)
 ;`;
 function get_people_list(org, cb) {
   db.all(sqlPeoplePermList, [org], (err, rows1) => {
@@ -268,6 +289,66 @@ function get_people_list(org, cb) {
       })
     }});
 }
+/*
+ * Nodes that asked to enrol and were refused. Module level because two things need it: POST /enrol
+ * writes to it and GET /nodes_list reads it, and they are in different scopes.
+ *
+ * In memory on purpose - see makeAttemptLog. Losing it on restart costs nothing, because a node
+ * that is still retrying reappears within minutes.
+ */
+const enrolAttempts = makeAttemptLog();
+
+// Which nodes have enrolled in an organization. No password, obviously - the point of the list is
+// to know what exists and to be able to forget one.
+const sqlNodesList = `
+  SELECT project, nodeid, lora, enrolled_at FROM nodes WHERE org = ? ORDER BY project, nodeid
+;`;
+/*
+ * Every node the organization knows anything about, with one state each (SECURITY.md S12):
+ *
+ *   enrolled  it has a credential
+ *   approved  an admin has said its next request may be accepted
+ *   denied    an admin has stopped it
+ *   failed    it asked and was refused - which is how an admin learns it exists at all
+ *
+ * Three sources, because they answer different questions: the nodes table is what exists, node_grants
+ * is what an admin decided, and the in-memory attempt log is who is asking now. A node that has never
+ * connected appears only in the third, which is exactly the node needing attention.
+ */
+function send_nodes_list(req, res) {
+  const org = req.params.org;
+  db.all(sqlNodesList, [org], (err, enrolled) => {
+    if (err) { return res.status(500).send(err.message); }
+    readGrants(db, org, (gerr, grants) => {
+      if (gerr) { return res.status(500).send(gerr.message); }
+      const byId = new Map();
+      for (const n of enrolled) {
+        byId.set(n.nodeid, { ...n, state: 'enrolled' });
+      }
+      for (const g of grants) {
+        const row = byId.get(g.nodeid) || { project: g.project, nodeid: g.nodeid, lora: 0, enrolled_at: null };
+        // A decision outranks "enrolled": a denied node still has a nodes row until it is forgotten
+        byId.set(g.nodeid, { ...row, state: g.state, decided_by: g.created_by, decided_at: g.created_at });
+      }
+      for (const a of enrolAttempts.forOrg(org)) {
+        if (byId.has(a.nodeid)) {
+          // Still asking despite being enrolled or decided - worth showing, not worth reclassifying
+          Object.assign(byId.get(a.nodeid),
+            { asked_at: a.at, asked_from: a.from, asked_reason: a.reason, asked_count: a.count });
+          continue;
+        }
+        byId.set(a.nodeid, {
+          project: a.project, nodeid: a.nodeid, lora: 0, enrolled_at: null, state: 'failed',
+          asked_at: a.at, asked_from: a.from, asked_reason: a.reason, asked_count: a.count,
+        });
+      }
+      const rows = [...byId.values()].sort((x, y) =>
+        (x.project || '').localeCompare(y.project || '') || x.nodeid.localeCompare(y.nodeid));
+      res.status(200).json(rows);
+    });
+  });
+}
+
 function send_people_list(req, res) {
   // TODO-N89 list all People for an organization
   get_people_list(req.params.org, (err, people) => {
@@ -286,7 +367,10 @@ function notePermissionsChanged(id) {
   permissionsChangedAt.set(Number(id), Date.now());
 }
 
-function add_permission(id, capability, org, cb) {
+// project is optional: '' means the whole organization, which is what a caller that knows nothing
+// about projects sends - so the admin UI keeps working unchanged until it grows a project field.
+function add_permission(id, capability, org, project, cb) {
+  project = project || '';
   if ((id === undefined)
     || (capability === undefined) || (capability.length < 2)
     || (org === undefined) || (org.length < 2)) {
@@ -295,22 +379,28 @@ function add_permission(id, capability, org, cb) {
     waterfall([
       (cb) => db.get('SELECT COUNT(id) FROM users WHERE id = ?', [id], cb),
       (n_users, cb) => { if (n_users["COUNT(id)"] != 1) { cb(new Error("User not found")); } else { cb(null); }},
-      (cb) => db.get('SELECT COUNT(id) FROM permissions WHERE id = ? AND capability = ? AND org = ?', [id, capability, org], cb),
+      (cb) => db.get('SELECT COUNT(id) FROM permissions WHERE id = ? AND capability = ? AND org = ? AND project = ?', [id, capability, org, project], cb),
       (n_perms, cb) => { if (n_perms["COUNT(id)"] != 0) { cb(new Error("Duplicate permission")); } else { cb(null); }},
-      (cb) => db.run(sqlAddPermission, [id, capability, org], cb),
+      (cb) => db.run(sqlAddPermission, [id, capability, org, project], cb),
       (cb) => { notePermissionsChanged(id); cb(null); },
+      // The database has changed, so the broker's idea of this user is now stale. Best effort.
+      (cb) => { syncUserById(db, config, id, () => cb(null)); },
     ], cb);
   }
 }
-function permissions_delete(id, capability, org, cb) {
+function permissions_delete(id, capability, org, project, cb) {
+  project = project || '';
   if ((id === undefined)
     || (capability === undefined) || (capability.length < 2)
     || (org === undefined) || (org.length < 2)) {
     cb(new Error("Invalid parameters"));
   } else {
     waterfall([
-      (cb) => db.get('DELETE FROM permissions WHERE id = ? AND capability = ? AND org = ?', [id,capability,org], cb),
+      (cb) => db.get('DELETE FROM permissions WHERE id = ? AND capability = ? AND org = ? AND project = ?', [id,capability,org,project], cb),
       (cb) => { notePermissionsChanged(id); cb(null); },
+      // Revoking has to reach the broker, or the credential already in a browser keeps working -
+      // which is the whole of SEC-1's second requirement.
+      (cb) => { syncUserById(db, config, id, () => cb(null)); },
     ], cb);
   }
 }
@@ -507,15 +597,24 @@ passport.use(new LocalStrategy(function verify(username, password, cb) {
         return cb(null, false, { message: 'Incorrect username or password*' });
       }
       db.all('SELECT * FROM permissions WHERE id = ? or id = 0', [ user.id ], function(err, permissions) {
-        // permissions is [{ id, capability, org }]
-        console.log("User",user.id, "with permissions", permissions.map(x => x.capability + " " +x.org).join(","));
+        // permissions is [{ id, capability, org, project }]
+        console.log("User",user.id, "with permissions", permissions.map(x => x.capability + " " +x.org + (x.project ? "/" + x.project : "")).join(","));
         if (err) {
           return cb(err);
         }
-        return cb(null, {
-          id: user.id, username: user.username, organization: user.organization,
-          name: user.name, email: user.email, phone: user.phone, permissions
-        }); // TO-ADD-REGISTRATION-FIELD
+        // Bring this user's broker account into line with what the database says, and get the
+        // credential their browser will use. Deliberately not fatal: if the broker cannot be
+        // reached the login still succeeds and the dashboard loses live data, which is far better
+        // than not being able to log in at all. The credential is derived, so it is returned either
+        // way and a later frugal-iot-rebuild-dynsec makes the broker agree.
+        syncUser(db, config, user, (serr, cred) => {
+          if (serr && !cred) console.log("No broker credential for", user.username, "-", serr.message);
+          return cb(null, {
+            id: user.id, username: user.username, organization: user.organization,
+            name: user.name, email: user.email, phone: user.phone, permissions,
+            mqtt_username: cred && cred.username, mqtt_password: cred && cred.password,
+          }); // TO-ADD-REGISTRATION-FIELD
+        });
       });
     });
   });
@@ -532,9 +631,9 @@ function loggedInOrRedirect(req, res, next) {
     res.redirect(307, `${loginUrl}?${q}`);
   }
 }
-function hasPermissions(user, org, permission) {
-  return user.permissions.some(x => x.capability == permission && x.org == org);
-}
+// hasPermissions now lives in lib/config-for-user.js and is re-exported here, because the config
+// filtering needs it and that had to be testable without booting a server.
+export { hasPermissions };
 // Like hasPermissions, but not org-scoped - true if the user has the capability on ANY org.
 function hasPermissionsAny(user, permission) {
   return user.permissions.some(x => x.capability == permission);
@@ -557,6 +656,20 @@ export function can_READ(req, res, next) {
     res.sendStatus(401);
   }
 }
+// Exported for reuse by lib/api-routes.js - reads org from req.params.org or res.locals.org, so callers must set one of those first.
+// Note WRITE is not implied by READ, or by ADMIN: an administrator who should also be able to change
+// a device needs a WRITE row of their own. Anything that changes a device must use this and not
+// can_READ - see the API's /devices/action, where using can_READ meant any reader could command any
+// device.
+export function can_WRITE(req, res, next) {
+  const org = req.params.org || res.locals.org;
+  if (req.isAuthenticated() && hasPermissions(req.user, org, "WRITE")) {
+    next();
+  } else {
+    console.log("Failing permission to Write", req.user, org);
+    res.sendStatus(401);
+  }
+}
 // Not used as check direct in Multer storage, (since Multer fills the body) but use as template for other permissions (and then delete this comment)
 // Exported for reuse by lib/api-routes.js - reads org from req.params.org, so callers without an :org URL segment must set it first.
 export function can_ADMIN(req, res, next) {
@@ -642,25 +755,9 @@ function addLoggedNodesToConfig() {
     });
   });
 }
-// Produce an "unsafe" copy of config, i.e. it is a subset of config but points to objects rather than copying. Don't change the result!
+// See lib/config-for-user.js - kept there so it can be tested without starting a server.
 function unsafeCopyConfigFor(user) {
-  let oo = {
-    organizations: {},
-    user: user, // All data in user and permissions is visible to the user
-  };
-  Object.entries(config).forEach(([key, value]) => {
-    if (key === 'organizations') {
-      // noinspection JSCheckFunctionSignatures
-      Object.entries(value).forEach(([orgid, org]) => {
-        if (hasPermissions(user, orgid, 'READ')) {
-          oo.organizations[orgid] = org;
-        }
-      });
-    } else {
-      oo[key] = value;
-    }
-  });
-  return oo;
+  return buildConfigFor(config, user);
 }
 // ============ End Helper functions ============
 
@@ -692,13 +789,18 @@ app.options('/', (req, res) => {
 
 // Start the recognition of specific URL paths
 
-app.get('/echo', (req, res) => {
-  res.status(200).json(req.headers);
-});
+// Both of these are commented out rather than deleted: each was added to debug one specific thing
+// and neither is needed in normal running. They sit here, above the session middleware, so nothing
+// authenticates them - /debug in particular listed every node, project and topic the logger knows
+// about to anyone who asked. Uncomment whichever is wanted while debugging, and comment it out
+// again afterwards.
+//app.get('/echo', (req, res) => {
+//  res.status(200).json(req.headers);
+//});
 // This /debug can be freely rewritten to help debug stuff, nothing should rely on what it does remaining constant
-app.get('/debug', (req, res) => {
-  res.status(200).json(mqttLogger.reportNodes());
-});
+//app.get('/debug', (req, res) => {
+//  res.status(200).json(mqttLogger.reportNodes());
+//});
 // Stick this as middleware to debug
 // noinspection JSUnusedLocalSymbols
 function debugRoutes(req, res, next) {
@@ -763,6 +865,73 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
       next();
     })
 
+    // Nodes enrol here to get their own broker credential. Deliberately not behind a session: a node
+    // has no session, and this is what replaces the shared password compiled into its firmware.
+    // Authenticated by the organization's enrolment secret, which grants only enrolment.
+    //
+    // Body is JSON, so express.json() has to have run - it is added below for /api, and this route
+    // brings its own parser rather than depending on the order of the two.
+    const enrolLimiter = makeRateLimiter();
+    app.post('/enrol', express.json({ limit: '4kb' }), (req, res) => {
+      enrol({ db, config, limiter: enrolLimiter,
+              syncNode: (node, cb) => syncNode(config, node, cb),
+              // So a refusal reaches the dashboard's Nodes card, which is the only way an admin
+              // learns that a node exists and is asking (SECURITY.md S12).
+              attempts: enrolAttempts, from: req.ip },
+        req.body, (err, cred) => {
+          if (!err) {
+            console.log("Enrolled", cred.username, req.body.lora ? "(LoRa-capable)" : "");
+            return res.status(200).json(cred);
+          }
+          const status = {
+            [ENROL.BAD_REQUEST]: 400,
+            [ENROL.REFUSED]: 403,
+            [ENROL.RATE_LIMITED]: 429,
+            [ENROL.NEEDS_RESET]: 409,
+            [ENROL.NO_PROJECT]: 404,
+            [ENROL.DENIED]: 403,
+            [ENROL.BROKER]: 503,
+          }[err.code] || 500;
+          // Every attempt is logged, successful or not: anyone holding an enrolment secret can add a
+          // fictitious node, which is a far smaller privilege than the password it replaces but
+          // should not be invisible.
+          console.log(`Enrolment refused (${err.code || 'error'}) for`,
+            `${req.body && req.body.org}/${req.body && req.body.project}/${req.body && req.body.nodeid}:`,
+            err.message);
+          res.status(status).json({ error: err.code || 'error', message: err.message });
+        });
+    });
+
+    /*
+     * What a bridged Pi may know about this server's people (SECURITY.md S11).
+     *
+     * Authenticated by a bearer token, one per Pi per organization, issued by
+     * frugal-iot-addbridge-prod - not by a session, because there is no person here: the Pi asks
+     * for itself, on a timer.
+     *
+     * Read-only, and scoped to the organization the token was issued for. What it returns is
+     * logins, the stored password hashes and the permission rows - nothing derived from a password,
+     * and no secret of this server's: the Pi authenticates logins itself and derives its own broker
+     * credentials from its own user_secret.
+     */
+    app.get('/replica/:org', (req, res) => {
+      const auth = req.get('Authorization') || '';
+      const token = auth.startsWith('Bearer ') ? auth.slice(7) : '';
+      bridgeForToken(db, token, (err, bridge) => {
+        if (err) { return res.status(500).json({ error: err.message }); }
+        // Same answer for an unknown token and for a token belonging to another organization: a
+        // bridge should not be able to discover which organizations exist here.
+        if (!bridge || bridge.org !== req.params.org) {
+          return res.status(403).json({ error: 'Not a bridge for this organization' });
+        }
+        replicaFor(db, bridge.org, (rerr, payload) => {
+          if (rerr) { return res.status(500).json({ error: rerr.message }); }
+          noteBridgePull(db, bridge.org, bridge.site);
+          res.status(200).json(payload);
+        });
+      });
+    });
+
     console.log("Doing OTA updates at /ota_update from", config.server.otadir);
     app.get('/ota_update/:org/:project/:node/:attribs', (req, res) => {
       //Intentionally no login
@@ -815,9 +984,34 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
     const routerNM = express.Router();
     app.use('/node_modules', routerNM);
     //routerData.use('/', (req, res, next) => { console.log("NM:", req.url); next(); });
+    /*
+     * Revalidated, not cached for a day.
+     *
+     * These URLs are not versioned - "/node_modules/mqtt/dist/mqtt.esm.js" serves whatever this
+     * release installed - so "immutable" was a promise the server could not keep. It means "never
+     * even ask again", so a browser went on running the previous release's code for up to 24 hours
+     * after an upgrade, with no way to know it was doing so. That is also why deploying to the test
+     * Pi appeared to do nothing.
+     *
+     * Still a day - these users are often on an expensive link with poor reception, so a request
+     * saved is worth more than a minute's freshness. What changes is only `immutable`, and the
+     * difference between the two is exactly the one that matters here: a RELOAD revalidates a
+     * max-age resource and does not revalidate an immutable one. So ordinary navigation still costs
+     * no requests, while a refresh picks up a new release instead of being told not to ask for a
+     * day. ETag and Last-Modified, which express.static already sends, make that a 304.
+     *
+     * Freshness for an installed PWA is the service worker's job, not this header's: it is
+     * cache-first, its CACHE_NAME follows the release, and it fetches with cache 'reload' on
+     * install, which bypasses this cache entirely.
+     *
+     * The complete answer is a version in the URL, which would allow a year here AND appear the
+     * instant a release changed it. That needs the client's module imports to carry the version, so
+     * it is not a one-line change - noted, not done.
+     */
+    const revalidate = { maxAge: 1000 * 60 * 60 * 24 };
     routerNM.use(
-      express.static(clientNodeModules, {immutable: true, maxAge: 1000 * 60 * 60 * 24}),
-      express.static(config.server.nodemodulesdir, {immutable: true, maxAge: 1000 * 60 * 60 * 24})
+      express.static(clientNodeModules, revalidate),
+      express.static(config.server.nodemodulesdir, revalidate)
     );
 
     openOrCreateDatabase((err, db) => {
@@ -834,8 +1028,36 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
         app.set('trust proxy', 1); // trust first proxy - see note in https://www.npmjs.com/package/express-session
         // TODO-N89 note need to setup session store, defaults to memory store which is not good for production
         // TODO-N89 think about cookie timeout and add "keep me logged in on this device" option that controls it
+        // The session secret signs the cookie, so a known one would let anyone mint a validly-signed
+        // one. That alone is not a way in - the cookie carries only a session id and the session
+        // lives server-side, so a forged one names no session - but it was the express-session
+        // README's own 'keyboard cat', which is not a state to leave a server in.
+        //
+        // frugal-iot-init writes these, but a server upgraded from a release before that has none,
+        // so anything missing is generated AND written to config.d/secrets.yaml here. Generating
+        // without saving would end every session on every restart, and the symptom does not point
+        // at the cause, so it would be lived with rather than fixed.
+        // One enrolment secret per organization, as a list so that rotating one does not strand
+        // nodes already flashed with the old value. Generated here if absent, so adding an
+        // organization needs no extra step and an existing installation fixes itself.
+        const enrolNames = Object.keys(config.organizations || {})
+          .filter((org) => !enrolmentSecretsFor(config, org).length)
+          .map((org) => `enrolment_${org}`);
+        const secretsResult = ensureSecrets(config.secrets, './config.d',
+          ['session_secret', 'user_secret', ...enrolNames]);
+        config.secrets = secretsResult.secrets;   // so the rest of this run sees them
+        if (secretsResult.generated.length) {
+          if (secretsResult.written) {
+            console.log("Generated", secretsResult.generated.join(", "),
+              "and saved them to config.d/secrets.yaml");
+          } else {
+            console.error("Could not write config.d/secrets.yaml:", secretsResult.error);
+            console.error("  Generated", secretsResult.generated.join(", "), "for this run only, so",
+              "every restart will log everyone out until that file is writable.");
+          }
+        }
         app.use(session({
-          secret: 'keyboard cat', // TODO-N89 probably change, try changing this, hopefully should just require re-login
+          secret: config.secrets.session_secret,
           resave: false,
           saveUninitialized: false,
           cookie: { secure: 'auto' }  // TODO-N89 cant be secure: true while testing on HTTP
@@ -853,6 +1075,12 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
               email: user.email,
               phone: user.phone,
               permissions: user.permissions,
+              // The broker credential this user's browser will use, derived at login by
+              // lib/dynsec-server.js. Kept in the session because that is where the rest of the
+              // login's results live, and sessions are in memory - so it dies with the process and
+              // is derived again at the next login. /config.json serves it from here.
+              mqtt_username: user.mqtt_username,
+              mqtt_password: user.mqtt_password,
               loginAt: Date.now(), // So a permission change can tell which sessions predate it
             });
           });
@@ -1057,8 +1285,15 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
                       return loginRedirect(res, notValid);
                     }
                     console.log("Password reset for", user.username);
-                    loginRedirect(res, { mode: 'signin', messagetype: 'info', url,
-                      message: 'Password reset - please sign in' });
+                    // The broker credential is derived from the stored hash, so changing the
+                    // password changes it - which is a useful property (the old one dies by
+                    // itself) but only if the broker is told. They have to sign in again anyway,
+                    // and that would sync it too; doing it here means the old credential stops
+                    // working now rather than whenever they next appear.
+                    syncUserById(db, config, user.id, () => {
+                      loginRedirect(res, { mode: 'signin', messagetype: 'info', url,
+                        message: 'Password reset - please sign in' });
+                    });
                   });
               });
             });
@@ -1093,8 +1328,11 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
           loggedInOrFail,
           can_OTAUPDATE,
           (req,res) => {
-            let remainingpath = req.params.remainingpath.join('/');
-            let dirpath = `${config.server.otadir}/${req.params.org}/${sanitize(remainingpath)}`;
+            let dirpath = safeJoin(config.server.otadir, req.params.org, ...[].concat(req.params.remainingpath));
+            if (!dirpath) {
+              res.status(400).send("Bad path");
+              return;
+            }
             console.log("Deleting OTA file", dirpath);
             rm(dirpath, { recursive: true, force: true }, (err, unused) => {
               if (err) {
@@ -1119,8 +1357,11 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
           loggedInOrFail,
           can_OTAUPDATE,
           (req,res) => {
-            let remainingpath = req.params.remainingpath.join('/');
-            let filepath = `${config.server.otadir}/${req.params.org}/${sanitize(remainingpath)}/firmware.bin`;
+            let filepath = safeJoin(config.server.otadir, req.params.org, ...[].concat(req.params.remainingpath), 'firmware.bin');
+            if (!filepath) {
+              res.status(400).send("Bad path");
+              return;
+            }
             console.log("Sending OTA file", filepath);
             res.sendFile(filepath, {}, (err) => {
               if (err) {
@@ -1139,7 +1380,7 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
           loggedInOrFail,
           can_ADMIN,  // Gets org from URL
           (req,res, next) => {
-            add_permission(req.query.id, req.query.capability,req.params.org, (err) => {
+            add_permission(req.query.id, req.query.capability, req.params.org, req.query.project, (err) => {
               if (err) {
                 res.status(400).send(err.message);
               } else {
@@ -1153,7 +1394,7 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
           loggedInOrFail,
           can_ADMIN,  // Gets org from URL
           (req,res, next) => {
-            permissions_delete(req.query.id, req.query.capability,req.params.org, (err) => {
+            permissions_delete(req.query.id, req.query.capability, req.params.org, req.query.project, (err) => {
               if (err) {
                 res.status(400).send(err.message);
               } else {
@@ -1163,6 +1404,195 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
           },
           send_people_list,
         );
+        app.get('/nodes_list/:org',
+          loggedInOrFail,
+          can_ADMIN,  // Gets org from URL
+          send_nodes_list,
+        );
+        /*
+         * Forget a node, so it can enrol again and be issued a new credential.
+         *
+         * The same job as frugal-iot-resetnode, for somebody who administers an organization but has
+         * no shell on the server - which is most people who will need it. A node whose filesystem
+         * has been erased cannot prove it is itself, and this is what says "yes, it really is".
+         *
+         * POST, not GET, unlike the older admin routes beside it (/add_permission,
+         * /permissions_delete). Those are GETs that change things, which means a link or an image
+         * tag can trigger them against anybody with a live admin session - see SEC-17. Rather than
+         * add another, this one is a POST; the older ones want the same treatment together.
+         */
+        /*
+         * Delete retained messages, on behalf of an organization's admin.
+         *
+         * The browser used to publish the empty payloads itself. It cannot any more - since S4 its
+         * broker credential may publish to "set/" topics only - and it failed silently, because a
+         * QoS 1 publish is acknowledged whether or not the broker will act on it. See
+         * lib/retained.js for why widening the browser's rights is the wrong repair.
+         */
+        /*
+         * The organization's enrolment secrets (SECURITY.md S10).
+         *
+         * An admin has to compile one into firmware, and until this existed only shell access could
+         * read it. Its own route, never /config.json: that is served to every logged-in user, and
+         * lib/config-for-user.js withholds the whole secrets section for exactly this reason.
+         *
+         * A list, not one value. Rotating means adding a new secret while the old is still
+         * accepted, or every node already flashed with the old value and not yet enrolled is
+         * stranded. So: add, reflash at leisure, then delete.
+         */
+        app.get('/enrolment_secret/:org',
+          loggedInOrFail,
+          can_ADMIN,  // Gets org from URL - an admin sees only their own organization's
+          (req, res) => {
+            const org = req.params.org;
+            // Worth a line: this is the one route that hands a secret to a browser.
+            console.log("Enrolment secrets read by", req.user.username, "for", org);
+            res.status(200).json({
+              org,
+              secrets: enrolmentSecretsFor(config, org),
+              enrol_url: `${req.protocol}://${req.get('host')}/enrol`,
+            });
+          },
+        );
+        app.post('/enrolment_secret/:org',
+          loggedInOrFail,
+          can_ADMIN,
+          (req, res) => {
+            const org = req.params.org;
+            addEnrolmentSecret('./config.d', org, enrolmentSecretsFor(config, org), (err, result) => {
+              // The in-memory copy either way, so POST /enrol accepts the new secret at once; a
+              // failed write means it works until the next restart, which is worth saying.
+              config.secrets[`enrolment_${org}`] = result.list;
+              console.log("Enrolment secret added by", req.user.username, "for", org,
+                          err ? `(NOT saved: ${err.message})` : '');
+              res.status(200).json({
+                org, secrets: result.list, saved: !err,
+                message: err
+                  ? `Added, but config.d/secrets.yaml could not be written (${err.message}) - it will be gone after a restart.`
+                  : 'Added. Existing secrets still work, so nodes already flashed are unaffected.',
+              });
+            });
+          },
+        );
+        app.delete('/enrolment_secret/:org',
+          loggedInOrFail,
+          can_ADMIN,
+          express.json({ limit: '4kb' }),
+          (req, res) => {
+            const org = req.params.org;
+            const secret = (req.body && req.body.secret) || '';
+            if (!secret) { return res.status(400).json({ error: 'secret is required' }); }
+            removeEnrolmentSecret('./config.d', org, secret, enrolmentSecretsFor(config, org),
+              (err, result) => {
+                if (!result.removed) {
+                  return res.status(404).json({ error: 'That is not one of this organization\'s secrets' });
+                }
+                config.secrets[`enrolment_${org}`] = result.list;
+                console.log("Enrolment secret withdrawn by", req.user.username, "for", org,
+                            err ? `(NOT saved: ${err.message})` : '');
+                res.status(200).json({
+                  org, secrets: result.list, saved: !err,
+                  message: result.list.length
+                    ? 'Withdrawn. A node flashed with it can no longer enrol.'
+                    : 'Withdrawn. No secrets left, so no new node can enrol until one is added or the server restarts.',
+                });
+              });
+          },
+        );
+        app.post('/retained_delete/:org',
+          loggedInOrFail,
+          can_ADMIN,  // Gets org from URL - and deleteRetained checks every topic is inside it
+          express.json({ limit: '512kb' }),   // a few thousand topic names
+          (req, res) => {
+            const topics = (req.body && req.body.topics) || [];
+            deleteRetained(config, req.params.org, topics, (err, result) => {
+              if (err) { return res.status(err.status || 500).json({ error: err.message }); }
+              console.log("Retained messages deleted by", req.user.username, "-",
+                          req.params.org, result.deleted, "topic(s)");
+              res.status(200).json(result);
+            });
+          },
+        );
+        /*
+         * An admin's decision about one node (SECURITY.md S12): approved, denied, or cleared.
+         *
+         * `denied` is the kill switch the system otherwise lacks - for a node publishing bad
+         * readings. It works by taking the credential away at the BROKER, not by anything on the
+         * node: the node still holds its copy, is refused, discards it after five refusals, asks to
+         * enrol, and is refused again by the stored decision. Its own nodes row is forgotten too,
+         * so that clearing the denial later lets it enrol normally instead of being asked to prove
+         * a password it no longer has.
+         */
+        app.post('/node_state/:org',
+          loggedInOrFail,
+          can_ADMIN,  // Gets org from URL
+          express.json({ limit: '4kb' }),
+          (req, res) => {
+            const org = req.params.org;
+            const { project, nodeid, state } = req.body || {};
+            if (!nodeid || !state) {
+              return res.status(400).json({ error: 'nodeid and state are required' });
+            }
+            if (!['approved', 'denied', 'cleared'].includes(state)) {
+              return res.status(400).json({ error: 'state must be approved, denied or cleared' });
+            }
+            const finish = (message) => {
+              console.log("Node", state, "by", req.user.username, "-", org, project || '?', nodeid);
+              res.status(200).json({ org, project, nodeid, state, message });
+            };
+            if (state === 'cleared') {
+              return clearGrant(db, org, nodeid, (err) => {
+                if (err) { return res.status(500).json({ error: err.message }); }
+                finish('Cleared. It may enrol again with a valid enrolment secret.');
+              });
+            }
+            setGrant(db, { org, project, nodeid, state, by: req.user.username }, (err) => {
+              if (err) { return res.status(500).json({ error: err.message }); }
+              if (state === 'approved') {
+                return finish('Approved. Its next request is accepted, whatever secret it presents. ' +
+                              'It may take a few minutes to ask again.');
+              }
+              // Denied: take the credential away, and forget the node so that clearing this later
+              // does not leave it unable to prove a password it has already discarded.
+              forgetNode(db, org, project || '', nodeid, () => {
+                dropNode(config, org, project || '', nodeid, (derr) => {
+                  finish(derr
+                    ? `Denied. The broker account could not be removed (${derr.message}) - run frugal-iot-rebuild-dynsec.`
+                    : 'Denied. Its broker account is gone and it cannot enrol again until this is cleared.');
+                });
+              });
+            });
+          },
+        );
+        app.post('/node_reset/:org',
+          loggedInOrFail,
+          can_ADMIN,  // Gets org from URL - an organization admin may reset only their own nodes
+          express.json({ limit: '4kb' }),
+          (req, res) => {
+            const { project, nodeid } = req.body || {};
+            if (!project || !nodeid) {
+              return res.status(400).json({ error: 'project and nodeid are required' });
+            }
+            forgetNode(db, req.params.org, project, nodeid, (err, changes) => {
+              if (err) { return res.status(500).json({ error: err.message }); }
+              if (!changes) {
+                return res.status(404).json({ error: `No enrolled node ${project}/${nodeid}` });
+              }
+              console.log("Node reset by", req.user.username, "-", req.params.org, project, nodeid);
+              // Remove the broker account too. Best effort: enrolling recreates it, and
+              // frugal-iot-diagnostic reports anything left behind as drift.
+              dropNode(config, req.params.org, project, nodeid, (derr) => {
+                res.status(200).json({
+                  reset: `${req.params.org}/${project}/${nodeid}`,
+                  brokerAccountRemoved: !derr,
+                  message: derr
+                    ? 'Forgotten. The broker account could not be removed, which is harmless - enrolling replaces it.'
+                    : 'Forgotten. The node will enrol again and be issued a new credential.',
+                });
+              });
+            });
+          },
+        );
         app.get('/projects_list/:org',
           loggedInOrFail,
           can_ADMIN, // Gets org from URL
@@ -1200,7 +1630,9 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
           shouldIBeLoggedIn, // redirect to ./login.html if not logged in then back here
           //(req,res,next) => {console.log("XXX back to /dashboard handler for", req.url); next(); }, // Log attempt
           //(req,res,next) => {console.log("XXX", config.server.htmldir); next(); }, // Log attempt
-          express.static(config.server.htmldir, {immutable: true, maxAge: 1000 * 60 * 60 * 24}) // Serve static
+          // Cached for a day but not immutable - see the note on routerNM above. This is the mount
+          // that serves the dashboard's own code, which changes every release at the same URL.
+          express.static(config.server.htmldir, revalidate) // Serve static
         );
 
         // Serve frugal-iot-logger data at /data but configure where to get them.
@@ -1215,8 +1647,23 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
         // Important that these aren't cached, or the data will not be updated.
         routerData.use(
           loggedInOrRedirect,
+          // The organization for the permission check comes out of the path, and express.static
+          // resolves "." and ".." afterwards - so before this, a request naming one organization
+          // and reading another's file passed the check: /data/dev/../varta/x.csv was authorised as
+          // "dev" and served varta's file. Percent-encoding (%2e%2e, ..%2f) did the same, and a
+          // browser hides it by normalising the path itself - but curl --path-as-is, or any script
+          // writing its own request, does not. Decoding and normalising here means the organization
+          // checked is the one whose file express.static will actually reach.
           (req,res,next) => {
-            res.locals.org = req.url.split("/")[1];
+            let decoded;
+            try {
+              decoded = decodeURIComponent(req.path);
+            } catch (e) { // A malformed escape; nothing legitimate sends one
+              res.sendStatus(400);
+              return;
+            }
+            // path.posix, not path: a URL separator is "/" whoever is running the server.
+            res.locals.org = path.posix.normalize(decoded).split("/")[1];
             next(); }, //
           can_READ,
           // Nothing logged here on purpose - morgan already reports the URL and status of this same
@@ -1278,10 +1725,12 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
             let dir;
             if (!req.body.otakey) {
               return cb(new Error("must specify either OTA key or Device ID"));
-            } else if (req.body.project) {
-              dir = `${config.server.otadir}/${req.body.organization}/${req.body.project}/${req.body.otakey}`;
             } else {
-              dir = `${config.server.otadir}/${req.body.organization}/+/${req.body.otakey}`;
+              dir = safeJoin(config.server.otadir, req.body.organization,
+                req.body.project || '+', req.body.otakey);
+              if (!dir) {
+                return cb(new Error("Parameters may not name a directory outside the OTA directory"));
+              }
             }
             mkdir(dir, {recursive: true}, (err, unusedpath) => {
               if (err) {
@@ -1337,18 +1786,49 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
         });
 
         // Serve HTML files from a configurable location
-        // Use a 1-day cache to keep traffic down
-        // Its important that frugaliot.css is cached, or the UX will flash while checking it hasn't changed.
+        // A day, but not immutable, so a reload can pick up a new release - see the note on
+        // routerNM above. Keeping it cached is what stops frugaliot.css being rechecked on every
+        // component render, which would flash unstyled on a slow link.
         // This has to come AFTER all the more specific paths like /data etc
         // Default catches rest (especially "/" so should be last)
         app.use(
-          express.static(config.server.publicdir, {immutable: true, maxAge: 1000 * 60 * 60 * 24})
+          express.static(config.server.publicdir, revalidate)
         );
         app.use(clientErrorHandler);
         // Now start the server
         startServer();
-        // And logger
-        mqttLogger.start();
+
+        // And the logger - but give it its OWN broker account first, and create that account before
+        // it tries to use it.
+        //
+        // It used to connect as the organization itself, with readwrite over the whole topic tree,
+        // which meant a confused or compromised logger could invent sensor readings. Its own account
+        // is in <org>-read and <org>-write, and <org>-write is set/-only - so it can still publish
+        // the platform API's device commands (which is why it needs write at all) and can no longer
+        // forge a reading.
+        //
+        // Not fatal if the broker cannot be reached: syncLoggers reports it and hands back the
+        // derived credentials anyway, and the logger falls back to the organization's shared
+        // password, which works until S8 retires it.
+        syncLoggers(config, Object.keys(config.organizations || {}), (err, creds) => {
+          Object.entries(creds || {}).forEach(([org, cred]) => {
+            const o = config.organizations[org];
+            if (!o) return;
+            // Separate field names rather than overwriting mqtt_password, so that
+            // config.organizations[x].mqtt_password means the same thing in memory as it does in
+            // the file it came from. Withheld from browsers - see lib/config-for-user.js.
+            o.logger_userid = cred.username;
+            o.logger_password = cred.password;
+          });
+          mqttLogger.start();
+        });
+
+        // On a bridged Pi, keep production's logins and permissions in step (SECURITY.md S11).
+        // Does nothing unless config.d/replica.yaml says where to pull from, so a production
+        // server and a standalone Pi are unaffected.
+        startReplica(config, db, {
+          onUser: (id) => syncUserById(db, config, id),
+        });
       }
     });
   }