frugal-iot-server 2.0.1 → 2.0.3

package/frugal-iot-server.js

@@ -109,7 +109,7 @@ import { MqttLogger } from "frugal-iot-logger";  // https://github.com/mitra42/f
 import { createAPIRouter, createAPIErrorHandler } from './lib/api-routes.js';
 import { buildConfigFor, hasPermissions } from './lib/config-for-user.js';
 import { ensureSecrets, addEnrolmentSecret, removeEnrolmentSecret } from './lib/secrets.js';
-import { syncUser, syncUserById, syncLoggers, syncNode, dropNode } from './lib/dynsec-server.js';
+import { syncUser, syncUserById, syncLoggers, syncNode, dropNode, syncBridge } from './lib/dynsec-server.js';
 import { enrol, ENROL, enrolmentSecretsFor, makeRateLimiter, forgetNode,
          makeAttemptLog, setGrant, clearGrant, readGrants } from './lib/enrol.js';
 import { deleteRetained } from './lib/retained.js';
@@ -293,13 +293,25 @@ function get_people_list(org, cb) {
  * Nodes that asked to enrol and were refused. Module level because two things need it: POST /enrol
  * writes to it and GET /nodes_list reads it, and they are in different scopes.
  *
- * In memory on purpose - see makeAttemptLog. Losing it on restart costs nothing, because a node
- * that is still retrying reappears within minutes.
+ * Kept in memory and written through to the node_attempts table - see makeAttemptLog. It used to be
+ * memory alone, which meant a restart hid a node that was asking for up to the half hour until its
+ * next attempt.
  */
 const enrolAttempts = makeAttemptLog();
 
 // Which nodes have enrolled in an organization. No password, obviously - the point of the list is
 // to know what exists and to be able to forget one.
+/*
+ * The Pis bridging to this server for one organization.
+ *
+ * The token is deliberately not selected: it is the credential a Pi presents to pull replicated
+ * logins, and this list is rendered in a browser. "site" is also the bridge's broker account name
+ * ("bridge-<site>"), which is what ties a row here to the live state the broker publishes on
+ * $SYS/broker/connection/<clientid>/state.
+ */
+const sqlBridgesList = `
+  SELECT site, created_at, last_pull FROM bridges WHERE org = ? ORDER BY site
+;`;
 const sqlNodesList = `
   SELECT project, nodeid, lora, enrolled_at FROM nodes WHERE org = ? ORDER BY project, nodeid
 ;`;
@@ -628,6 +640,11 @@ function loggedInOrRedirect(req, res, next) {
   } else {
     // If originalUrl is /private/index.html then req.url is just /index.html
     const q = new URLSearchParams({ mode: 'signin', message: 'Please login', url: req.originalUrl });
+    // Never store this one. The paths behind it are fetched by script as well as navigated to - the
+    // dashboard asks for /data/.../<date>.csv - so a redirect kept in a cache would go on sending a
+    // browser that has since logged in to the login page, and that arrives as a 200 full of HTML
+    // where the caller expected CSV.
+    res.set('Cache-Control', 'no-store');
     res.redirect(307, `${loginUrl}?${q}`);
   }
 }
@@ -715,10 +732,13 @@ export function loggedInOrFail(req, res, next) {
 }
 // While serving from frugal-iot-client it is only the dashboard we want to protect
 // as need user to be logged in to access config etc
-// Note if originalUrl is /dashboard/index.html then req.url is just /index.html
+// Note if originalUrl is /dashboard/index.html then req.path is just /index.html
 function shouldIBeLoggedIn(req, res, next) {
   //console.log("XXX shouldIBeLoggedIn", req.user, req.params.org);
-  if ((['/','/index.html'].includes(req.url)) && !req.isAuthenticated()) {
+  // req.path, not req.url: the latter carries the query string, and the page is reached with one in
+  // ordinary use - login.html passes "lang" back - so /dashboard/?lang=fr would match neither name
+  // here and be served to whoever asked.
+  if ((['/','/index.html'].includes(req.path)) && !req.isAuthenticated()) {
     console.log(`Not authenticated redirecting ${req.url} for login`);
     // Capture the full original URL as-is in "url" (so e.g. /data?... comes back with all its params
     // intact), encoded so its own query string can't corrupt this redirect's query string. Carry "lang"
@@ -726,6 +746,9 @@ function shouldIBeLoggedIn(req, res, next) {
     // renders in the right language.
     const q = new URLSearchParams({ mode: 'signin', message: 'Please login', url: req.originalUrl });
     if (req.query.lang) { q.set('lang', req.query.lang); }
+    // Whether a browser is logged in has to be asked again every time, so say so rather than
+    // relying on a 307 being uncacheable by default - as loggedInOrRedirect does for the same reason.
+    res.set('Cache-Control', 'no-store');
     res.redirect(307, `${loginUrl}?${q}`);
   } else {
     next();
@@ -1023,6 +1046,15 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
             console.error("Error loading projects into config", err);
           }
         });
+        // Reload the nodes that were asking to enrol when this server last stopped, so a restart
+        // does not hide them until each retries - which for a node is half an hour away.
+        enrolAttempts.useDatabase(db, (err, loaded) => {
+          if (err) {
+            console.error("Could not read past enrolment attempts", err.message);
+          } else if (loaded) {
+            console.log("Enrolment attempts pending:", loaded);
+          }
+        });
         // app.use(express.json()); // Not needed
         app.use(express.urlencoded({ extended: true })); // Passport will not function without this
         app.set('trust proxy', 1); // trust first proxy - see note in https://www.npmjs.com/package/express-session
@@ -1409,6 +1441,26 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
           can_ADMIN,  // Gets org from URL
           send_nodes_list,
         );
+        /*
+         * Which Pis bridge to this server, and when each last checked in.
+         *
+         * last_pull is stamped by lib/replica.js every time a Pi collects this organization's
+         * logins, so it answers "is that Pi alive" without needing the broker at all. Whether the
+         * BRIDGE itself is up is a different question, answered by the retained message the broker
+         * publishes on $SYS/broker/connection/<account>/state - which the dashboard subscribes to
+         * directly, since it already holds an MQTT connection. The two differ in a way worth
+         * seeing: a Pi that is up with a broken bridge still pulls logins.
+         */
+        app.get('/bridges_list/:org',
+          loggedInOrFail,
+          can_ADMIN,  // Gets org from URL
+          (req, res) => {
+            db.all(sqlBridgesList, [req.params.org], (err, rows) => {
+              if (err) { return res.status(500).send(err.message); }
+              res.json(rows.map((r) => ({ ...r, account: `bridge-${r.site}` })));
+            });
+          },
+        );
         /*
          * Forget a node, so it can enrol again and be issued a new credential.
          *
@@ -1564,6 +1616,66 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
             });
           },
         );
+        /*
+         * Add a Pi bridge, or re-issue an existing one's credentials.
+         *
+         * The dashboard equivalent of frugal-iot-addbridge-prod, and it exists because that script
+         * needs root: it writes the password file and edits /etc/mosquitto/aclfile. Here the
+         * account and its rules go into the dynamic-security plugin, which the server can drive
+         * with the credential it already holds - so an organization's admin can add a bridge
+         * without anybody having a shell on this machine.
+         *
+         * The password is DERIVED from user_secret and the row's created_at, so nothing new is
+         * stored and frugal-iot-rebuild-dynsec can recreate the account exactly. Re-running this
+         * for a site that already exists moves created_at, which changes the derived password and
+         * mints a new replica token - deliberately, since that is how a Pi's credentials are
+         * rotated, and it revokes the old ones.
+         *
+         * POST because it changes things: a GET here could be triggered against a live admin
+         * session by any page that can make this browser fetch a URL (SEC-17).
+         */
+        app.post('/bridge_add/:org',
+          loggedInOrFail,
+          can_ADMIN,  // Gets org from URL - an admin may add a bridge only to their own organization
+          express.json({ limit: '4kb' }),
+          (req, res) => {
+            const org = req.params.org;
+            const site = String((req.body && req.body.site) || '').trim();
+            // The site becomes a broker account name and an MQTT topic level, so keep it to what
+            // is safe in both, and to what the Pi's own config can carry unquoted.
+            if (!/^[a-z0-9][a-z0-9-]{0,19}$/.test(site)) {
+              return res.status(400).json({ error:
+                'Site must be 1-20 characters, lower-case letters, digits and hyphens, starting with a letter or digit' });
+            }
+            const now = Date.now();
+            const token = crypto.randomBytes(32).toString('base64url');
+            db.run(`INSERT INTO bridges (org, site, token, created_at) VALUES (?, ?, ?, ?)
+                    ON CONFLICT(org, site) DO UPDATE SET token = excluded.token,
+                                                         created_at = excluded.created_at`,
+              [org, site, token, now], (err) => {
+                if (err) { return res.status(500).json({ error: err.message }); }
+                syncBridge(config, { org, site, createdAt: now }, (serr, result) => {
+                  if (serr) {
+                    // The row is written but the broker has no account, so the Pi could not connect.
+                    // Say so plainly rather than handing over a credential that does not work.
+                    return res.status(500).json({ error:
+                      `The bridge was recorded but its broker account could not be created: ${serr.message}` });
+                  }
+                  console.log("Bridge added by", req.user.username, "-", org, site);
+                  res.status(200).json({
+                    site,
+                    account: result.username,
+                    password: result.password,
+                    replica_token: token,
+                    // The command to run on the Pi. Neither secret is in it on purpose: that
+                    // script prompts for both precisely so they stay out of the Pi's shell history,
+                    // and putting them here would undo that.
+                    command: `npx --no frugal-iot-addbridge-pi ${org} <this-server-fqdn> ${result.username}`,
+                  });
+                });
+              });
+          },
+        );
         app.post('/node_reset/:org',
           loggedInOrFail,
           can_ADMIN,  // Gets org from URL - an organization admin may reset only their own nodes

package/lib/dynsec-plan.js

@@ -28,6 +28,15 @@ export const names = {
   gatewayRole: (org) => `${org}-gateway`,
   discoverRole:(org) => `${org}-discover`,
   ownSubtreeRole: 'own-subtree',      // one for the whole broker: publishClientSend %u/#
+  /*
+   * Reading whether a bridge is connected, for the dashboard's Bridges card.
+   *
+   * A GROUP and not just a role, although only one role goes in it, because applyUser removes a
+   * client from groups it no longer qualifies for but never removes a role - so losing ADMIN would
+   * leave the rights behind if this were granted as a role.
+   */
+  bridgeStateRole:  'bridge-state',
+  bridgeStateGroup: 'bridge-state',
   publicRole:     'public-read',      // one for the whole broker: the id-0 "everyone" rows
   publicGroup:    'public',           // what setAnonymousGroup points at
   nodeClient:  (org, project, nodeid) => `${org}/${project}/${nodeid}`,
@@ -66,6 +75,18 @@ export const names = {
    */
   orgAdminClient: (org) => `${org}-admin`,
   clearRole: (org) => `${org}-clear`,
+
+  /*
+   * A Pi bridge. The account name is fixed by history: addbridge-prod.zsh has always called it
+   * "bridge-<site>", the Pi's mosquitto config carries that as remote_clientid, and the state topic
+   * the dashboard reads is named after it - so this cannot be renamed without reconfiguring every
+   * bridge that exists.
+   *
+   * Its role is per-bridge, not per-organization, because one of its rules names the account:
+   * a bridge may publish its OWN connection state and no one else's.
+   */
+  bridgeClient: (site) => `bridge-${site}`,
+  bridgeRole:   (site) => `bridge-${site}`,
 };
 
 // Topic filters, in one place for the same reason.
@@ -83,6 +104,8 @@ const topics = {
   // dashboard showing no devices while the readings themselves went through perfectly.
   discovery:   (org, project) => `${org}/${project}`,
   bridgeState: '$SYS/broker/connection/+/state',
+  // One bridge's own state, which only that bridge may publish
+  bridgeOwnState: (account) => `$SYS/broker/connection/${account}/state`,
 };
 
 /*
@@ -111,6 +134,12 @@ export function groupsForUser(rows) {
     const project = r.project || ORGWIDE;
     if (r.capability === 'READ')  groups.add(names.readGroup(r.org, project));
     if (r.capability === 'WRITE') groups.add(names.writeGroup(r.org, project));
+    // An organization's admin may see whether its Pis' bridges are up. The topic is not
+    // org-scoped - a clientid is "bridge-<site>" with no organization in it - so this does show an
+    // admin of one organization that another's bridges exist, by name. Site names are not secret
+    // and the alternative is naming bridge accounts per organization, which would break every
+    // bridge already configured.
+    if (r.capability === 'ADMIN') groups.add(names.bridgeStateGroup);
   }
   // Every user also gets whatever is public, as one role rather than per-organization membership.
   roles.add(names.publicRole);
@@ -124,7 +153,7 @@ export function groupsForUser(rows) {
  *   scopes    [{org, project}]  - '' project meaning organization-wide
  *   publics   [{org, project}]  - the id-0 READ rows, as ACLs on the one shared public role
  */
-export function desiredRolesAndGroups(scopes, publics, projects) {
+export function desiredRolesAndGroups(scopes, publics, projects, bridges) {
   const roles = {};
   const groups = {};
 
@@ -133,6 +162,15 @@ export function desiredRolesAndGroups(scopes, publics, projects) {
     { acltype: 'publishClientSend', topic: topics.ownSubtree, allow: true },
   ];
 
+  // Whether each bridge is connected: mosquitto publishes a retained 1/0 per bridge account, and
+  // both ACLs are needed - subscribePattern to be allowed to ask, publishClientReceive to actually
+  // be sent it.
+  roles[names.bridgeStateRole] = [
+    { acltype: 'subscribePattern', topic: topics.bridgeState, allow: true },
+    { acltype: 'publishClientReceive', topic: topics.bridgeState, allow: true },
+  ];
+  groups[names.bridgeStateGroup] = { roles: [names.bridgeStateRole] };
+
   // Public read, as ACL pairs on a single role - so withdrawing it is one removeRoleACL, for
   // everybody, rather than removing every user from a group.
   roles[names.publicRole] = [];
@@ -199,6 +237,25 @@ export function desiredRolesAndGroups(scopes, publics, projects) {
       roles: [names.readRole(org, ORGWIDE), names.ownSubtreeRole, names.discoverRole(org)] };
     groups[names.gatewayGroup(org)] = { roles: [names.gatewayRole(org)] };
   }
+
+  /*
+   * One role per bridge: the whole of its organization, plus its own connection state.
+   *
+   * The state rule is what the dashboard's Bridges card depends on. A bridge's LOCAL side is exempt
+   * from access control, so a Pi always has its own state; the REMOTE side is an ordinary client,
+   * and without this it is refused - silently, which is how a bridge that was up and relaying
+   * perfectly still showed as "Unknown".
+   */
+  for (const { org, site } of bridges || []) {
+    if (!org || !site) continue;
+    const account = names.bridgeClient(site);
+    roles[names.bridgeRole(site)] = [
+      { acltype: 'subscribePattern', topic: topics.orgTree(org, ORGWIDE), allow: true },
+      { acltype: 'publishClientSend', topic: topics.orgTree(org, ORGWIDE), allow: true },
+      { acltype: 'publishClientReceive', topic: topics.orgTree(org, ORGWIDE), allow: true },
+      { acltype: 'publishClientSend', topic: topics.bridgeOwnState(account), allow: true },
+    ];
+  }
   return { roles, groups };
 }
 
@@ -224,6 +281,23 @@ export function deriveLoggerPassword(userSecret, org) {
  * then a confused or compromised logger could invent sensor data - and the readings are the whole
  * point of the system. The <org>-write role is set/-only, which is exactly that.
  */
+/*
+ * A bridge's password. Derived, like the logger's and the org admin's, so nothing has to be stored
+ * and "the broker can be rebuilt from the database" stays true - a bridge account recreated by
+ * frugal-iot-rebuild-dynsec gets exactly the password the Pi is already configured with.
+ *
+ * created_at is in the input so that a bridge CAN be given a new password: updating that column
+ * changes the derivation, which is what re-running the add makes happen, and the old credential
+ * stops working. Without it a bridge password could never be rotated short of changing user_secret,
+ * which would reset every account on the broker.
+ */
+export function deriveBridgePassword(userSecret, site, createdAt) {
+  if (!userSecret) throw new Error('No user_secret - run frugal-iot-init');
+  return createHmac('sha256', userSecret)
+    .update('bridge:').update(String(site)).update(':').update(String(createdAt))
+    .digest('base64url').slice(0, 22);
+}
+
 // The server's administrative account for one organization. Derived, like the logger's, with its
 // own label so the two can never collide.
 export function deriveOrgAdminPassword(userSecret, org) {

package/lib/dynsec-server.js

@@ -9,7 +9,8 @@
  */
 
 import { dynsecConnect } from './dynsec.js';
-import { readUserRows, applyUser, applyLogger, applyNode, removeUser } from './dynsec-sync.js';
+import { readUserRows, applyUser, applyLogger, applyNode, removeUser,
+         applyRolesAndGroups, applyBridge, removeBridge } from './dynsec-sync.js';
 import { derivePassword, deriveLoggerPassword, names } from './dynsec-plan.js';
 
 let unavailableReported = false;
@@ -226,6 +227,45 @@ export function dropNode(config, org, project, nodeid, cb) {
   });
 }
 
+/*
+ * Create or re-issue one Pi bridge's broker account, and its role.
+ *
+ * Both the role and the client, because a bridge's role names the bridge - unlike a user, whose
+ * groups exist already. applyRolesAndGroups is given just this one bridge: it is idempotent and
+ * only ever adds, so the roles and groups of everything else are left as they are.
+ *
+ * Unlike addbridge-prod.zsh this needs no root and no shell: the account goes into the plugin
+ * rather than the password file, and the ACL into a dynsec role rather than /etc/mosquitto/aclfile.
+ * That is what lets an organization's admin add a bridge from the dashboard.
+ */
+export function syncBridge(config, { org, site, createdAt }, cb) {
+  const userSecret = config.secrets && config.secrets.user_secret;
+  if (!userSecret) return cb(new Error('No user_secret in config.d/secrets.yaml'));
+  if (!dynsecConfigured(config)) {
+    return cb(new Error('This server has no dynamic security credential, so it cannot create '
+      + 'broker accounts. Use frugal-iot-addbridge-prod from a shell on the server instead.'));
+  }
+  withDynsec(config, (cerr, dynsec, done) => {
+    if (cerr) { reportUnavailable(`bridge ${site}`, cerr); return cb(cerr); }
+    applyRolesAndGroups(dynsec, [], [], [], [{ org, site }], (rerr) => {
+      if (rerr) { done(); return cb(rerr); }
+      applyBridge(dynsec, { org, site, createdAt, userSecret }, (err, res) => { done(); cb(err, res); });
+    });
+  });
+}
+
+// Remove a bridge's account and its role, for when a Pi is retired.
+export function dropBridge(config, site, cb) {
+  cb = cb || (() => {});
+  withDynsec(config, (cerr, dynsec, done) => {
+    if (cerr) { reportUnavailable(`removing bridge ${site}`, cerr); return cb(cerr); }
+    removeBridge(dynsec, site, (err) => {
+      done();
+      cb(err && !/not found/i.test(err.message) ? err : null);
+    });
+  });
+}
+
 // When an account goes away. Not wired up yet - nothing deletes users - but the rebuild tool cannot
 // notice a user that is no longer in the database, so this is what the deletion path will need.
 export function dropUser(config, username, cb) {

package/lib/dynsec-sync.js

@@ -7,7 +7,7 @@
  */
 
 import { desiredRolesAndGroups, groupsForUser, groupsForNode, groupsForLogger,
-         derivePassword, deriveLoggerPassword, deriveOrgAdminPassword, names } from './dynsec-plan.js';
+         derivePassword, deriveLoggerPassword, deriveOrgAdminPassword, deriveBridgePassword, names } from './dynsec-plan.js';
 
 // ---- reading the database -------------------------------------------------------------------
 // Every (org, project) any permission row mentions, plus every organization in the config, so an
@@ -51,8 +51,8 @@ export function readUserRows(db, id, cb) {
  * Create every role and group the plan calls for, and set their ACLs. Idempotent: re-running is how
  * the rebuild tool works, and how a release that adds a rule delivers it.
  */
-export function applyRolesAndGroups(dynsec, scopes, publics, projects, cb) {
-  const { roles, groups } = desiredRolesAndGroups(scopes, publics, projects);
+export function applyRolesAndGroups(dynsec, scopes, publics, projects, bridges, cb) {
+  const { roles, groups } = desiredRolesAndGroups(scopes, publics, projects, bridges);
   const create = [];
   const acls = [];
   for (const [rolename, list] of Object.entries(roles)) {
@@ -171,6 +171,37 @@ export function applyOrgAdmin(dynsec, { org, userSecret }, cb) {
   });
 }
 
+/*
+ * One Pi bridge's broker account.
+ *
+ * Its role is attached DIRECTLY rather than through a group, like the org admin's, because the role
+ * names this one account in its state rule - a group would invite a second bridge being put in it,
+ * which would let each publish the other's connection state.
+ *
+ * Nothing here needs root, which is the point: addbridge-prod.zsh had to run mosquitto_passwd and
+ * edit /etc/mosquitto/aclfile, so only someone with a shell on the server could add a bridge.
+ */
+export function applyBridge(dynsec, { org, site, createdAt, userSecret }, cb) {
+  const username = names.bridgeClient(site);
+  const password = deriveBridgePassword(userSecret, site, createdAt);
+  const role = names.bridgeRole(site);
+  dynsec.idempotent([{ command: 'createClient', username, password }], (err) => {
+    if (err) return cb(err);
+    dynsec.getClient(username, (err2, client) => {
+      if (err2) return cb(err2);
+      const haveRoles = ((client && client.roles) || []).map((r) => r.rolename || r);
+      const cmds = [{ command: 'setClientPassword', username, password }];
+      if (!haveRoles.includes(role)) cmds.push({ command: 'addClientRole', username, rolename: role });
+      dynsec.idempotent(cmds, (err3) => cb(err3, { username, password, org }));
+    });
+  });
+}
+
+export function removeBridge(dynsec, site, cb) {
+  dynsec.raw([{ command: 'deleteClient', username: names.bridgeClient(site) },
+              { command: 'deleteRole', rolename: names.bridgeRole(site) }], (err) => cb(err));
+}
+
 export function removeUser(dynsec, username, cb) {
   dynsec.raw([{ command: 'deleteClient', username: names.userClient(username) }], (err) => {
     if (err && /not found/i.test(err.message)) return cb(null);
@@ -184,7 +215,7 @@ export function removeUser(dynsec, username, cb) {
  * frugal-iot-rebuild-dynsec --check reports and what scripts/diagnostic.zsh calls: reporting drift
  * is useful more often than repairing it.
  */
-export function checkRolesAndGroups(dynsec, scopes, publics, projects, cb) {
+export function checkRolesAndGroups(dynsec, scopes, publics, projects, bridges, cb) {
   const { roles, groups } = desiredRolesAndGroups(scopes, publics, projects);
   const differences = [];
   dynsec.listRoles((err, haveRoles) => {

package/lib/enrol.js

@@ -72,14 +72,40 @@ function sameSecret(a, b) {
  * leaked secret filling the database with invented nodes, and per node id stops repeated attempts
  * against one real node. In memory is enough - it survives as long as the process, a restart is a
  * fair reset, and persisting it would mean a disk write per attempt.
+ *
+ * The numbers were set for a node that retries every half hour - twice an hour, well under a limit
+ * of five. THAT IS NOT WHAT A DEEP-SLEEPING NODE DOES. nextEnrolTime lives in RAM, which deep sleep
+ * discards, so such a node asks once per WAKE: at the default ten-minute cycle that is six an hour,
+ * over the old per-node limit, and three such nodes exhausted the whole organization's twenty. A
+ * fleet flashed together and waiting to be approved is the ordinary case, not an attack, and the
+ * limit turned it into `rate_limited` rows that hid the real reason each node was being refused.
+ *
+ * So: twelve per node (a five-minute cycle) and a hundred and twenty per organization (twenty such
+ * nodes). What this costs is small, because the limiter is no longer the only thing bounding the
+ * damage - makeAttemptLog evicts to perOrg rows, throttles repeats to one write a minute, and
+ * writes nothing at all for a rate-limited request. And the per-node number never protected much:
+ * the only thing it guards by counting is the `current_password` proof, which is 24 random bytes
+ * and is not reachable by guessing at five an hour or at fifty.
  */
-export function makeRateLimiter({ perOrg = 20, perNode = 5, windowMs = 3600000, now = Date.now } = {}) {
+export function makeRateLimiter({ perOrg = 120, perNode = 12, windowMs = 3600000, now = Date.now,
+                                  sweepAbove = 1000 } = {}) {
   const hits = new Map();   // key -> [timestamps]
   const record = (key, limit) => {
     const t = now();
     const kept = (hits.get(key) || []).filter((x) => x > t - windowMs);
     kept.push(t);
     hits.set(key, kept);
+    /*
+     * Sweep expired keys, rather than only expired timestamps within a key.
+     *
+     * Nothing removed a key once made, which was a slow leak even when every key was a valid node
+     * id, and is a faster one now that the limiter runs BEFORE the shape check and so sees whatever
+     * an unauthenticated caller sent. Only above a threshold, so the ordinary case - a handful of
+     * nodes in a handful of organizations - never walks the map at all.
+     */
+    if (hits.size > sweepAbove) {
+      for (const [k, v] of hits) if (!v.length || v[v.length - 1] <= t - windowMs) hits.delete(k);
+    }
     return kept.length <= limit;
   };
   return {
@@ -105,28 +131,87 @@ const NAME = /^[a-z0-9]{1,32}$/;
  * are therefore UNTRUSTED - anyone can make an id appear by attempting enrolment - so each keeps
  * when it asked and from where, for the admin to recognise rather than to trust.
  *
- * In memory, and bounded. The nodes table deliberately has no last_seen column because that would
- * be a database write per reading on an SD card; a write per enrolment attempt is the same mistake
- * on a smaller scale. Losing the list on restart costs nothing: a node that is still retrying
- * reappears within minutes.
+ * Bounded, and PERSISTED once useDatabase() is called - see the node_attempts table for why that
+ * changed. Briefly: "losing the list on a restart costs nothing, a node still retrying reappears
+ * within minutes" was wrong on the interval. A node retries every HALF HOUR, so a restart made a
+ * node that was asking indistinguishable from a node that was not asking at all, for up to thirty
+ * minutes - and that is exactly the question an administrator restarts a server to investigate.
  */
-export function makeAttemptLog({ perOrg = 50, now = Date.now } = {}) {
-  const rows = new Map();   // "org/nodeid" -> {org, project, nodeid, at, from, reason, count}
+export function makeAttemptLog({ perOrg = 50, now = Date.now, minWriteMs = 60000 } = {}) {
+  const rows = new Map();   // "org/nodeid" -> {org, project, nodeid, at, from, reason, count, written}
+  let db = null;            // set by useDatabase(), because the database is opened after this is made
+  const keyOf = (org, nodeid) => `${org}/${nodeid}`;
+
+  const write = (r) => {
+    if (!db) return;
+    r.written = r.at;
+    db.run(`INSERT INTO node_attempts (org, project, nodeid, reason, from_addr, at, count)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(org, nodeid)
+            DO UPDATE SET project = excluded.project, reason = excluded.reason,
+                          from_addr = excluded.from_addr, at = excluded.at, count = excluded.count`,
+      [r.org, r.project || '', r.nodeid, r.reason || '', r.from || '', r.at, r.count],
+      // Best effort: this is a diagnostic aid, and failing to record one must not turn an
+      // enrolment refusal into a 500.
+      (err) => { if (err) console.error("Could not record enrolment attempt:", err.message); });
+  };
+  const drop = (org, nodeid) => {
+    rows.delete(keyOf(org, nodeid));
+    if (db) db.run('DELETE FROM node_attempts WHERE org = ? AND nodeid = ?', [org, nodeid], () => {});
+  };
+
   return {
-    record({ org, project, nodeid, from, reason }) {
-      const key = `${org}/${nodeid}`;
+    /*
+     * Adopt a database: load what is already there, and write through from then on.
+     *
+     * Separate from the constructor only because of when each happens - the log is made at module
+     * level so that POST /enrol and GET /nodes_list share one, and the database is opened later.
+     * Until this is called the log behaves exactly as it used to, in memory alone, so a caller
+     * that never calls it (a test) needs no database.
+     */
+    useDatabase(database, cb) {
+      db = database;
+      db.all('SELECT org, project, nodeid, reason, from_addr, at, count FROM node_attempts', [],
+        (err, saved) => {
+          if (err) return cb(err);
+          for (const r of saved || []) {
+            rows.set(keyOf(r.org, r.nodeid), {
+              org: r.org, project: r.project, nodeid: r.nodeid, from: r.from_addr,
+              reason: r.reason, at: r.at, count: r.count, written: r.at,
+            });
+          }
+          cb(null, (saved || []).length);
+        });
+    },
+    /*
+     * durable: false records in memory but does not write. Used for a rate-limited request, which
+     * is the one kind that arrives faster than the limiter caps - the limiter refuses it but this
+     * is still called, so without that flag an unauthenticated flood would be a database write per
+     * request. A node that is genuinely retrying has already been written by its first few
+     * attempts, so nothing an administrator needs is lost.
+     */
+    record({ org, project, nodeid, from, reason, durable = true }) {
+      const key = keyOf(org, nodeid);
       const existing = rows.get(key);
-      rows.set(key, {
+      const row = {
         org, project, nodeid, from, reason,
         at: now(),
         count: existing ? existing.count + 1 : 1,
-      });
+        written: existing ? (existing.written || 0) : 0,
+      };
+      rows.set(key, row);
+      // A new node, or one whose refusal reason has changed, is written at once - those are what an
+      // administrator is waiting to see. A node repeating the same refusal is throttled, because
+      // the only thing changing is a timestamp and a count.
+      if (durable && (!existing || existing.reason !== reason || row.at - row.written >= minWriteMs)) {
+        write(row);
+      }
       // Oldest out first, per organization, so one organization cannot crowd out another
       const mine = [...rows.values()].filter((r) => r.org === org).sort((a, b) => a.at - b.at);
-      for (const r of mine.slice(0, Math.max(0, mine.length - perOrg))) rows.delete(`${r.org}/${r.nodeid}`);
+      for (const r of mine.slice(0, Math.max(0, mine.length - perOrg))) drop(r.org, r.nodeid);
     },
     forOrg(org) { return [...rows.values()].filter((r) => r.org === org); },
-    forget(org, nodeid) { rows.delete(`${org}/${nodeid}`); },
+    forget(org, nodeid) { drop(org, nodeid); },
   };
 }
 
@@ -164,30 +249,48 @@ export function readGrants(db, org, cb) {
  */
 export function enrol(deps, body, cb) {
   const { db, config, limiter, syncNode, attempts, from } = deps;
-  // Every refusal past the shape check is worth showing an admin: it is the only way they learn
-  // that a node exists and is asking. The shape check is not - a request that is not even
-  // org/project/nodeid tells them nothing they could act on.
-  const refuse = (code, message) => {
-    if (attempts) {
-      attempts.record({
-        org: String((body && body.org) || ''), project: String((body && body.project) || ''),
-        nodeid: String((body && body.nodeid) || ''), from, reason: code,
-      });
-    }
-    return cb(new EnrolError(code, message));
-  };
   const org = String((body && body.org) || '');
   const project = String((body && body.project) || '');
   const nodeid = String((body && body.nodeid) || '');
   const secret = String((body && body.enrolment_secret) || '');
   const lora = !!(body && body.lora);
 
-  if (!NAME.test(org) || !NAME.test(project) || !NODEID.test(nodeid)) {
-    return cb(new EnrolError(ENROL.BAD_REQUEST, 'org, project and nodeid are required and must be plain names'));
+  /*
+   * EVERY refusal is worth showing an admin, including a malformed one. It is the only way they
+   * learn that a node exists and is asking at all.
+   *
+   * The shape check used to be excluded, on the grounds that a request which is not even
+   * org/project/nodeid tells an admin nothing they could act on. That was wrong twice over: a node
+   * whose project is mistyped (a capital letter, a hyphen) fails the shape check, and then the ONE
+   * symptom of the typo - "nothing appears anywhere" - is identical to the node being dead, off the
+   * network, or pointed at a different server. Seeing the row is what distinguishes them, even
+   * though approving it would not help. What an admin can act on is the FIRMWARE.
+   *
+   * Two conditions on recording, both because the strings below the shape check are arbitrary input
+   * from an unauthenticated request:
+   *   - the organization must be a nameable one, or there is no admin who could ever see the row
+   *     and nothing bounds how many organizations an attacker can invent;
+   *   - project and nodeid are truncated, since they are shown in a table.
+   */
+  const refuse = (code, message, durable = true) => {
+    if (attempts && NAME.test(org)) {
+      attempts.record({
+        org, project: project.slice(0, 64), nodeid: nodeid.slice(0, 64), from, reason: code, durable,
+      });
+    }
+    return cb(new EnrolError(code, message));
+  };
+
+  // Before the shape check, not after it as it used to be. Now that a malformed request is recorded,
+  // it has to be capped the same way a well-formed one is, or a flood of invented node ids would
+  // reach the log unlimited.
+  if (limiter && !limiter.allow(org, nodeid.slice(0, 64))) {
+    // durable: false - the one refusal that by definition arrives faster than the limit allows
+    return refuse(ENROL.RATE_LIMITED, 'Too many enrolment attempts - try later', false);
   }
 
-  if (limiter && !limiter.allow(org, nodeid)) {
-    return refuse(ENROL.RATE_LIMITED, 'Too many enrolment attempts - try later');
+  if (!NAME.test(org) || !NAME.test(project) || !NODEID.test(nodeid)) {
+    return refuse(ENROL.BAD_REQUEST, 'org, project and nodeid are required and must be plain names');
   }
 
   /*