frugal-iot-server 2.0.1 → 2.0.2

package/lib/enrol.js

@@ -72,14 +72,40 @@ function sameSecret(a, b) {
  * leaked secret filling the database with invented nodes, and per node id stops repeated attempts
  * against one real node. In memory is enough - it survives as long as the process, a restart is a
  * fair reset, and persisting it would mean a disk write per attempt.
+ *
+ * The numbers were set for a node that retries every half hour - twice an hour, well under a limit
+ * of five. THAT IS NOT WHAT A DEEP-SLEEPING NODE DOES. nextEnrolTime lives in RAM, which deep sleep
+ * discards, so such a node asks once per WAKE: at the default ten-minute cycle that is six an hour,
+ * over the old per-node limit, and three such nodes exhausted the whole organization's twenty. A
+ * fleet flashed together and waiting to be approved is the ordinary case, not an attack, and the
+ * limit turned it into `rate_limited` rows that hid the real reason each node was being refused.
+ *
+ * So: twelve per node (a five-minute cycle) and a hundred and twenty per organization (twenty such
+ * nodes). What this costs is small, because the limiter is no longer the only thing bounding the
+ * damage - makeAttemptLog evicts to perOrg rows, throttles repeats to one write a minute, and
+ * writes nothing at all for a rate-limited request. And the per-node number never protected much:
+ * the only thing it guards by counting is the `current_password` proof, which is 24 random bytes
+ * and is not reachable by guessing at five an hour or at fifty.
  */
-export function makeRateLimiter({ perOrg = 20, perNode = 5, windowMs = 3600000, now = Date.now } = {}) {
+export function makeRateLimiter({ perOrg = 120, perNode = 12, windowMs = 3600000, now = Date.now,
+                                  sweepAbove = 1000 } = {}) {
   const hits = new Map();   // key -> [timestamps]
   const record = (key, limit) => {
     const t = now();
     const kept = (hits.get(key) || []).filter((x) => x > t - windowMs);
     kept.push(t);
     hits.set(key, kept);
+    /*
+     * Sweep expired keys, rather than only expired timestamps within a key.
+     *
+     * Nothing removed a key once made, which was a slow leak even when every key was a valid node
+     * id, and is a faster one now that the limiter runs BEFORE the shape check and so sees whatever
+     * an unauthenticated caller sent. Only above a threshold, so the ordinary case - a handful of
+     * nodes in a handful of organizations - never walks the map at all.
+     */
+    if (hits.size > sweepAbove) {
+      for (const [k, v] of hits) if (!v.length || v[v.length - 1] <= t - windowMs) hits.delete(k);
+    }
     return kept.length <= limit;
   };
   return {
@@ -105,28 +131,87 @@ const NAME = /^[a-z0-9]{1,32}$/;
  * are therefore UNTRUSTED - anyone can make an id appear by attempting enrolment - so each keeps
  * when it asked and from where, for the admin to recognise rather than to trust.
  *
- * In memory, and bounded. The nodes table deliberately has no last_seen column because that would
- * be a database write per reading on an SD card; a write per enrolment attempt is the same mistake
- * on a smaller scale. Losing the list on restart costs nothing: a node that is still retrying
- * reappears within minutes.
+ * Bounded, and PERSISTED once useDatabase() is called - see the node_attempts table for why that
+ * changed. Briefly: "losing the list on a restart costs nothing, a node still retrying reappears
+ * within minutes" was wrong on the interval. A node retries every HALF HOUR, so a restart made a
+ * node that was asking indistinguishable from a node that was not asking at all, for up to thirty
+ * minutes - and that is exactly the question an administrator restarts a server to investigate.
  */
-export function makeAttemptLog({ perOrg = 50, now = Date.now } = {}) {
-  const rows = new Map();   // "org/nodeid" -> {org, project, nodeid, at, from, reason, count}
+export function makeAttemptLog({ perOrg = 50, now = Date.now, minWriteMs = 60000 } = {}) {
+  const rows = new Map();   // "org/nodeid" -> {org, project, nodeid, at, from, reason, count, written}
+  let db = null;            // set by useDatabase(), because the database is opened after this is made
+  const keyOf = (org, nodeid) => `${org}/${nodeid}`;
+
+  const write = (r) => {
+    if (!db) return;
+    r.written = r.at;
+    db.run(`INSERT INTO node_attempts (org, project, nodeid, reason, from_addr, at, count)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(org, nodeid)
+            DO UPDATE SET project = excluded.project, reason = excluded.reason,
+                          from_addr = excluded.from_addr, at = excluded.at, count = excluded.count`,
+      [r.org, r.project || '', r.nodeid, r.reason || '', r.from || '', r.at, r.count],
+      // Best effort: this is a diagnostic aid, and failing to record one must not turn an
+      // enrolment refusal into a 500.
+      (err) => { if (err) console.error("Could not record enrolment attempt:", err.message); });
+  };
+  const drop = (org, nodeid) => {
+    rows.delete(keyOf(org, nodeid));
+    if (db) db.run('DELETE FROM node_attempts WHERE org = ? AND nodeid = ?', [org, nodeid], () => {});
+  };
+
   return {
-    record({ org, project, nodeid, from, reason }) {
-      const key = `${org}/${nodeid}`;
+    /*
+     * Adopt a database: load what is already there, and write through from then on.
+     *
+     * Separate from the constructor only because of when each happens - the log is made at module
+     * level so that POST /enrol and GET /nodes_list share one, and the database is opened later.
+     * Until this is called the log behaves exactly as it used to, in memory alone, so a caller
+     * that never calls it (a test) needs no database.
+     */
+    useDatabase(database, cb) {
+      db = database;
+      db.all('SELECT org, project, nodeid, reason, from_addr, at, count FROM node_attempts', [],
+        (err, saved) => {
+          if (err) return cb(err);
+          for (const r of saved || []) {
+            rows.set(keyOf(r.org, r.nodeid), {
+              org: r.org, project: r.project, nodeid: r.nodeid, from: r.from_addr,
+              reason: r.reason, at: r.at, count: r.count, written: r.at,
+            });
+          }
+          cb(null, (saved || []).length);
+        });
+    },
+    /*
+     * durable: false records in memory but does not write. Used for a rate-limited request, which
+     * is the one kind that arrives faster than the limiter caps - the limiter refuses it but this
+     * is still called, so without that flag an unauthenticated flood would be a database write per
+     * request. A node that is genuinely retrying has already been written by its first few
+     * attempts, so nothing an administrator needs is lost.
+     */
+    record({ org, project, nodeid, from, reason, durable = true }) {
+      const key = keyOf(org, nodeid);
       const existing = rows.get(key);
-      rows.set(key, {
+      const row = {
         org, project, nodeid, from, reason,
         at: now(),
         count: existing ? existing.count + 1 : 1,
-      });
+        written: existing ? (existing.written || 0) : 0,
+      };
+      rows.set(key, row);
+      // A new node, or one whose refusal reason has changed, is written at once - those are what an
+      // administrator is waiting to see. A node repeating the same refusal is throttled, because
+      // the only thing changing is a timestamp and a count.
+      if (durable && (!existing || existing.reason !== reason || row.at - row.written >= minWriteMs)) {
+        write(row);
+      }
       // Oldest out first, per organization, so one organization cannot crowd out another
       const mine = [...rows.values()].filter((r) => r.org === org).sort((a, b) => a.at - b.at);
-      for (const r of mine.slice(0, Math.max(0, mine.length - perOrg))) rows.delete(`${r.org}/${r.nodeid}`);
+      for (const r of mine.slice(0, Math.max(0, mine.length - perOrg))) drop(r.org, r.nodeid);
     },
     forOrg(org) { return [...rows.values()].filter((r) => r.org === org); },
-    forget(org, nodeid) { rows.delete(`${org}/${nodeid}`); },
+    forget(org, nodeid) { drop(org, nodeid); },
   };
 }
 
@@ -164,30 +249,48 @@ export function readGrants(db, org, cb) {
  */
 export function enrol(deps, body, cb) {
   const { db, config, limiter, syncNode, attempts, from } = deps;
-  // Every refusal past the shape check is worth showing an admin: it is the only way they learn
-  // that a node exists and is asking. The shape check is not - a request that is not even
-  // org/project/nodeid tells them nothing they could act on.
-  const refuse = (code, message) => {
-    if (attempts) {
-      attempts.record({
-        org: String((body && body.org) || ''), project: String((body && body.project) || ''),
-        nodeid: String((body && body.nodeid) || ''), from, reason: code,
-      });
-    }
-    return cb(new EnrolError(code, message));
-  };
   const org = String((body && body.org) || '');
   const project = String((body && body.project) || '');
   const nodeid = String((body && body.nodeid) || '');
   const secret = String((body && body.enrolment_secret) || '');
   const lora = !!(body && body.lora);
 
-  if (!NAME.test(org) || !NAME.test(project) || !NODEID.test(nodeid)) {
-    return cb(new EnrolError(ENROL.BAD_REQUEST, 'org, project and nodeid are required and must be plain names'));
+  /*
+   * EVERY refusal is worth showing an admin, including a malformed one. It is the only way they
+   * learn that a node exists and is asking at all.
+   *
+   * The shape check used to be excluded, on the grounds that a request which is not even
+   * org/project/nodeid tells an admin nothing they could act on. That was wrong twice over: a node
+   * whose project is mistyped (a capital letter, a hyphen) fails the shape check, and then the ONE
+   * symptom of the typo - "nothing appears anywhere" - is identical to the node being dead, off the
+   * network, or pointed at a different server. Seeing the row is what distinguishes them, even
+   * though approving it would not help. What an admin can act on is the FIRMWARE.
+   *
+   * Two conditions on recording, both because the strings below the shape check are arbitrary input
+   * from an unauthenticated request:
+   *   - the organization must be a nameable one, or there is no admin who could ever see the row
+   *     and nothing bounds how many organizations an attacker can invent;
+   *   - project and nodeid are truncated, since they are shown in a table.
+   */
+  const refuse = (code, message, durable = true) => {
+    if (attempts && NAME.test(org)) {
+      attempts.record({
+        org, project: project.slice(0, 64), nodeid: nodeid.slice(0, 64), from, reason: code, durable,
+      });
+    }
+    return cb(new EnrolError(code, message));
+  };
+
+  // Before the shape check, not after it as it used to be. Now that a malformed request is recorded,
+  // it has to be capped the same way a well-formed one is, or a flood of invented node ids would
+  // reach the log unlimited.
+  if (limiter && !limiter.allow(org, nodeid.slice(0, 64))) {
+    // durable: false - the one refusal that by definition arrives faster than the limit allows
+    return refuse(ENROL.RATE_LIMITED, 'Too many enrolment attempts - try later', false);
   }
 
-  if (limiter && !limiter.allow(org, nodeid)) {
-    return refuse(ENROL.RATE_LIMITED, 'Too many enrolment attempts - try later');
+  if (!NAME.test(org) || !NAME.test(project) || !NODEID.test(nodeid)) {
+    return refuse(ENROL.BAD_REQUEST, 'org, project and nodeid are required and must be plain names');
   }
 
   /*

package/package.json

@@ -12,8 +12,8 @@
     "cookie-parser": "^1.4.7",
     "express": "^5.2.1",
     "express-session": "^1.19.0",
-    "frugal-iot-client": "^2.0.1",
-    "frugal-iot-logger": "^2.0.1",
+    "frugal-iot-client": "^2.0.2",
+    "frugal-iot-logger": "^2.0.2",
     "hash-wasm": "^4.12.0",
     "js-yaml": "^4.2.0",
     "morgan": "^1.11.0",
@@ -73,7 +73,10 @@
   "license": "MIT",
   "main": "frugal-iot-server.js",
   "name": "frugal-iot-server",
-  "repository": "https://github.com/mitra42/frugal-iot-server",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mitra42/frugal-iot-server.git"
+  },
   "scripts": {
     "test": "vitest run",
     "test:watch": "vitest",
@@ -83,5 +86,5 @@
     "check-files": "node scripts/check-published-files.js"
   },
   "type": "module",
-  "version": "2.0.1"
+  "version": "2.0.2"
 }

package/public/service-worker.js

@@ -6,7 +6,7 @@
 // it, an installed browser keeps serving the files it already cached - and since the list now covers
 // the dashboard's own code, that means old JS behind freshly fetched HTML. Any edit to the list
 // needs this bumped, or it does not reach anyone who already has the old one.
-const CACHE_NAME = 'frugal-iot-cache-2.0.1';
+const CACHE_NAME = 'frugal-iot-cache-2.0.2';
 /*
  * Everything the app needs, so that a phone on a poor or expensive link fetches it once.
  *

package/scripts/addbridge-prod.zsh

@@ -144,15 +144,28 @@ fi
 # user *before* reading it - so the broker then cannot read its own password file and refuses to
 # start, with an error that says nothing about ownership. mosquitto_passwd also writes a temporary
 # backup beside the file, so it needs the directory and not only the file.
+#
+# Whoever runs it must be able to write the DIRECTORY, not merely the file, and the file's owner
+# frequently cannot: a password file at /etc/mosquitto/mosquitto_passwords is owned by mosquitto
+# inside a directory owned by root, and picking the owner then fails with "Error creating backup
+# password file". So the last resort is root - and because that leaves the file owned by root,
+# where mosquitto (which drops privileges before reading it) could no longer read it, the
+# ownership and mode are put back immediately afterwards.
+# stat's spelling differs between Linux (-c) and BSD/macOS (-f)
+PWDIR="${PWFILE:h}"
+PWFILE_OWNER=$(stat -c '%U' "$PWFILE" 2>/dev/null || stat -f '%Su' "$PWFILE" 2>/dev/null || true)
+PWFILE_GROUP=$(stat -c '%G' "$PWFILE" 2>/dev/null || stat -f '%Sg' "$PWFILE" 2>/dev/null || true)
+PWFILE_MODE=$(stat -c '%a' "$PWFILE" 2>/dev/null || stat -f '%Lp' "$PWFILE" 2>/dev/null || true)
+RESTORE_OWNER=0
 MOSQ_PASSWD_CMD=(mosquitto_passwd)
-if [[ ! -w "${PWFILE:h}" || ( -e "$PWFILE" && ! -w "$PWFILE" ) ]]; then
+if [[ ! -w "$PWDIR" || ( -e "$PWFILE" && ! -w "$PWFILE" ) ]]; then
   if command -v sudo >/dev/null; then
-    # stat's spelling differs between Linux (-c) and BSD/macOS (-f)
-    PWFILE_OWNER=$(stat -c '%U' "$PWFILE" 2>/dev/null || stat -f '%Su' "$PWFILE" 2>/dev/null || true)
-    if [[ -n "$PWFILE_OWNER" && "$PWFILE_OWNER" != "$(id -un)" && "$PWFILE_OWNER" != "root" ]]; then
+    if [[ -n "$PWFILE_OWNER" && "$PWFILE_OWNER" != "$(id -un)" && "$PWFILE_OWNER" != "root" ]] \
+       && sudo -u "$PWFILE_OWNER" test -w "$PWDIR" 2>/dev/null; then
       MOSQ_PASSWD_CMD=(sudo -u "$PWFILE_OWNER" mosquitto_passwd)
     else
       MOSQ_PASSWD_CMD=(sudo mosquitto_passwd)
+      [[ -n "$PWFILE_OWNER" && "$PWFILE_OWNER" != "root" ]] && RESTORE_OWNER=1
     fi
   fi
 fi
@@ -162,6 +175,12 @@ if ! $MOSQ_PASSWD_CMD -b "$PWFILE" "$ACCOUNT" "$PASSWORD"; then
   echo "Error: mosquitto_passwd failed - nothing has been changed." >&2
   exit 1
 fi
+if (( RESTORE_OWNER )); then
+  # Written as root, so hand it back before mosquitto next needs to read it
+  sudo chown "${PWFILE_OWNER}:${PWFILE_GROUP}" "$PWFILE"
+  [[ -n "$PWFILE_MODE" ]] && sudo chmod "$PWFILE_MODE" "$PWFILE"
+  echo "  (written as root, then given back to ${PWFILE_OWNER}:${PWFILE_GROUP} mode ${PWFILE_MODE})"
+fi
 
 # The access control rule. Without one the account can connect and reach nothing at all, because a
 # mosquitto acl_file is deny-by-default.
@@ -237,16 +256,21 @@ if [[ -z "$REPLICA_TOKEN" ]]; then
 fi
 
 echo ""
-echo "Done. On the Pi at site '${SITE}', run:"
+echo "Done. Collect these, then run the command at the bottom on the Pi."
 echo ""
-echo "  npx --no frugal-iot-addbridge-pi ${ORG_ID} <this server's public name> ${ACCOUNT}"
+echo "1. THE FQDN OF THIS SERVER - the DNS name on its TLS certificate."
+echo "   The Pi verifies the certificate against exactly what you type, so a local hostname, an IP"
+echo "   address, or a domain the certificate does not cover will fail to connect."
+echo "   TO TEST IT: open https://<name>/ in a browser. Loading with no certificate warning means"
+echo "   the name matches the certificate, which is the part that is easy to get wrong. The bridge"
+echo "   then uses port 8883 on that same host, so that port has to be open to the Pi as well."
+echo "   This host calls itself \"$(hostname -f 2>/dev/null || hostname)\" - use that only if it is"
+echo "   also the name on the certificate and one the Pi can resolve. Not guessed here, because"
+echo "   what a machine calls itself usually is not its public name."
 echo ""
-echo "Positional arguments on purpose: npm swallows any --flag it does not recognise and passes"
-echo "only the value on, so the --org form would arrive at that script as a bare word."
+echo "2. THE PASSWORD for ${ACCOUNT}, which the Pi's script will prompt for:"
 echo ""
-echo "It will ask for this password:"
-echo ""
-echo "  ${PASSWORD}"
+echo "   ${PASSWORD}"
 echo ""
 if [[ "$GENERATED" == true ]]; then
   echo "That was generated just now and is not stored anywhere you can read it back - copy it before"
@@ -255,15 +279,20 @@ if [[ "$GENERATED" == true ]]; then
   echo "and use that on the Pi instead."
 fi
 if [[ -n "$REPLICA_TOKEN" ]]; then
-  echo "And this replica token, so people registered here can log in on the Pi as well:"
+  echo "3. THE REPLICA TOKEN, so people registered here can log in on the Pi as well:"
   echo ""
-  echo "  ${REPLICA_TOKEN}"
+  echo "   ${REPLICA_TOKEN}"
   echo ""
-  echo "A new one replaces the old, so re-running this script revokes the Pi's previous token."
+  echo "   A new one replaces the old, so re-running this script revokes the Pi's previous token."
   echo ""
 fi
-echo "Fill in the public name yourself - it is deliberately not guessed here, because what this"
-echo "machine calls itself is usually not it. This host answers to \"$(hostname -f 2>/dev/null || hostname)\","
-echo "which is only the right answer if that is also the DNS name the Pi can reach and the name on"
-echo "the TLS certificate. The Pi verifies the certificate against whatever you put here, so a"
-echo "local hostname, or a domain the certificate does not cover, fails to connect."
+echo "Then, on the Pi at site '${SITE}':"
+echo ""
+if [[ -n "$REPLICA_TOKEN" ]]; then
+  echo "  npx --no frugal-iot-addbridge-pi ${ORG_ID} <fqdn-from-1-above> ${ACCOUNT} -- --replica-token <token-from-3>"
+else
+  echo "  npx --no frugal-iot-addbridge-pi ${ORG_ID} <fqdn-from-1-above> ${ACCOUNT}"
+fi
+echo ""
+echo "The first three are positional on purpose: npm swallows any --flag it does not recognise and"
+echo "passes only the value on, so the --org form would arrive at that script as a bare word."

package/scripts/addorganization.zsh

@@ -107,19 +107,38 @@ fi
 # opening it, and mosquitto_passwd writes a temporary backup beside it, so it needs the directory too
 # - neither of which we get by running as ourselves against a file owned by mosquitto or by root.
 # On Raspberry Pi OS the first user has passwordless sudo, so this is invisible.
+#
+# The owner is not always able to do it either: a file at /etc/mosquitto/mosquitto_passwords is
+# owned by mosquitto inside a directory owned by root, and the backup then cannot be written -
+# "Error creating backup password file". So check the chosen account can write the directory, and
+# fall back to root, putting the ownership back afterwards so mosquitto can still read the file.
+# stat's spelling differs between Linux (-c) and BSD/macOS (-f)
+MOSQUITTO_PASSWD_DIR="${MOSQUITTO_PASSWD_FILE:h}"
+MOSQUITTO_PASSWD_OWNER=$(stat -c '%U' "$MOSQUITTO_PASSWD_FILE" 2>/dev/null \
+  || stat -f '%Su' "$MOSQUITTO_PASSWD_FILE" 2>/dev/null || true)
+MOSQUITTO_PASSWD_GROUP=$(stat -c '%G' "$MOSQUITTO_PASSWD_FILE" 2>/dev/null \
+  || stat -f '%Sg' "$MOSQUITTO_PASSWD_FILE" 2>/dev/null || true)
+MOSQUITTO_PASSWD_MODE=$(stat -c '%a' "$MOSQUITTO_PASSWD_FILE" 2>/dev/null \
+  || stat -f '%Lp' "$MOSQUITTO_PASSWD_FILE" 2>/dev/null || true)
+MOSQUITTO_RESTORE_OWNER=0
 MOSQUITTO_PASSWD_CMD=(mosquitto_passwd)
-if [[ ! -w "${MOSQUITTO_PASSWD_FILE:h}" || ( -e "$MOSQUITTO_PASSWD_FILE" && ! -w "$MOSQUITTO_PASSWD_FILE" ) ]]; then
+if [[ ! -w "$MOSQUITTO_PASSWD_DIR" || ( -e "$MOSQUITTO_PASSWD_FILE" && ! -w "$MOSQUITTO_PASSWD_FILE" ) ]]; then
   if command -v sudo >/dev/null; then
-    # stat's spelling differs between Linux (-c) and BSD/macOS (-f)
-    MOSQUITTO_PASSWD_OWNER=$(stat -c '%U' "$MOSQUITTO_PASSWD_FILE" 2>/dev/null \
-      || stat -f '%Su' "$MOSQUITTO_PASSWD_FILE" 2>/dev/null || true)
-    if [[ -n "$MOSQUITTO_PASSWD_OWNER" && "$MOSQUITTO_PASSWD_OWNER" != "$(id -un)" && "$MOSQUITTO_PASSWD_OWNER" != "root" ]]; then
+    if [[ -n "$MOSQUITTO_PASSWD_OWNER" && "$MOSQUITTO_PASSWD_OWNER" != "$(id -un)" && "$MOSQUITTO_PASSWD_OWNER" != "root" ]] \
+       && sudo -u "$MOSQUITTO_PASSWD_OWNER" test -w "$MOSQUITTO_PASSWD_DIR" 2>/dev/null; then
       MOSQUITTO_PASSWD_CMD=(sudo -u "$MOSQUITTO_PASSWD_OWNER" mosquitto_passwd)
     else
       MOSQUITTO_PASSWD_CMD=(sudo mosquitto_passwd)
+      [[ -n "$MOSQUITTO_PASSWD_OWNER" && "$MOSQUITTO_PASSWD_OWNER" != "root" ]] && MOSQUITTO_RESTORE_OWNER=1
     fi
   fi
 fi
+# Called after any successful mosquitto_passwd run below
+restore_passwd_owner() {
+  (( MOSQUITTO_RESTORE_OWNER )) || return 0
+  sudo chown "${MOSQUITTO_PASSWD_OWNER}:${MOSQUITTO_PASSWD_GROUP}" "$MOSQUITTO_PASSWD_FILE"
+  [[ -n "$MOSQUITTO_PASSWD_MODE" ]] && sudo chmod "$MOSQUITTO_PASSWD_MODE" "$MOSQUITTO_PASSWD_FILE"
+}
 
 # ---- Escaping helpers ----
 # Escape a value for embedding in a double-quoted YAML string
@@ -229,6 +248,7 @@ else
     echo "If it complains about creating a backup file, the directory holding that file is not" >&2
     echo "writable by you - which is why the command above uses sudo." >&2
   else
+    restore_passwd_owner
     echo "Set mosquitto password for ${ORG_ID} in ${MOSQUITTO_PASSWD_FILE}"
   fi
 fi