frugal-iot-server 2.0.0 → 2.0.1

package/lib/dynsec-server.js

@@ -34,6 +34,26 @@ function dynsecConfigured(config) {
   return !!(s.dynsec_admin_user && s.dynsec_admin_password);
 }
 
+/*
+ * Another server owns this broker: derive credentials, never try to create them.
+ *
+ * For working on the server or the client against live data - a laptop pointed at the production
+ * broker. It has no admin credential and must not have one, but the accounts it needs already
+ * exist, created by the server that does. Given the same user_secret it derives exactly the same
+ * passwords, so what it hands a browser works.
+ *
+ * Declared in config.d/secrets.yaml rather than config.d/mqtt.yaml, for two reasons: that file is
+ * never committed, so this cannot be switched on by accident in a checkout; and it is where the
+ * borrowed user_secret has to go anyway, which is the real access control here. Without that
+ * secret a checkout of this repository cannot reach live data at all, whatever it sets.
+ *
+ * What does NOT work on such a server, by design: enrolling a node, resetting one, and
+ * frugal-iot-rebuild-dynsec - all of which create or delete accounts.
+ */
+function brokerManagedElsewhere(config) {
+  return !!(config && config.secrets && config.secrets.broker_managed_elsewhere);
+}
+
 // A connection per operation rather than one held open. These happen at login and at permission
 // changes - rare, and never in a hot path - and a short-lived connection cannot go stale, which on
 // a Pi that sleeps or loses its network it otherwise would.
@@ -69,6 +89,15 @@ export function syncUser(db, config, user, cb) {
     reportUnavailable(`user ${user.username}`, err);
     return cb(err);
   }
+  // Borrowing another server's broker: the account is already there, so derive and hand it over
+  // without trying to reach a plugin this server has no credential for.
+  if (brokerManagedElsewhere(config)) {
+    return cb(null, {
+      username: names.userClient(user.username),
+      password: derivePassword(userSecret, user.username, user.hashed_password),
+      brokerUpdated: false,
+    });
+  }
   // Nothing to hand a browser on a server that has never driven the plugin: the account cannot
   // exist, so the dashboard says live data is unavailable rather than failing to connect with a
   // credential that was never real. See dynsecConfigured.
@@ -136,7 +165,7 @@ export function syncLoggers(config, orgs, cb) {
   const creds = {};
   // No credentials at all means the logger falls back to the organization's shared password, which
   // works until S8. That is the right answer whenever an account of its own could not exist.
-  if (!userSecret || !dynsecConfigured(config)) {
+  if (!userSecret || (!dynsecConfigured(config) && !brokerManagedElsewhere(config))) {
     reportUnavailable('logger accounts',
       new Error(userSecret ? 'No dynsec_admin_user in config.d/secrets.yaml'
                            : 'No user_secret in config.d/secrets.yaml'));
@@ -145,6 +174,10 @@ export function syncLoggers(config, orgs, cb) {
   for (const org of orgs) {
     creds[org] = { username: names.loggerClient(org), password: deriveLoggerPassword(userSecret, org) };
   }
+  // Same as for a user: the accounts exist, this server just did not make them. Two loggers on one
+  // account is fine - mqtt.js generates a client id per connection, and MQTT evicts by client id,
+  // not by user - so a laptop recording alongside production does not disturb it.
+  if (brokerManagedElsewhere(config)) return cb(null, creds);
   withDynsec(config, (cerr, dynsec, done) => {
     // Configured but not answering: the accounts are almost certainly there from an earlier run,
     // so the derived credentials are handed over and the logger reconnects when the broker returns.

package/package.json

@@ -12,8 +12,8 @@
     "cookie-parser": "^1.4.7",
     "express": "^5.2.1",
     "express-session": "^1.19.0",
-    "frugal-iot-client": "^2.0.0",
-    "frugal-iot-logger": "^2.0.0",
+    "frugal-iot-client": "^2.0.1",
+    "frugal-iot-logger": "^2.0.1",
     "hash-wasm": "^4.12.0",
     "js-yaml": "^4.2.0",
     "morgan": "^1.11.0",
@@ -83,5 +83,5 @@
     "check-files": "node scripts/check-published-files.js"
   },
   "type": "module",
-  "version": "2.0.0"
+  "version": "2.0.1"
 }

package/public/service-worker.js

@@ -6,7 +6,7 @@
 // it, an installed browser keeps serving the files it already cached - and since the list now covers
 // the dashboard's own code, that means old JS behind freshly fetched HTML. Any edit to the list
 // needs this bumped, or it does not reach anyone who already has the old one.
-const CACHE_NAME = 'frugal-iot-cache-2.0.0';
+const CACHE_NAME = 'frugal-iot-cache-2.0.1';
 /*
  * Everything the app needs, so that a phone on a poor or expensive link fetches it once.
  *

package/scripts/check-schema.js

@@ -25,7 +25,8 @@
  *    is what decides how many decimals a reading is shown to; without it the UI guesses.
  *
  *  - A numeric topic with no "units". Often right - plenty of readings are dimensionless - but
- *    worth a look, because the UI has nothing to put after the number.
+ *    worth a look, because the UI has nothing to put after the number. The topics that are
+ *    deliberately dimensionless are listed in UNITS_EXEMPT below, and are not mentioned.
  *
  * And, separately, things that are never deliberate and so count as errors:
  *
@@ -35,8 +36,13 @@
  * Usage:
  *   node scripts/check-schema.js                     # checks ./config.d/schema
  *   node scripts/check-schema.js <dir> [<dir> ...]   # each dir holding topics.yaml and modules.yaml
+ *   node scripts/check-schema.js -q <dir>            # say nothing unless there is something to say
  *   node scripts/check-schema.js --resolve <otakey>  # which devices.yaml entry that OTA key uses
  *
+ * -q (--quiet) prints nothing at all when a schema is clean, so it can run from a release script
+ * without burying the one line that matters. Warnings and errors print exactly as they otherwise
+ * would, and the exit code is the same either way.
+ *
  * Exits 0 for warnings, so it can run from a prerelease script without blocking a release for
  * something that may well be deliberate. Exits 1 for an error, or if it could not read a schema.
  * Both callers guard it with "|| true", so neither blocks on either.
@@ -51,6 +57,17 @@ import path from 'path';
 const TYPES_LOGGED_BY_DEFAULT = ['float', 'int', 'bool'];
 // The types with decimals to decide, and so the ones that need a width
 const NUMERIC_TYPES = ['float', 'exponential'];
+/*
+ * Topics that are dimensionless on purpose, so "no units" is the right answer rather than
+ * something to look at. Without this the same three warnings appear at every release, which is
+ * exactly how a real one comes to be skimmed past. Add a name here only once you are sure the
+ * reading genuinely has no unit - not because you have not decided yet.
+ */
+const UNITS_EXEMPT = [
+  'controlfloat',   // a control's value, in whatever the thing it controls is measured in
+  'hdop',           // dilution of precision - a ratio
+  'loadcell',       // raw ADC counts until a calibration turns them into grams
+];
 
 function loadYaml(file) {
   if (!existsSync(file)) return null;
@@ -111,7 +128,7 @@ function checkSchemaDir(dir) {
           + ` characters before any decimal point, so values will overflow it`);
       }
     }
-    if (hasDecimals && (topic.units === undefined)) {
+    if (hasDecimals && (topic.units === undefined) && !UNITS_EXEMPT.includes(name)) {
       warnings.push(`topic "${name}" has no "units" - fine if it is dimensionless, but the UI has`
         + ` nothing to put after the number`);
     }
@@ -187,6 +204,7 @@ function resolveDeviceEntry(devices, otakey) {
 }
 
 const argv = process.argv.slice(2);
+const quiet = argv.includes('-q') || argv.includes('--quiet');
 const resolveAt = argv.indexOf('--resolve');
 if (resolveAt !== -1) {
   const otakey = argv[resolveAt + 1];
@@ -201,7 +219,8 @@ if (resolveAt !== -1) {
   process.exit(0);
 }
 
-const dirs = argv.length ? argv : ['config.d/schema'];
+const named = argv.filter((a) => !a.startsWith('-'));
+const dirs = named.length ? named : ['config.d/schema'];
 let unreadable = 0;
 let totalWarnings = 0;
 let totalErrors = 0;
@@ -214,7 +233,8 @@ for (const dir of dirs) {
     continue;
   }
   if (!warnings.length && !errors.length) {
-    console.log(`${dir}: nothing to report`);
+    // Under -q a clean schema says nothing, so whatever a release script does print is a finding
+    if (!quiet) console.log(`${dir}: nothing to report`);
     continue;
   }
   if (warnings.length) {

package/scripts/copy-schema-to-examples.zsh

@@ -8,16 +8,31 @@
 # schema is maintained.
 #
 # Usage:
-#   scripts/copy-schema-to-examples.zsh [<frugal-iot-logger directory>]
+#   scripts/copy-schema-to-examples.zsh [-q|--quiet] [<frugal-iot-logger directory>]
 #
 # With no argument it looks for frugal-iot-logger beside this checkout, which is how these are
 # developed. Says what it did and changes nothing else; committing is up to you.
+#
+# -q (--quiet) names only the files it changed. A run that copies nothing says nothing, so from a
+# release script the output is the list of files you now have to commit.
 
 set -euo pipefail
 
+QUIET=0
+ARGS=()
+for arg in "$@"; do
+  case "$arg" in
+    -q|--quiet) QUIET=1 ;;
+    *)          ARGS+=("$arg") ;;
+  esac
+done
+
+# Progress, as opposed to a change or a problem: silent under -q
+say() { (( QUIET )) || print -r -- "$@" }
+
 HERE="${0:A:h:h}"                     # The frugal-iot-server checkout this script is in
 SCHEMA="${HERE}/config.d/schema"
-LOGGER="${1:-${HERE:h}/frugal-iot-logger}"
+LOGGER="${ARGS[1]:-${HERE:h}/frugal-iot-logger}"
 
 if [[ ! -d "$LOGGER" ]]; then
   echo "  No frugal-iot-logger checkout at ${LOGGER}, so nothing copied."
@@ -44,7 +59,7 @@ COPIED=0
 for d in $DIRS; do
   for f in $FILES; do
     if cmp -s "${SCHEMA}/${f}" "${d}/${f}"; then
-      echo "  same     ${d#${LOGGER}/}/${f}"
+      say "  same     ${d#${LOGGER}/}/${f}"
     else
       cp "${SCHEMA}/${f}" "${d}/${f}"
       echo "  updated  ${d#${LOGGER}/}/${f}"

package/scripts/prerelease.zsh

@@ -4,6 +4,18 @@
 # afterwards. Run it, read what it says, then publish:
 #
 #   npm run prerelease
+#   zsh scripts/prerelease.zsh --release-version 2.1.0
+#   zsh scripts/prerelease.zsh -q
+#
+# -q (--quiet) prints only findings and changes, so a clean run prints nothing at all. That is what
+# the release script uses; run it without -q to see each check confirm itself.
+#
+# --release-version says which version this release is about to publish everywhere, and is what the
+# release script in frugal-iot-demo passes. Without it, "requires a version nobody has published"
+# is a blocker, because that is what it means on an ordinary day. During a release it is the normal
+# state of affairs - the logger and client are about to be published at that version and have not
+# been yet - so with it, a range that only the version being released satisfies is reported and
+# allowed, and CACHE_NAME is set to that version rather than to whatever is on npm today.
 #
 # It checks the sensor schema and copies it into the examples that ship with frugal-iot-logger, sets
 # the service worker's cache version, and looks for the package being wired to a checkout on this
@@ -21,27 +33,52 @@
 
 set -euo pipefail
 
+RELEASE_VERSION=""
+QUIET=0
+while (( $# )); do
+  case "$1" in
+    --release-version) RELEASE_VERSION="${2:-}"; shift 2 || true ;;
+    --release-version=*) RELEASE_VERSION="${1#*=}"; shift ;;
+    -q|--quiet) QUIET=1; shift ;;
+    -h|--help) sed -n '2,34p' "$0" | sed 's/^# \?//'; exit 0 ;;
+    *) print -u2 "Unknown option: $1 (try --release-version <version>, -q, or -h)"; exit 2 ;;
+  esac
+done
+
+# A check confirming itself, as opposed to a finding or a change: silent under -q
+say() { (( QUIET )) || print -r -- "$@" }
+
+# Passed on to the scripts that take the same flag
+QARG=()
+(( QUIET )) && QARG=(-q)
+
 HERE="${0:A:h:h}"                     # The frugal-iot-server checkout this script is in
 SCHEMA="${HERE}/config.d/schema"
 
+# Does the version being released satisfy this range? Only asked when --release-version was given.
+satisfied_by_release() {
+  [[ -n "$RELEASE_VERSION" ]] || return 1
+  node -e "process.exit(require('semver').satisfies('${RELEASE_VERSION}', process.argv[1]) ? 0 : 1)" "$1" 2>/dev/null
+}
+
 cd "$HERE"
 
 BLOCKERS=()   # never right in a published package - this script exits non-zero for these
 WARNINGS=()   # probably worth doing something about, but yours to judge
 
-echo "=== Checking ${SCHEMA} ==="
-node scripts/check-schema.js config.d/schema || true
+say "=== Checking ${SCHEMA} ==="
+node scripts/check-schema.js "${QARG[@]}" config.d/schema || true
 
-echo
-echo "=== Copying the schema into the frugal-iot-logger examples ==="
-zsh scripts/copy-schema-to-examples.zsh
+say
+say "=== Copying the schema into the frugal-iot-logger examples ==="
+zsh scripts/copy-schema-to-examples.zsh "${QARG[@]}"
 
 # ---- Is this package wired to a checkout on this machine? ----
 # Developing the server, logger and client together means pointing the server at sibling checkouts,
 # and every way of doing that is invisible once you have got used to it. Published, each one either
 # breaks every install outright or ships code nobody else has.
-echo
-echo "=== Checking nothing is wired to a local checkout ==="
+say
+say "=== Checking nothing is wired to a local checkout ==="
 
 # 1. "file:" or "link:" dependencies - these would make "npm install frugal-iot-server" fail for
 #    everybody, because the path they name does not exist on anyone else's machine
@@ -59,7 +96,7 @@ if [[ -n "$LOCALDEPS" ]]; then
   print -r -- "$LOCALDEPS" | sed 's/^/  /'
   BLOCKERS+=("package.json names a dependency by path - publishing that breaks every install. Put the version range back.")
 else
-  echo "  package.json names no dependencies by path"
+  say "  package.json names no dependencies by path"
 fi
 
 # 2. npm link - node_modules/<pkg> being a symlink means what you have been testing is your
@@ -74,8 +111,9 @@ if (( ${#LINKED} )); then
   for l in $LINKED; do echo "  linked: $l"; done
   echo "  What you have been running is that checkout, not the published package."
   echo "  Publish the linked package first, then require the new version here."
+  WARNINGS+=("node_modules links to a checkout (${(j:, :)LINKED}) - what the tests exercised is that checkout, not the published package.")
 else
-  echo "  node_modules holds published packages, not links to checkouts"
+  say "  node_modules holds published packages, not links to checkouts"
 fi
 
 # 3. The development alternatives inside the source, which are switched by editing rather than by
@@ -83,12 +121,12 @@ fi
 if grep -qE '^\s*import .*from\s+"\.\./frugal-iot-logger' frugal-iot-server.js 2>/dev/null; then
   BLOCKERS+=("frugal-iot-server.js is importing the logger from ../frugal-iot-logger - comment that line out and uncomment the \"frugal-iot-logger\" one.")
 else
-  echo "  frugal-iot-server.js imports the logger as a package, not by path"
+  say "  frugal-iot-server.js imports the logger as a package, not by path"
 fi
 if grep -qE '^\s*(htmldir|nodemodulesdir):\s*\.\./' config.d/server.yaml 2>/dev/null; then
   BLOCKERS+=("config.d/server.yaml points htmldir or nodemodulesdir at ../frugal-iot-client - that is the development setting and ships to everyone who installs.")
 else
-  echo "  config.d/server.yaml points at ./node_modules, not at a sibling checkout"
+  say "  config.d/server.yaml points at ./node_modules, not at a sibling checkout"
 fi
 
 # ---- Are the versions this requires actually available? ----
@@ -96,8 +134,8 @@ fi
 # on a Pi as well as here, and nothing in the repo shows it. Asking for one that is published but
 # not the newest is a lesser problem - a fresh install resolves to the newest anyway, but an
 # existing one is free to stay where it is, so your changes reach nobody who already has it.
-echo
-echo "=== Checking the versions this requires have been published ==="
+say
+say "=== Checking the versions this requires have been published ==="
 
 # Highest published version of $1 satisfying range $2, or empty if none - and it prints nothing and
 # exits 1 when nothing matches, which is an answer rather than a failure, hence the "|| true"
@@ -110,15 +148,19 @@ for p in frugal-iot-logger frugal-iot-client; do
   RANGE=$(node -e "console.log(require('./package.json').dependencies['$p'] || '')")
   if [[ -z "$RANGE" ]]; then continue; fi
   if ! LATEST=$(npm view "$p" version 2>/dev/null); then
-    echo "  $p: could not ask npm (offline?) - requires ${RANGE}, check it yourself"
+    say "  $p: could not ask npm (offline?) - requires ${RANGE}, check it yourself"
     continue
   fi
   MATCH=$(published_match "$p" "$RANGE")
-  if [[ -z "$MATCH" ]]; then
-    echo "  $p: requires ${RANGE}, but the newest published is ${LATEST}"
+  if [[ -z "$MATCH" ]] && satisfied_by_release "$RANGE"; then
+    # Mid-release: this range was raised to the version being published, and $p has not been
+    # published at it yet. Expected, and the publishing step is about to deal with it.
+    say "  $p: requires ${RANGE}, awaiting the ${RELEASE_VERSION} publish (newest so far is ${LATEST})"
+  elif [[ -z "$MATCH" ]]; then
+    say "  $p: requires ${RANGE}, but the newest published is ${LATEST}"
     BLOCKERS+=("Nothing published satisfies ${p}@${RANGE} (newest is ${LATEST}) - publish ${p} first, or every \"npm install frugal-iot-server\" fails.")
   elif [[ "$MATCH" != "$LATEST" ]]; then
-    echo "  $p: requires ${RANGE} -> ${MATCH}, but ${LATEST} is published"
+    say "  $p: requires ${RANGE} -> ${MATCH}, but ${LATEST} is published"
     WARNINGS+=("${p}@${RANGE} resolves to ${MATCH}, not the newest published ${LATEST} - bump the range if you meant installs to pick up ${LATEST}.")
   else
     # The range allows the newest, but if its floor is older then an existing install can sit on
@@ -126,10 +168,10 @@ for p in frugal-iot-logger frugal-iot-client; do
     FLOOR=${RANGE#[\^~>=]}
     FLOOR=${FLOOR#=}
     if [[ -n "$FLOOR" && "$FLOOR" != "$LATEST" ]]; then
-      echo "  $p: requires ${RANGE}, newest published ${LATEST} (an existing install may stay on ${FLOOR})"
+      say "  $p: requires ${RANGE}, newest published ${LATEST} (an existing install may stay on ${FLOOR})"
       WARNINGS+=("${p} requires ${RANGE}, so an install that already has ${FLOOR} is not obliged to move to ${LATEST}. Raise the range to ^${LATEST} if this release needs it.")
     else
-      echo "  $p: requires ${RANGE}, newest published ${LATEST} - in step"
+      say "  $p: requires ${RANGE}, newest published ${LATEST} - in step"
     fi
   fi
 done
@@ -139,27 +181,33 @@ done
 # installed PWA fetch it again. Left behind, people keep running the client they already had, which
 # looks exactly like a release that did nothing. It tracks the client version, so set it from
 # whichever published client this release will actually resolve to.
-echo
-echo "=== Checking the service worker's cache version ==="
+say
+say "=== Checking the service worker's cache version ==="
 SW=public/service-worker.js
 CLIENT_RANGE=$(node -e "console.log(require('./package.json').dependencies['frugal-iot-client'] || '')")
 if [[ ! -f "$SW" ]]; then
-  echo "  no ${SW} here"
+  say "  no ${SW} here"
 elif [[ -z "$CLIENT_RANGE" ]]; then
-  echo "  package.json has no frugal-iot-client dependency to match against"
+  say "  package.json has no frugal-iot-client dependency to match against"
 else
   SW_VERSION=$(sed -n "s/^const CACHE_NAME = 'frugal-iot-cache-\([^']*\)'.*/\1/p" "$SW" | head -1)
   CLIENT_LATEST=$(npm view frugal-iot-client version 2>/dev/null || true)
   CLIENT_WANTED=$(published_match frugal-iot-client "$CLIENT_RANGE")
+  # Mid-release the client this ships with is the one about to be published, not the newest on npm
+  # today, so follow that instead - otherwise CACHE_NAME would be set back to last month's version.
+  if satisfied_by_release "$CLIENT_RANGE"; then
+    CLIENT_WANTED="$RELEASE_VERSION"
+    CLIENT_LATEST="$RELEASE_VERSION"
+  fi
   if [[ -z "$SW_VERSION" ]]; then
-    echo "  could not find a CACHE_NAME line of the form frugal-iot-cache-<version> in ${SW}"
+    say "  could not find a CACHE_NAME line of the form frugal-iot-cache-<version> in ${SW}"
     WARNINGS+=("Could not read CACHE_NAME from ${SW} - check by hand that it changes when the client does.")
   elif [[ -z "$CLIENT_WANTED" ]]; then
-    echo "  CACHE_NAME says ${SW_VERSION}, and nothing published satisfies ${CLIENT_RANGE}"
+    say "  CACHE_NAME says ${SW_VERSION}, and nothing published satisfies ${CLIENT_RANGE}"
     BLOCKERS+=("Cannot set CACHE_NAME: no published frugal-iot-client satisfies ${CLIENT_RANGE}.")
   else
     if [[ "$SW_VERSION" == "$CLIENT_WANTED" ]]; then
-      echo "  CACHE_NAME is ${SW_VERSION}, matching the client this release installs"
+      say "  CACHE_NAME is ${SW_VERSION}, matching the client this release installs"
     else
       # An in-place edit rather than a warning: there is only one right answer, and a stale cache
       # name is invisible until somebody's browser keeps serving them last month's dashboard
@@ -174,16 +222,19 @@ else
 fi
 
 # ---- What would and would not be published ----
-echo
-echo "=== Checking what the package would contain ==="
-if ! node scripts/check-published-files.js; then
+say
+say "=== Checking what the package would contain ==="
+if ! FILESOUT=$(node scripts/check-published-files.js 2>&1); then
+  print -r -- "$FILESOUT"
   BLOCKERS+=("Something git ignores would be published - see above.")
+else
+  say "$FILESOUT"
 fi
 
-echo
-echo "=== Reminders ==="
-echo "  - Is the version in package.json the one you mean to publish?  $(node -e 'console.log(require("./package.json").version)')"
-echo "  - Nothing here is committed or published - do that yourself."
+say
+say "=== Reminders ==="
+say "  - Is the version in package.json the one you mean to publish?  $(node -e 'console.log(require("./package.json").version)')"
+say "  - Nothing here is committed or published - do that yourself."
 
 if (( ${#WARNINGS} )); then
   echo