frugal-iot-server 0.3.8 → 2.0.0

package/config.d/schema/topics.yaml

@@ -41,7 +41,7 @@ altitude:
   min:      -500
   max:      9000
   width:    6
-  color:    black
+  color:    "#000000"
   log:      true
   duplicates:
     significantdate: 900000
@@ -67,7 +67,7 @@ aqi:
   display:  bar
   min:      0
   max:      5
-  color:    purple
+  color:    "#800080"
   rw:       r
   graphable:  true
   log:      true
@@ -81,7 +81,7 @@ aqi500:
   display:  bar
   min:      0
   max:      500
-  color:    brown
+  color:    "#a52a2a"
   rw:       r
   graphable:  true
   log:      true
@@ -95,7 +95,7 @@ battery:
   display:  text
   min:      3000
   max:      5000
-  color:    green
+  color:    "#008000"
   rw:       r
   graphable:  true
   log:      true
@@ -120,7 +120,7 @@ bus:
   min:      0
   max:      32
   width:    5
-  color:    green
+  color:    "#008000"
   rw:       r
   log:      true
   units:    V
@@ -134,7 +134,7 @@ button:
   type:     bool
   log:      true
   display:  toggle
-  color:    black
+  color:    "#000000"
   rw:       r
   # A press changes the value, so it is still recorded - this only drops the repeats in between
   duplicates:
@@ -155,7 +155,7 @@ controlfloat:
   type:     float
   log:      false
   display:  text
-  color:    black
+  color:    "#000000"
   wireable: true
   rw:       w
   retain:   true
@@ -166,7 +166,7 @@ controlint:
   type:     int
   log:      false
   display:  text
-  color:    black
+  color:    "#000000"
   wireable: true
   rw:       w
   retain:   true
@@ -177,7 +177,7 @@ controltext:
   type:     text
   log:      false
   display:  text
-  color:    black
+  color:    "#000000"
   wireable: true
   rw:       w
   retain:   true
@@ -186,7 +186,7 @@ controlintoggle:
   type:     bool
   log:      false
   display:  toggle
-  color:    black
+  color:    "#000000"
   rw:       w
   wireable: false
 controlouttoggle:
@@ -194,7 +194,7 @@ controlouttoggle:
   type:     bool
   log:      true
   display:  toggle
-  color:    black
+  color:    "#000000"
   rw:       r
   wireable: true
   # Read-only bool, so logged by default - match the "out" and "on" topics it stands in for
@@ -209,7 +209,7 @@ course:
   min:      0
   max:      360
   width:    5
-  color:    black
+  color:    "#000000"
   log:      true
   duplicates:
     significantdate: 900000
@@ -225,7 +225,7 @@ current:
   min:      -3200
   max:      3200
   width:    7
-  color:    red
+  color:    "#ff0000"
   rw:       r
   log:      true
   units:    mA
@@ -250,7 +250,7 @@ do:
   min:      0
   max:      20
   width:    5
-  color:    blue
+  color:    "#0000ff"
   rw:       r
   log:      true
   units:    mg/L
@@ -265,7 +265,7 @@ eco2:
   display:  bar
   min:      300
   max:      900
-  color:    brown
+  color:    "#a52a2a"
   log:      true
   units:    ppm
   duplicates:
@@ -273,6 +273,18 @@ eco2:
     significantvalue: 5
   rw:       r
   graphable:  true
+# Unix time in seconds, as the node's clock currently reads it - published by the "time" module so
+# the client can see how far a node has drifted. A clock reading rather than a measurement: logging
+# it would fill the CSVs with a number that only ever counts up.
+epoch:
+  leaf:     epoch
+  name:     Epoch
+  type:     int
+  log:      false
+  display:  text
+  color:    "#000000"
+  rw:       r
+  units:    s
 gas:
   leaf:     gas
   name:     Gas Resistance
@@ -281,7 +293,7 @@ gas:
   min:      0
   max:      500
   width:    5
-  color:    green
+  color:    "#008000"
   rw:       r
   log:      true
   units:    kOhm
@@ -297,7 +309,7 @@ hdop:
   min:      0
   max:      50
   width:    5
-  color:    black
+  color:    "#000000"
   log:      false
   duplicates:
     significantdate: 900000
@@ -312,7 +324,7 @@ humidity:
   min:      0
   max:      100
   width:    5
-  color:    blue
+  color:    "#0000ff"
   log:      true
   duplicates:
     significantdate: 900000
@@ -341,7 +353,7 @@ language_code:
   type:     text
   log:      false
   display:  text
-  color:    black
+  color:    "#000000"
   rw:       r
 lastseen:
   leaf:     lastseen
@@ -359,7 +371,7 @@ latitude:
   min:      -90
   max:      +90
   width:    9
-  color:    black
+  color:    "#000000"
   log:      true
   duplicates:
     significantdate: 900000
@@ -375,7 +387,7 @@ load:
   min:      0
   max:      32
   width:    5
-  color:    green
+  color:    "#008000"
   rw:       r
   log:      true
   units:    V
@@ -391,7 +403,7 @@ loadcell:
   min:      0
   max:      65000
   width:    7
-  color:    yellow
+  color:    "#b8860b"
   log:      true
   duplicates:
     significantdate: 900000
@@ -407,7 +419,7 @@ longitude:
   min:      -180
   max:      +180
   width:    10
-  color:    black
+  color:    "#000000"
   log:      true
   duplicates:
     significantdate: 900000
@@ -423,7 +435,7 @@ lux:
   min:      0
   max:      65000
   width:    5
-  color:    yellow
+  color:    "#b8860b"
   log:      true
   rw:       r
   duplicates:
@@ -436,7 +448,7 @@ manual:
   name:     Manual
   type:     bool
   display:  toggle
-  color:    black
+  color:    "#000000"
   rw:       r
   log:      true
   graphable:  true
@@ -452,7 +464,7 @@ message:
   type:     text
   log:      false
   display:  text
-  color:    black
+  color:    "#000000"
   wireable: true
   rw:       w
   retain:   true
@@ -464,7 +476,7 @@ mm:
   min:      0
   max:      500
   width:    5
-  color:    blue
+  color:    "#0000ff"
   rw:       r
   log:      true
   duplicates:
@@ -481,12 +493,23 @@ name:
   display:  text
   rw:       w
   retain:   true
+# How far the node is displaying from UTC, so it can show local time. Like epoch, a setting the node
+# reports back rather than anything measured, so not logged.
+offset:
+  leaf:     offset
+  name:     UTC Offset
+  type:     int
+  log:      false
+  display:  text
+  color:    "#000000"
+  rw:       r
+  units:    min
 on:
   leaf:     on
   name:     On
   type:     bool
   display:  toggle
-  color:    black
+  color:    "#000000"
   rw:       w
   log:      true
   duplicates:
@@ -498,7 +521,7 @@ out:
   name:     Out
   type:     bool
   display:  toggle
-  color:    black
+  color:    "#000000"
   rw:       r
   log:      true
   wireable: true
@@ -506,13 +529,33 @@ out:
   duplicates:
     significantdate: 900000
     significantvalue: 1
+percent:
+  # A 0-100% figure that is worth recording and graphing - state of charge, battery health. Unlike
+  # controlfloat it is read-only and logged, so it needs its own duplicate rule; [name] and the
+  # duplicate thresholds should be overridden per module.
+  leaf:     percent
+  name:     Percent
+  type:     float
+  display:  bar
+  min:      0
+  max:      100
+  width:    3
+  color:    "#008000"
+  rw:       r
+  wireable: false
+  log:      true
+  duplicates:
+    significantdate: 900000
+    significantvalue: 1
+  graphable:  true
+  units:    "%"
 position:
   leaf:     position
   name:     Position
   type:     text
   log:      false
   display:  text
-  color:    black
+  color:    "#000000"
   rw:       r
 power:
   leaf:     power
@@ -522,7 +565,7 @@ power:
   min:      0
   max:      100000
   width:    8
-  color:    orange
+  color:    "#ffa500"
   rw:       r
   log:      true
   units:    mW
@@ -539,7 +582,7 @@ pressure:
   max:      1100
   width:    6
   units:    hPa
-  color:    blue
+  color:    "#0000ff"
   rw:       r
   log:      true
   duplicates:
@@ -554,7 +597,7 @@ rain:
   min:      0
   max:      500
   width:    5
-  color:    blue
+  color:    "#0000ff"
   rw:       r
   log:      true
   duplicates:
@@ -570,7 +613,7 @@ satellites:
   min:      0
   max:      32
   width:    2
-  color:    black
+  color:    "#000000"
   log:      false
   duplicates:
     significantdate: 900000
@@ -585,7 +628,7 @@ select:
   log:      false
   min:      0
   display:  text
-  color:    black
+  color:    "#000000"
   wireable: false
   rw:       w
   retain:   true
@@ -597,7 +640,7 @@ shunt:
   min:      -320
   max:      320
   width:    6
-  color:    blue
+  color:    "#0000ff"
   rw:       r
   log:      true
   units:    mV
@@ -613,7 +656,7 @@ speed:
   min:      0
   max:      999
   width:    5
-  color:    black
+  color:    "#000000"
   log:      true
   duplicates:
     significantdate: 900000
@@ -628,7 +671,7 @@ soil:
   display:  bar
   min:      0
   max:      100
-  color:    brown
+  color:    "#a52a2a"
   rw:       r
   log:      true
   duplicates:
@@ -637,6 +680,22 @@ soil:
   graphable:  true
   calibrate:  true
   units:    "%"
+tank:
+  leaf:     tank
+  name:     Water Tank
+  type:     float
+  display:  bar
+  min:      0
+  max:      100
+  width:    5
+  color:    "#0000ff"
+  rw:       r
+  log:      true
+  duplicates:
+    significantdate: 900000
+    significantvalue: 1
+  graphable:  true
+  units:    "%"
 temperature:
   leaf:     temperature
   name:     Temperature
@@ -646,7 +705,7 @@ temperature:
   min:      0
   max:      50
   width:    4
-  color:    red
+  color:    "#ff0000"
   wireable: false
   log:      true
   units:    Cel
@@ -663,7 +722,7 @@ timeoff:
   type:     float
   log:      false
   display:  text
-  color:    black
+  color:    "#000000"
   rw:       w
   wireable: true
   units:    s
@@ -676,7 +735,7 @@ timeon:
   type:     float
   log:      false
   display:  text
-  color:    black
+  color:    "#000000"
   rw:       w
   wireable: true
   units:    s
@@ -687,7 +746,7 @@ tvoc:
   display:  bar
   min:      0
   max:      99
-  color:    green
+  color:    "#008000"
   rw:       r
   log:      true
   units:    ppb
@@ -703,7 +762,7 @@ ultrasonic:
   min:      0
   max:      7500
   width:    4
-  color:    blue
+  color:    "#0000ff"
   rw:       r
   log:      true
   units:    mm
@@ -718,8 +777,43 @@ utc_time:
   type:     text
   log:      false
   display:  text
-  color:    black
+  color:    "#000000"
+  rw:       r
+voltage:
+  # A DC voltage read through a resistor divider - see Sensor_Voltage in the library. Millivolts,
+  # like `battery`, but without that topic's single-lithium-cell range or its special-casing in
+  # the UX. [name, min, max] should be overridden per module.
+  leaf:     voltage
+  name:     Voltage
+  type:     int
+  display:  text
+  min:      0
+  max:      30000
+  color:    "#800080"
   rw:       r
+  graphable:  true
+  log:      true
+  units:    mV
+  duplicates:
+    significantdate: 900000
+    significantvalue: 100
+volts:
+  leaf:     volts
+  name:     Volts
+  type:     float
+  display:  slider
+  rw:       w
+  min:      0
+  max:      3.3
+  width:    4
+  color:    "#800080"
+  units:    V
+  wireable: true
+  log:      true
+  duplicates:
+    significantdate: 900000
+    significantvalue: 0.05
+  graphable:  true
 wifibars:
   leaf:     wifibars
   name:     WiFi
@@ -728,7 +822,7 @@ wifibars:
   display:  text
   min:      0
   max:      4
-  color:    blue
+  color:    "#0000ff"
   rw:       r
   # Logged, because an int that is read-only is logged unless told otherwise. Only 5 values, so
   # any change is worth recording and nothing else is
@@ -741,5 +835,5 @@ wifissid:
   type:     text
   log:      false
   display:  text
-  color:    blue
+  color:    "#0000ff"
   rw:       r

package/extras/aclfile

@@ -0,0 +1,38 @@
+# Frugal IoT access control. Named by acl_file in extras/mosquitto.conf.
+#
+# DENY BY DEFAULT: the moment the broker names an acl_file, an account with no rule here can still
+# log in but can reach no topic at all. So adding an account means adding its rule in the same
+# change - see scripts/addbridge-prod.zsh, which appends one per Pi.
+#
+# This file is read AFTER mosquitto drops privileges, so it must be readable by the mosquitto user.
+# Install it as mosquitto:mosquitto mode 600 - NOT root-owned 644. Mosquitto 2.0.21 warns
+# "owner is not mosquitto ... Future versions will refuse to load this file" and separately
+# "has world readable permissions", so 644 root works today and is on notice to stop working. The
+# rule is the same one the password file follows: owned by whoever opens it.
+
+# Each organization reaches only its own topic tree. "%u" is the connecting account name, and the
+# first element of every Frugal IoT topic is the organization id - so this is what the nodes,
+# dashboards and the server's own logger already do. It just stops being optional.
+pattern readwrite %u/#
+
+# Whether a bridge is up. With "notifications" left on (the default) a bridge publishes a retained
+# 1 or 0 to $SYS/broker/connection/<remote_clientid>/state, and that is what frugal-iot-diagnostic
+# reads to say whether a site is connected.
+#
+# Needed because "pattern readwrite %u/#" does not match $SYS, so without this line the diagnostic
+# reports every bridge as never having connected - measured, not guessed. This is the narrowest rule
+# that fixes it: it exposes which bridges exist and whether each is up, and nothing else. Reading
+# all of $SYS ("pattern read $SYS/#") would also work and would hand every account the broker's
+# client counts, subscription counts and traffic totals, which on a multi-organization broker tells
+# each organization about the others.
+#
+# Mosquitto logs "Warning: ACL pattern ... does not contain '%c' or '%u'" for the line below, on
+# every start and reload. It is harmless and expected: a "pattern" line is the only way to say
+# "every authenticated account", because a bare "topic" line applies to anonymous clients only.
+# Do not "fix" it by changing it to a topic line - that would silently grant nothing.
+pattern read $SYS/broker/connection/+/state
+
+# Bridge connections are exempt from this file on their local broker - a bridge with no
+# local_username still carries traffic in both directions with the rules above in force (measured on
+# mosquitto 2.0.21). So a Pi's own bridge needs nothing here; it is the far end, on production, that
+# needs a rule, and addbridge-prod.zsh writes it.

package/extras/mosquitto.conf

@@ -9,6 +9,35 @@ protocol websockets
 # directory has to be writable by that same user. See INSTALLATION.md step A5.
 password_file /var/lib/mosquitto/passwords
 
+# Which topics each account may reach. Without this, every account that can log in can publish and
+# subscribe anywhere on the broker, across all organizations - so an organization's own password,
+# which is handed to every node and every logged-in browser, would reach every other organization's
+# data too.
+#
+# DENY BY DEFAULT: naming this file stops any account that has no rule in it. The shipped file
+# grants each organization its own topic tree, which is all the nodes, dashboards and logger use, so
+# nothing should notice. But an account added by hand needs a rule adding by hand.
+#
+# Installed from extras/aclfile - which must be in place BEFORE this line takes effect, or the
+# broker will not start.
+acl_file /etc/mosquitto/aclfile
+
+# Accounts and permissions the server can create and remove while the broker is running, over
+# $CONTROL/dynamic-security/v1. This is what lets each user and each node have its own credential
+# rather than sharing the organization's - see SECURITY-REVIEW.md.
+#
+# The plugin's path is set by the installer, because it varies by distribution: Debian and
+# Raspberry Pi OS put it in /usr/lib/<architecture>/, others in a mosquitto/ subdirectory.
+#
+# Both files must exist before this takes effect or the broker will not start. The state file is
+# created by "mosquitto_ctrl dynsec init", which install-pi.sh does.
+#
+# Plugin checks run BEFORE the built-in password_file and acl_file checks, and an account dynsec
+# does not know is passed through to them - measured on 2.0.21, and what makes it possible to move
+# accounts across one at a time rather than all at once.
+plugin PLUGIN_PATH_SET_BY_INSTALLER
+plugin_opt_config_file /var/lib/mosquitto/dynamic-security.json
+
 # A node on a weak WiFi signal reconnects often, and each connection and disconnection would be a
 # line in /var/log/mosquitto/mosquitto.log. On an SD card those writes wear the card out, and
 # nothing is lost by dropping them, since the readings themselves are recorded by the server.

package/frugal-iot-createdb.sql

@@ -20,12 +20,93 @@ CREATE TABLE IF NOT EXISTS `users` (
   `email` TEXT,
   `phone` TEXT
 );
+-- `project` scopes a permission to one project of an organization. Empty string means the whole
+-- organization, which is how every row created before this column existed behaves.
+--
+-- NOT NULL DEFAULT '' rather than nullable, and that matters: SQLite treats NULLs as DISTINCT in a
+-- UNIQUE constraint, so with a nullable column `(2,'READ','dev',NULL)` could be inserted twice and
+-- the constraint that used to prevent duplicate organization-wide rows would silently stop working.
+-- An empty string compares equal to itself, so uniqueness holds. It also keeps the queries simple:
+-- `project = ?` everywhere, never `IS NULL`.
 CREATE TABLE IF NOT EXISTS `permissions` (
   `id` INTEGER NOT NULL,
   `capability` TEXT NOT NULL,
   `org` TEXT NOT NULL,
-  UNIQUE(`id`, `capability`, `org`)
+  `project` TEXT NOT NULL DEFAULT '',
+  UNIQUE(`id`, `capability`, `org`, `project`)
+);
+-- One row per node that has enrolled, holding the broker credential it was issued.
+--
+-- The password is stored, not derived: a node keeps its copy in LittleFS and cannot recompute
+-- anything, so deriving it from a secret would mean that losing the secret strands the whole fleet.
+-- Storing it here is also what makes "the database is the source of truth, and dynsec can be
+-- rebuilt from it" true for nodes as well as users.
+--
+-- No last_seen column on purpose: the logger already tracks that in memory and serves it through
+-- reportNodes(), and writing it here would be a database write per reading - the SD-card wear the
+-- logger's own buffering exists to avoid.
+--
+-- `lora` records whether the firmware was built with LoRaMesher, declared at enrolment. It decides
+-- whether the node joins <org>-gateways, which grants the broader publish a gateway cannot avoid
+-- needing (it republishes other nodes' readings under its own account). Scoped by what the build
+-- can do rather than by what it is doing at the time, because any node that sees WiFi can promote
+-- itself to gateway at runtime.
+CREATE TABLE IF NOT EXISTS `nodes` (
+  `org` TEXT NOT NULL,
+  `project` TEXT NOT NULL,
+  `nodeid` TEXT NOT NULL,
+  `password` TEXT NOT NULL,
+  `lora` INTEGER NOT NULL DEFAULT 0,
+  `enrolled_at` INTEGER NOT NULL,
+  UNIQUE(`org`, `project`, `nodeid`)
 );
+
+-- A Pi authorised to bridge into this server, and the token it pulls users with (SECURITY.md S11).
+--
+-- A bridge relays topics, not accounts, so a person who logs into this server has no account on the
+-- Pi's broker. GET /replica/:org lets the Pi fetch the logins and permissions for the organizations
+-- it hosts, so the same people can log in there - including when this server is unreachable, which
+-- is the reason the Pi exists.
+--
+-- Nothing derived from a password travels: the Pi authenticates the login itself against the
+-- replicated hash and derives its own broker credential from its own user_secret, so the two
+-- brokers issue different passwords for the same person and neither secret is shared.
+--
+-- One token per Pi per organization, so one site can be revoked without disturbing another. The
+-- token is a bearer credential for reading that organization's logins - not for changing anything.
+CREATE TABLE IF NOT EXISTS `bridges` (
+  `org` TEXT NOT NULL,
+  `site` TEXT NOT NULL,
+  `token` TEXT NOT NULL,
+  `created_at` INTEGER NOT NULL,
+  `last_pull` INTEGER,
+  UNIQUE(`org`, `site`)
+);
+
+-- An admin's decision about one node, for the Enrolled column on the dashboard's Nodes card
+-- (SECURITY.md S12). Two states are worth storing:
+--
+--   approved  the next enrolment request from this node id is accepted whatever secret it presents,
+--             and without having to prove it holds a credential - for a node whose secret was
+--             withdrawn, or that was flashed with none, and which cannot be reached physically.
+--             Consumed on use, so an approval admits one node once.
+--   denied    its broker account is deleted and enrolment is refused - the kill switch for a node
+--             publishing bad readings.
+--
+-- Keyed by (org, nodeid), NOT by project: a node states its own project, so keying on it would let
+-- a denied node return by claiming a different one.
+--
+-- Persisted, unlike the record of failed attempts, because a restart must not un-deny a node.
+CREATE TABLE IF NOT EXISTS `node_grants` (
+  `org` TEXT NOT NULL,
+  `project` TEXT NOT NULL DEFAULT '',
+  `nodeid` TEXT NOT NULL,
+  `state` TEXT NOT NULL,
+  `created_by` TEXT NOT NULL DEFAULT '',
+  `created_at` INTEGER NOT NULL,
+  UNIQUE(`org`, `nodeid`)
+);
+
 CREATE TABLE IF NOT EXISTS `projects` (
   `org` TEXT NOT NULL,
   `id` TEXT NOT NULL,