frugal-iot-server 0.3.5 → 0.3.7

package/README.md

@@ -17,7 +17,7 @@ data and database; npm keeps the software itself under it in `node_modules`.
 mkdir ~/frugal-iot
 cd ~/frugal-iot
 npm install frugal-iot-server
-npx frugal-iot-init
+npx --no frugal-iot-init
 ```
 `frugal-iot-init` copies in the configuration files, creates the `data`, `ota` and
 `config.d/organizations` directories, and creates the database. It never overwrites anything
@@ -27,24 +27,24 @@ Edit `config.yaml` and `config.d/mqtt.yaml` (which broker to talk to) if the def
 
 The database is created with a `superuser` account that has no password, so give it one:
 ```
-npx frugal-iot-setpassword superuser "<a-good-password>"
+npx --no frugal-iot-setpassword superuser "<a-good-password>"
 ```
 
 Then add an organization - this writes its yaml file into `config.d/organizations`, creates a
 login account of the same name with its permissions, and sets its password on the MQTT broker:
 ```
-npx frugal-iot-addorganization <org-id> <org-name> <email> <phone> <broker-password>
+npx --no frugal-iot-addorganization <org-id> <org-name> <email> <phone> <broker-password>
 ```
 
 That password is the one used by the nodes and the broker. It will also work for login, so it is
 recommended to give the login its own:
 ```
-npx frugal-iot-setpassword <org-id> <login-password>
+npx --no frugal-iot-setpassword <org-id> <login-password>
 ```
 
 Now start it:
 ```
-npx frugal-iot-server
+npx --no frugal-iot-server
 ```
 If its working correctly you should see something like
 ```
@@ -70,7 +70,7 @@ and receives data from nodes attached to it.
 
 Open a browser pointing at for example `localhost:8080` and you should see the UI.
 
-To upgrade later: `npm update frugal-iot-server` then `npx frugal-iot-init`.
+To upgrade later: `npm update frugal-iot-server` then `npx --no frugal-iot-init`.
 
 #### Developing the server, client or logger
 

package/config.d/logger.yaml

@@ -4,6 +4,22 @@
 # BUT this doesnt work yet - since there are permissions issues if login ot local server then get from production
 # url:  https://frugaliot.naturalinnovation.org/data
 url: /data
+
+# Report every message received on the console, and so into the systemd journal - one line per
+# message, which is one disk write per message. Off here because the usual installation runs from
+# an SD card, where writes wear the card out; turn it on while working out why a node's readings
+# are not appearing, and turn it back off afterwards. A server that does not have this setting at
+# all reports messages, so upgrading never quietens a machine that was talking before.
+verbose: false
+
+# How often the readings collected in memory are written out to the CSV files under data/, in
+# seconds. Writing each reading as it arrives is what wears an SD card out, so they are gathered
+# up and written in one go. The cost is that pulling the power loses up to this much data, since
+# what has not been written yet is only in memory; a clean shutdown or restart writes it out
+# first, and so does anything reading the data back over /data. 0, or leaving this out altogether,
+# writes each reading as it arrives, which is what versions before this one did.
+flushseconds: 300
+
 #TODO add info that server should use when launching logger
 #TODO add info that logger uses for its own configuraton
 #TODO see https://github.com/mitra42/frugal-iot-server/issues/7 and

package/config.d/schema/modules.yaml

@@ -7,11 +7,45 @@
 # topics: list of topics, each with leaf, optionally leaf_from if it should be copied, and optionally any fields of topics to be overridden
 # slot: if it is supposed to fit in a specific slot of frugaliot
 # insidefrugaliot: true if it should be hidden inside the Frugal_IoT drop-down on the name of a sensor, false or not set if it should be shown as a separate module
+aht20:
+  name: "AHT20"
+  topics:
+    - leaf: temperature
+    - leaf: humidity
+aht21:
+  name: "AHT21"
+  topics:
+    - leaf: temperature
+    - leaf: humidity
 battery:
   name: "Battery"
   insidefrugaliot: true
   topics:
     - leaf:   battery
+bme280:
+  name: "BME280"
+  topics:
+    - leaf: temperature
+    - leaf: humidity
+    - leaf: pressure
+      max:  1100
+      min:  300
+bme680:
+  name: "BME680"
+  topics:
+    - leaf: temperature
+    - leaf: humidity
+    - leaf: pressure
+      max:  1100
+      min:  300
+    - leaf: gas
+bmp280:
+  name: "BMP280"
+  topics:
+    - leaf: temperature
+    - leaf: pressure
+      max:  1100
+      min:  300
 button:
   name: Button
   topics:
@@ -112,6 +146,33 @@ ds18b20:
   topics:
     - leaf: ds18b20
       leaf_from: temperature
+do:
+  name: "Dissolved Oxygen"
+  topics:
+    - leaf: do
+ens160:
+  name: "ENS160"
+  topics:
+    # Compensation inputs, not readings. The ENS160 has no thermometer of its own: it takes
+    # ambient temperature and humidity from whatever is wired to it - an AHT21 sitting on the
+    # same breakout, or any other sensor on the node. So these come from controlfloat (wireable,
+    # rw: w, not logged) rather than from the temperature/humidity topics, which are read-only
+    # readings - whichever sensor actually measures them is the one that logs them.
+    - leaf: temperature
+      leaf_from: controlfloat
+      name: Temperature
+      max:  50
+      color: red
+    - leaf: humidity
+      leaf_from: controlfloat
+      name: Humidity
+      color: blue
+    - leaf: aqi
+    - leaf: tvoc
+    - leaf: eco2
+    - leaf: aqi500
+# Legacy - firmware before the ENS160/AHT21 split published everything under one module id.
+# Kept so already-deployed nodes still display; new nodes use `ens160` and `aht21` above.
 ensaht:
   name: "ENS AHT"
   topics:
@@ -120,7 +181,7 @@ ensaht:
     - leaf:  aqi
     - leaf:  tvoc
     - leaf:  eco2
-    - leaf:  agi500
+    - leaf:  aqi500
 frugal_iot:
   name: XXX
   topics:
@@ -138,6 +199,8 @@ gps:
     - leaf:     speed
     - leaf:     satellites
     - leaf:     hdop
+    - leaf:     position
+    - leaf:     utc_time
 health:
   name: System
   insidefrugaliot: true
@@ -149,6 +212,18 @@ ht:
   topics:
     - leaf: temperature
     - leaf: humidity
+ina219:
+  name: "Power Monitor"
+  topics:
+    - leaf: shunt
+    - leaf: bus
+    - leaf: current
+    - leaf: power
+    - leaf: load
+lcd:
+  name: LCD
+  topics:
+    - leaf: message
 ledbuiltin:
   name: LED
   slot: ledbuiltin
@@ -193,3 +268,7 @@ soil:
   name: Soil
   topics:
     - leaf: soil
+ultrasonic:
+  name: "Ultrasonic Distance"
+  topics:
+    - leaf: ultrasonic

package/config.d/schema/topics.yaml

@@ -17,6 +17,13 @@
 # graphable  true if can be displayed on a graph
 # wireable true,false, generally only  controls are wireable
 # wired not valid in this context, it will come from MQTT broker
+# log  whether readings of this topic are recorded to disk. Say so either way on every topic: with
+#   no setting at all, any float, int or bool with "rw: r" is recorded, which is easy to be
+#   surprised by. scripts/check-schema.js warns about a topic that does not say.
+# duplicates  when to bother recording a reading: significantvalue is how much it has to have moved
+#   since the last one recorded, significantdate how long (in milliseconds) since that one, and a
+#   reading that meets neither is dropped. Every topic with "log: true" wants a rule here - without
+#   one, every reading is written to the SD card as it arrives, which wears the card out.
 # Note battery gets special cased
 # See https://www.iana.org/assignments/senml/senml.xhtml for units
 altitude:
@@ -40,6 +47,11 @@ analog:
   rw:       r
   graphable:  true
   log:      true
+  # A raw ADC reading with no declared range, so this threshold is a guess - lower it if real
+  # changes are being missed, raise it if the graph is mostly noise
+  duplicates:
+    significantdate: 900000
+    significantvalue: 10
 aqi:
   leaf:     aqi
   name:     AQI
@@ -87,21 +99,43 @@ brightness:
   leaf:     brightness
   name:     Brightness
   type:     int
+  log:      false
   max:      255
   min:      0
   display:  slider
   rw:       w
+bus:
+  leaf:     bus
+  name:     Bus Voltage
+  type:     float
+  display:  text
+  min:      0
+  max:      32
+  color:    green
+  rw:       r
+  log:      true
+  units:    V
+  duplicates:
+    significantdate: 900000
+    significantvalue: 0.1
+  graphable:  true
 button:
   leaf:     button
   name:     Button
   type:     bool
+  log:      true
   display:  toggle
   color:    black
   rw:       r
+  # A press changes the value, so it is still recorded - this only drops the repeats in between
+  duplicates:
+    significantdate: 900000
+    significantvalue: 1
 color:
   leaf:     color
   name:     Color
   type:     color
+  log:      false
   display:  color
   rw:       w
 controlfloat:
@@ -109,6 +143,7 @@ controlfloat:
   min:      0
   max:      100
   type:     float
+  log:      false
   display:  text
   color:    black
   wireable: true
@@ -119,6 +154,7 @@ controlint:
   min:      0
   max:      65535
   type:     int
+  log:      false
   display:  text
   color:    black
   wireable: true
@@ -129,6 +165,7 @@ controltext:
   min:      0
   max:      100
   type:     text
+  log:      false
   display:  text
   color:    black
   wireable: true
@@ -137,6 +174,7 @@ controltext:
 controlintoggle:
   #[leaf, name] should be overridden
   type:     bool
+  log:      false
   display:  toggle
   color:    black
   rw:       w
@@ -144,10 +182,15 @@ controlintoggle:
 controlouttoggle:
   #[leaf, name] should be overridden
   type:     bool
+  log:      true
   display:  toggle
   color:    black
   rw:       r
   wireable: true
+  # Read-only bool, so logged by default - match the "out" and "on" topics it stands in for
+  duplicates:
+    significantdate: 900000
+    significantvalue: 1
 course:
   leaf:     course
   name:     Course
@@ -163,14 +206,45 @@ course:
   rw:       r
   graphable:  true
   units:    deg
+current:
+  leaf:     current
+  name:     Current
+  type:     float
+  display:  text
+  min:      -3200
+  max:      3200
+  color:    red
+  rw:       r
+  log:      true
+  units:    mA
+  duplicates:
+    significantdate: 900000
+    significantvalue: 1
+  graphable:  true
 description:
   leaf:     description
   slot:     description
   name:     Description
   type:     text
+  log:      false
   display:  text
   rw:       w
   retain:   true
+do:
+  leaf:     do
+  name:     Dissolved Oxygen
+  type:     float
+  display:  bar
+  min:      0
+  max:      20
+  color:    blue
+  rw:       r
+  log:      true
+  units:    mg/L
+  duplicates:
+    significantdate: 900000
+    significantvalue: 0.1
+  graphable:  true
 eco2:
   leaf:     eco2
   name:     eCO2
@@ -186,6 +260,21 @@ eco2:
     significantvalue: 5
   rw:       r
   graphable:  true
+gas:
+  leaf:     gas
+  name:     Gas Resistance
+  type:     float
+  display:  text
+  min:      0
+  max:      500
+  color:    green
+  rw:       r
+  log:      true
+  units:    kOhm
+  duplicates:
+    significantdate: 900000
+    significantvalue: 1
+  graphable:  true
 hdop:
   leaf:     hdop
   name:     HDOP
@@ -220,18 +309,21 @@ id:
   slot:     id
   name:     Node ID
   type:     text
+  log:      false
   display:  text
   rw:       r
 key:
   leaf:     key
   name:     Key
   type:     text
+  log:      false
   display:  text
   rw:       r
-languagecode:
-  leaf:     languagecode
+language_code:
+  leaf:     language_code
   name:     Language
   type:     text
+  log:      false
   display:  text
   color:    black
   rw:       r
@@ -240,6 +332,7 @@ lastseen:
   slot:     lastseen
   name:     Last Seen
   type:     text
+  log:      false
   display:  text
   rw:       r
 latitude:
@@ -257,6 +350,21 @@ latitude:
   rw:       r
   graphable:  true
   units:    lat
+load:
+  leaf:     load
+  name:     Load Voltage
+  type:     float
+  display:  text
+  min:      0
+  max:      32
+  color:    green
+  rw:       r
+  log:      true
+  units:    V
+  duplicates:
+    significantdate: 900000
+    significantvalue: 0.1
+  graphable:  true
 loadcell:
   leaf:     loadcell
   name:     Load Cell
@@ -321,6 +429,7 @@ message:
   min:      0
   max:      100
   type:     text
+  log:      false
   display:  text
   color:    black
   wireable: true
@@ -346,6 +455,7 @@ name:
   slot:     name
   name:     Node Name
   type:     text
+  log:      false
   display:  text
   rw:       w
   retain:   true
@@ -374,6 +484,29 @@ out:
   duplicates:
     significantdate: 900000
     significantvalue: 1
+position:
+  leaf:     position
+  name:     Position
+  type:     text
+  log:      false
+  display:  text
+  color:    black
+  rw:       r
+power:
+  leaf:     power
+  name:     Power
+  type:     float
+  display:  text
+  min:      0
+  max:      100000
+  color:    orange
+  rw:       r
+  log:      true
+  units:    mW
+  duplicates:
+    significantdate: 900000
+    significantvalue: 10
+  graphable:  true
 pressure:
   leaf:     pressure
   name:     Pressure
@@ -422,12 +555,28 @@ select:
   leaf:     select
   name:     Select
   type:     int
+  log:      false
   min:      0
   display:  text
   color:    black
   wireable: false
   rw:       w
   retain:   true
+shunt:
+  leaf:     shunt
+  name:     Shunt Voltage
+  type:     float
+  display:  text
+  min:      -320
+  max:      320
+  color:    blue
+  rw:       r
+  log:      true
+  units:    mV
+  duplicates:
+    significantdate: 900000
+    significantvalue: 1
+  graphable:  true
 speed:
   leaf:     speed
   name:     Speed
@@ -481,6 +630,7 @@ timeoff:
   min:      0
   max:      3600
   type:     float
+  log:      false
   display:  text
   color:    black
   rw:       w
@@ -492,6 +642,7 @@ timeon:
   min:      0
   max:      3600
   type:     float
+  log:      false
   display:  text
   color:    black
   rw:       w
@@ -512,19 +663,50 @@ tvoc:
     significantdate: 900000
     significantvalue: 1
   graphable:  true
+ultrasonic:
+  leaf:     ultrasonic
+  name:     Distance
+  type:     float
+  display:  bar
+  min:      0
+  max:      7500
+  color:    blue
+  rw:       r
+  log:      true
+  units:    mm
+  duplicates:
+    significantdate: 900000
+    significantvalue: 5
+  graphable:  true
+  calibrate:  true
+utc_time:
+  leaf:     utc_time
+  name:     UTC Time
+  type:     text
+  log:      false
+  display:  text
+  color:    black
+  rw:       r
 wifibars:
   leaf:     wifibars
   name:     WiFi
   type:     int
+  log:      true
   display:  text
   min:      0
   max:      4
   color:    blue
   rw:       r
+  # Logged, because an int that is read-only is logged unless told otherwise. Only 5 values, so
+  # any change is worth recording and nothing else is
+  duplicates:
+    significantdate: 900000
+    significantvalue: 1
 wifissid:
   leaf:     wifissid
   name:     SSID
   type:     text
+  log:      false
   display:  text
   color:    blue
   rw:       r

package/config.d/server.yaml

@@ -1,12 +1,45 @@
 port: 8080
-#Production mode
-nodemodulesdir: ./node_modules
+
+# Log every HTTP request. Off here because the usual installation runs from an SD card, where each
+# line logged is a write and writes wear the card out. Set it to true (or to a morgan format
+# string) while debugging. A server whose server.yaml does not mention morgan at all logs requests,
+# so this setting arriving in a new release never quietens a machine that was logging before.
+morgan: false
+
+# Where the web client is. Installed with npm it is a dependency, so it is here:
 htmldir: ./node_modules/frugal-iot-client
-# Development mode (uncomment these two, and comment out the two above, when working on the client)
+# To work on the client, either point this at your checkout of it
 #htmldir: ../frugal-iot-client
-#nodemodulesdir: ../frugal-iot-client/node_modules
+# or, better, leave this line alone and link the checkout in, which needs no edit here and so
+# nothing to remember to change back:
+#   cd ../frugal-iot-client && npm link
+#   cd ../frugal-iot-server && npm link frugal-iot-client
+#
+# nodemodulesdir is where the libraries the client loads (chart.js, luxon ...) are served from.
+# It does not normally need setting: the server looks inside htmldir first and then here, which
+# covers both an npm install (libraries hoisted to this directory) and a checkout (libraries in
+# the checkout's own node_modules). Set it only if yours are somewhere else again.
+#nodemodulesdir: ./node_modules
+
 # The path to the private directory is relative to the server's working directory.
 privatedir: ./private
 publicdir: ./public
 datadir: ./data
 otadir: ./ota
+
+# Looking after the readings in datadir, once a day. Readings are one file per topic per day and
+# nothing used to remove them, so a server left running would eventually fill its disk and stop.
+housekeeping:
+  # Compress days older than this. A day's file never changes once the day is over, and they
+  # compress to about a third. Nothing else needs to know - the server answers a request for a
+  # ".csv" with the ".csv.gz" when that is what it has. Never less than 2 days, whatever is set
+  # here, because the logger is still writing to today's file and possibly yesterday's.
+  compressafterdays: 2
+  # Delete readings older than this many days. 0 keeps them for ever, which is the default -
+  # deleting is not something to start doing to somebody's data without being asked.
+  deleteafterdays: 0
+  # Whatever the setting above, if the disk falls below this percentage free, the oldest days are
+  # deleted until it is back above. This is the safety net that stops a full card taking the
+  # machine down; set it to 0 to turn even that off and let the disk fill.
+  freespacepercent: 10
+  # enabled: false      # turns all of the above off

package/extras/mosquitto.conf

@@ -7,4 +7,12 @@ protocol websockets
 # which belongs to root. Both the broker and mosquitto_passwd warn unless the file belongs to
 # whoever opened it, and mosquitto_passwd also writes a temporary backup beside it, so the
 # directory has to be writable by that same user. See INSTALLATION.md step A5.
-password_file /var/lib/mosquitto/passwords
+password_file /var/lib/mosquitto/passwords
+
+# A node on a weak WiFi signal reconnects often, and each connection and disconnection would be a
+# line in /var/log/mosquitto/mosquitto.log. On an SD card those writes wear the card out, and
+# nothing is lost by dropping them, since the readings themselves are recorded by the server.
+# Comment this out while working out why a particular node keeps dropping off the network.
+# Everything else the broker reports, including why it refused to start, is unaffected - that log is
+# the only place some of those failures appear, so it stays.
+connection_messages false