frugal-iot-server 2.0.9 → 2.0.11

package/README.md

@@ -3,7 +3,7 @@
 ## Installation
 
 For a self-contained server on a Raspberry Pi - including the MQTT broker, and starting from a
-blank SD card - see [INSTALLATION.md](INSTALLATION.md), which also has a script that does the whole
+blank SD card - see [docs/INSTALLATION.md](docs/INSTALLATION.md), which also has a script that does the whole
 thing in one go once you can ssh in.
 
 On a unix box that is already running ...
@@ -91,7 +91,7 @@ verbose: true
 and each message reappears as `Received dev/lotus/esp8266-85ea2b/humidity   71.8`. `morgan: true`
 in `config.d/server.yaml` does the same for web requests. Turn both off again afterwards.
 There is more on what gets written, and how to see how much, under
-[Wear and tear on the SD card](INSTALLATION.md#wear-and-tear-on-the-sd-card).
+[Wear and tear on the SD card](docs/INSTALLATION.md#wear-and-tear-on-the-sd-card).
 
 Open a browser pointing at for example `localhost:8080` and you should see the UI.
 
@@ -161,7 +161,7 @@ expensive on an SD card.
 To set it up as a service that runs at startup (and instructions vary between flavors of Linux)
 
 copy and edit `extras/frugaliot.service` to `/etc/systemd/system/frugaliot.service`. As shipped it
-matches the Raspberry Pi install in [INSTALLATION.md](INSTALLATION.md) - user `pi`, installed in
+matches the Raspberry Pi install in [docs/INSTALLATION.md](docs/INSTALLATION.md) - user `pi`, installed in
 `/home/pi/frugal-iot` - so on any other machine change `User`, `WorkingDirectory` and `ExecStart`
 to the account and directory you installed into.
 ```

package/config.d/email.yaml

@@ -3,7 +3,7 @@
 #
 # Leave it commented out on a server that should not send mail.
 #
-# For Gmail - the common case, written out step by step in INSTALLATION.md section 9a:
+# For Gmail - the common case, written out step by step in docs/INSTALLATION.md section 9a:
 #   1. Turn on 2-Step Verification: https://myaccount.google.com/signinoptions/twosv
 #      App passwords do not exist until you do, which is where most people get stuck.
 #   2. Create an app password:      https://myaccount.google.com/apppasswords

package/config.d/mapping.yaml

@@ -0,0 +1,56 @@
+# The map: where its tiles come from, and whether this server keeps them.
+
+# Each source is a set of tiles the map can draw. Sources named "Map" are drawn everywhere; any other
+# name (e.g. "Imagery") is drawn over the map where the source has tiles, and the map offers a switch
+# between them. Sources sharing a name switch together. bounds ([[south, west], [north, east]]) and
+# minzoom are optional - outside them nothing is asked for.
+sources:
+  osm:
+    name: Map
+    enabled: true
+    upstream: https://tile.openstreetmap.org/{z}/{x}/{y}.png
+    attribution: '&copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors'
+    maxzoom: 19
+  nsw:
+    name: Imagery
+    # Aerial imagery of New South Wales from NSW Spatial Services, free and with no key.
+    # Licence: Creative Commons Attribution 4.0, though their own information sheet links the
+    # no-derivatives version - https://www.spatial.nsw.gov.au/__data/assets/pdf_file/0005/202559/NSW_Imagery_rebranded.pdf
+    # Showing it with the attribution is fine either way. It is unclear whether caching it here
+    # (cache and prefetch, below) constitutes redistribution, or is just good behaviour.
+    enabled: false
+    upstream: https://maps.six.nsw.gov.au/arcgis/rest/services/public/NSW_Imagery/MapServer/tile/{z}/{y}/{x}
+    attribution: '&copy; State Government of NSW and Spatial Services (DCS)'
+    bounds: [[-38.4, 138.4], [-26.5, 159.3]]
+    minzoom: 15
+    maxzoom: 21
+
+# Keeping tiles on this server. Off, the browser fetches them from each source itself.
+# On, this server keeps every tile anyone looks at and serves it to the map from then on, so a Pi
+# that goes offline still has a map. On a server that is always online it means one fetch from a
+# source per tile rather than one per browser.
+cache:
+  enabled: false
+  # Not under datadir: that is served at /data, and its housekeeping walks every file in it
+  dir: ./tiles
+  # A kept tile is checked with its source after this many days. Never less than 7 - OSM asks that.
+  # A check that finds it unchanged costs no rewrite, and offline the old tile is served regardless.
+  refreshdays: 30
+  # Tiles looked at beyond this are dropped once a day, least recently used first. A tile is 5-50 KB.
+  maxmb: 200
+
+# Fetching, while online, the tiles nobody has looked at yet - for a Pi that will go offline. Needs
+# the cache on. Not needed on a server that is always online. Tiles it fetches are never dropped
+# while still wanted.
+prefetch:
+  enabled: false
+  # Each organization's area, at every zoom while it fits in this many tiles - zoom 3 for the
+  # whole world, about 12 for a district. So zooming out always finds a map. Not for sources with
+  # bounds, which get only the squares around devices inside them.
+  organizationtiles: 64
+  # The same for each project's area
+  projecttiles: 256
+  # A square this many metres around each device, down to devicezoom (19 is OSM's deepest).
+  # About 100 tiles, 1-2 MB, per device and source. Each zoom deeper is four times as many.
+  deviceradius: 250
+  devicezoom: 19

package/config.d/opensearch.yaml

@@ -1,4 +1,4 @@
-# This server's own OpenSearch - the one INSTALL-OPENSEARCH.md sets up on the server itself.
+# This server's own OpenSearch - the one docs/INSTALL-OPENSEARCH.md sets up on the server itself.
 #
 # Leave it all commented out on a server without one. With it filled in, an organization's
 # administrator can choose "OpenSearch on this server" on the dashboard's Forwarders card, and make a
@@ -23,5 +23,5 @@
 #   password: change-me
 #
 # # The address Dashboards on someone's own computer uses to reach OpenSearch: https, through
-# # Apache (INSTALL-OPENSEARCH.md), never the port 9200 above.
+# # Apache (docs/INSTALL-OPENSEARCH.md), never the port 9200 above.
 # public: https://search.example.org

package/config.d/schema/modules.yaml

@@ -339,6 +339,7 @@ frugal_iot:
     - leaf:     id
     - leaf:     name
     - leaf:     description
+    - leaf:     location
     - leaf:     lastseen
 gps:
   name: GPS
@@ -383,6 +384,13 @@ heatsink:
       name: Heatsink Temperature
       min: 0
       max: 120
+# Seeed HM3301 particulate sensor ("Grove - Laser PM2.5 Sensor") - Sensor_HM3301.
+hm3301:
+  name: "HM3301"
+  topics:
+    - leaf: pm1
+    - leaf: pm25
+    - leaf: pm10
 ht:
   name: HT
   topics:
@@ -617,6 +625,12 @@ sector:
     - leaf: valve
       leaf_from: controlouttoggle
       name: Valve
+# Nova Fitness SDS011 particulate sensor, as in sensor.community's airRohr - Sensor_SDS011.
+sds011:
+  name: "SDS011"
+  topics:
+    - leaf: pm25
+    - leaf: pm10
 sht:
   name: SHT
   topics:

package/config.d/schema/topics.yaml

@@ -411,6 +411,16 @@ loadcell:
   rw:       r
   calibrate:  true
   graphable:  true
+# "lat,lon" in one payload, so a pin is never moved half-way; set from the UI, held by the node
+location:
+  leaf:     location
+  slot:     location
+  name:     Location
+  type:     text
+  log:      false
+  display:  text
+  rw:       w
+  retain:   true
 longitude:
   leaf:     longitude
   name:     Longitude
@@ -549,6 +559,56 @@ percent:
     significantvalue: 1
   graphable:  true
   units:    "%"
+# Particulate matter, from the SDS011 or HM3301 - see sensor/pm.h in Frugal-IoT. Both are noisy,
+# and unreliable above ~70% humidity (they count droplets as particles). Only the HM3301 has pm1.
+pm1:
+  leaf:     pm1
+  name:     PM1.0
+  type:     float
+  display:  bar
+  min:      0
+  max:      100
+  width:    5
+  color:    "#d2691e"
+  units:    ug/m3
+  rw:       r
+  graphable:  true
+  log:      true
+  duplicates:
+    significantdate: 900000
+    significantvalue: 1
+pm10:
+  leaf:     pm10
+  name:     PM10
+  type:     float
+  display:  bar
+  min:      0
+  max:      200
+  width:    5
+  color:    "#696969"
+  units:    ug/m3
+  rw:       r
+  graphable:  true
+  log:      true
+  duplicates:
+    significantdate: 900000
+    significantvalue: 1
+pm25:
+  leaf:     pm25
+  name:     PM2.5
+  type:     float
+  display:  bar
+  min:      0
+  max:      100
+  width:    5
+  color:    "#a0522d"
+  units:    ug/m3
+  rw:       r
+  graphable:  true
+  log:      true
+  duplicates:
+    significantdate: 900000
+    significantvalue: 1
 position:
   leaf:     position
   name:     Position

package/docs/OPENWRT.md

@@ -0,0 +1,208 @@
+# Running a Frugal IoT server on an OpenWrt router
+
+**Status: worked once, on one router, after fixes.** The Raspberry Pi installation
+([INSTALLATION.md](INSTALLATION.md)) is proven; this is the same thing re-done for OpenWrt. The first
+real run - an OpenWrt One, OpenWrt 25.12.5, see [Tested on](#tested-on) - needed five attempts and
+found ten problems, which are fixed or described below. Expect the next router to find more. This
+document says what a router needs, how to prepare it, gives two scripts, and says what to send back.
+
+## What you are installing
+
+A Frugal IoT server is three programs on one machine:
+
+* **mosquitto**, an MQTT broker. Small sensor boards (ESP32s) publish readings to it, and the
+  dashboard in your browser listens to it.
+* **the Frugal IoT server**, a Node.js program. It serves the dashboard on port 8080, keeps the
+  user logins and settings in an SQLite database, and writes every reading to disk.
+* nothing else. No cloud, no internet needed once it is installed.
+
+The router is simply where they run. It does not stop being a router.
+
+## What the router needs
+
+Run the preflight script (below) and it checks every line of this table for you.
+
+| | Needed | Why |
+| --- | --- | --- |
+| **CPU** | **aarch64 or x86_64.** 32-bit ARM might work. MIPS will not. | OpenWrt's `node` package is only built for some CPUs, and the database module (`sqlite3`) is a compiled binary that has to exist for that CPU and for OpenWrt's C library (musl). Both are most likely on 64-bit. |
+| **Memory** | 512 MB or more (256 MB at the very least) | The server uses about 250 MB once running. |
+| **Storage** | **About 400 MB free** on ext4, f2fs or btrfs | The software is ~100 MB; npm needs room to work; readings accumulate. **Most routers' own flash is far too small**, so this means a USB stick or SSD, set up as extroot or just mounted, and `--dir` pointing at it. Not FAT, exFAT or NTFS: the install needs symlinks and Unix permissions. The image may also lack the USB storage driver - see [Preparing the router](#preparing-the-router). |
+| **Node.js** | **20.17 or later**, and npm | From OpenWrt's feed if it has one - **OpenWrt 25.12 has none for aarch64**, in which case it is unpacked by hand from a musl build ([how](#if-the-feed-has-no-node)). The preflight and the installer accept either. |
+| **mosquitto** | A build with **websockets** | The browser dashboard connects over websockets. If the router's mosquitto has none, the dashboard cannot work. |
+| | Ideally the **dynamic security plugin** | It gives every node and every user its own broker password. Without it the server falls back to one shared password per organization: it works, but it is the older and weaker scheme. |
+| **Network** | Internet access while installing | opkg and npm download from their own servers. |
+| **Free ports** | 1883 (nodes), 9012 (dashboard websocket), 8080 (dashboard) | Nothing on OpenWrt uses these by default; LuCI is on 80/443. |
+
+Better to have, not required: a fixed LAN address (a router has one already), and a drive that
+tolerates constant small writes - the server logs readings all day, which is hard on a cheap USB stick.
+
+## Preparing the router
+
+Two things a router often lacks, both found on the first real run. Commands are shown for `apk`
+(OpenWrt 25.12 and later); on older releases use `opkg install` instead of `apk add`.
+
+### A drive the router can see
+
+The preflight tells you when none is visible. A USB drive formatted ext4 was invisible on the test
+router because the image carried no USB storage driver:
+
+```sh
+apk add kmod-usb-storage kmod-fs-ext4 block-mount
+block info                          # is the drive listed now? what is it called?
+block detect > /etc/config/fstab    # NOTE: replaces your mount configuration - if you already have
+block mount                         #   mounts there, add the new one by hand (LuCI: System > Mount Points)
+df -h /mnt/sda1
+```
+
+Check that the drive is mounted again after a reboot (`uci show fstab`, or LuCI's Mount Points), or
+the server will not find its files and will not start.
+
+### If the feed has no node
+
+The preflight's feed check says `no node package in this feed` when this applies. Node.js publishes
+musl builds, which is what OpenWrt uses, at <https://unofficial-builds.nodejs.org/download/release/>.
+Pick a version of 20.17 or later (24.9.0 worked) and the file for your CPU: `linux-arm64-musl` for
+aarch64, `linux-x64-musl` for x86_64. **Take the `.tar.gz`, not the `.tar.xz`** - BusyBox `tar` cannot
+unpack xz. Put it on the drive, not the router's flash:
+
+```sh
+cd /mnt/sda1
+wget https://unofficial-builds.nodejs.org/download/release/v24.9.0/node-v24.9.0-linux-arm64-musl.tar.gz
+tar xzf node-v24.9.0-linux-arm64-musl.tar.gz
+for t in node npm npx; do ln -sf /mnt/sda1/node-v24.9.0-linux-arm64-musl/bin/$t /usr/bin/$t; done
+apk add libstdcpp6 libatomic1       # the musl build needs both
+node -v; npm -v
+```
+
+If `node -v` says a library is missing, that is the `apk add` line. The preflight and installer then
+see node on the router and do not look for a package.
+
+## Doing it
+
+You need `ssh` access to the router as root. On the router, fetch the two scripts from the `main` branch:
+
+```sh
+B=https://raw.githubusercontent.com/mitra42/frugal-iot-server/main/scripts
+wget -O openwrt-preflight.sh $B/openwrt-preflight.sh
+wget -O install-openwrt.sh   $B/install-openwrt.sh
+```
+
+If `wget` complains about https, the router has no TLS support for it yet
+(`apk add ca-bundle libustream-mbedtls`, or `opkg update && opkg install ...` on older releases), or download the files on your computer
+and copy them across with `scp`.
+
+### 1. Preflight - on each candidate router
+
+```sh
+sh openwrt-preflight.sh --dir /mnt/sda1/frugal-iot --install 2>&1 | tee preflight.txt
+```
+
+* `--dir` is where you intend to put the server: a path on the drive. The default is `/opt/frugal-iot`.
+* Without `--install` it only reads things, and refreshes the package list. **With `--install` it
+  installs node, mosquitto and the other packages**, after asking, so that it can try mosquitto and
+  sqlite3 for real rather than guess. That is the same set the installer would put there anyway, but it
+  does use the router's own flash: it prints how much is free first. If the flash is nearly full, run
+  it without `--install` first.
+* It ends with `PASS`/`WARN`/`FAIL` counts and a verdict. **Please send the whole of `preflight.txt`
+  for every router you try**, including the ones that fail - which ones fail, and why, is the most
+  useful thing this stage can tell us.
+
+Which router to choose is then mostly: one with no `FAIL`, a USB port with a decent drive in it, and
+512 MB or more.
+
+### 2. Install - on the one you pick
+
+```sh
+sh install-openwrt.sh --org myfarm --name "My Farm" --email you@example.com \
+     --phone +61123456789 --dir /mnt/sda1/frugal-iot
+```
+
+* `--org` is 1-10 lower-case letters or digits. It becomes the first part of every MQTT topic, and
+  the sensor boards must be flashed with the same one. Pick something you will keep.
+* Passwords you do not give are generated and printed at the end. The whole run is also logged to
+  `/root/frugal-iot-install-<date>.log` (readable only by root; the generated passwords are in it).
+* `--dry-run` checks the arguments and the router and says what it would do, changing nothing.
+* It is safe to run again after a failure: steps already done are skipped. Give it the **same**
+  `--org` and `--dir` as before - a different `--org` creates a second organization. Passwords already
+  set are left alone, and the summary says "unchanged" rather than printing a made-up one.
+* It runs the server as root, as most OpenWrt services do. `--user frugal` creates and uses a
+  separate account instead, which is tidier but is another thing that has not been tried.
+
+When it finishes it prints the dashboard address and the settings the sensor boards need.
+
+### 3. If something goes wrong
+
+Send back:
+
+1. `preflight.txt`
+2. the install log (`/root/frugal-iot-install-*.log`) - or just the end of the terminal output
+3. `logread -e frugaliot | tail -50` and `logread -e mosquitto | tail -50`
+4. the output of `apk list -I | grep -E 'mosquitto|node|sqlite|zsh'` (`opkg list-installed | grep ...`
+   on older releases)
+
+## What is different from a Raspberry Pi, and why it might fail
+
+The installer is the Pi one, redone, so these are the places it can be wrong:
+
+* **No systemd.** The service is a procd init script, `/etc/init.d/frugaliot`. Control it with
+  `service frugaliot restart`. Logs go to the system log: `logread -e frugaliot`.
+* **`/var` is RAM.** On OpenWrt `/var` is `/tmp`, emptied at every reboot. The broker's password file
+  and account database would normally live in `/var/lib/mosquitto`, which would lose every account on
+  restart, so the installer keeps them in `/etc/mosquitto/state` and points the configuration there.
+  It also moves mosquitto's own persistence (the retained values) there if it was set to a
+  RAM path; the original configuration is kept as `/etc/mosquitto/mosquitto.conf.pre-frugal`.
+* **No bash, no zsh, no sudo, no apt.** The installer is plain `sh` and installs zsh itself (the
+  server's own commands are zsh scripts). BusyBox lacks `install(1)` (it is in `coreutils-install`),
+  which stopped the first run, so the installer places files with `cp`, `chown`, `chmod` and `mv`
+  instead; and its `tar` cannot unpack `.xz`. `stat`, `sed`, `grep` and `su` worked.
+* **The sqlite3 module.** The most likely single failure. If a binary does not exist for your CPU
+  and musl, `npm install` will try to compile it, and there is no compiler. The installer stops there
+  with the error. A fix exists (compile it on another machine, or use a build environment) but it
+  is a different project.
+* **The mosquitto package.** Which parts OpenWrt builds in - websockets, the dynamic security plugin,
+  `mosquitto_ctrl`, `mosquitto_passwd` - is the main thing nobody here knows. The preflight tries each
+  one, and the installer lists every mosquitto package the feed offers into the log.
+* **No WiFi fallback network.** On the Pi, a setup network is raised if its WiFi is lost. A router is
+  the access point, so this is skipped.
+* **Time.** Routers without a battery clock start at an old date until NTP runs. The server stamps
+  readings when they arrive, so check the router has the right time.
+
+## After it is running
+
+* **Do not open the ports to the internet.** OpenWrt accepts LAN connections and refuses WAN ones by
+  default; leave 1883, 9012 and 8080 that way. HTTPS (needed for over-the-air firmware updates to the
+  boards) is not covered here.
+* **Upgrading:**
+  ```sh
+  cd /mnt/sda1/frugal-iot && npm update frugal-iot-server && service frugaliot restart
+  ```
+  (and re-run `install-openwrt.sh` with the same arguments afterwards, which brings the broker
+  configuration up to date.)
+* **If `http://<router>:8080/` answers 404** (the dashboard at `/dashboard` may still work), the
+  server cannot find its own web pages. An install whose `frugal-iot-init` failed part-way and was
+  re-run - as on the first OpenWrt run, before `init` was fixed - keeps a `config.d/server.yaml`
+  with `publicdir: ./public`. Set these in `<install dir>/config.d/server.yaml`, then
+  `service frugaliot restart`:
+  ```yaml
+  htmldir: ./node_modules/frugal-iot-client
+  nodemodulesdir: ./node_modules
+  publicdir: ./node_modules/frugal-iot-server/public
+  privatedir: ./node_modules/frugal-iot-server/private
+  ```
+  Without `publicdir` the home page, `/about.html` and the PWA's `service-worker.js` are missing too.
+* **`npx --no frugal-iot-diagnostic`** was written for systemd and will report some things that do
+  not apply here. Treat it as a hint, not a verdict.
+* **Removing it:**
+  ```sh
+  service frugaliot stop; /etc/init.d/frugaliot disable; rm /etc/init.d/frugaliot
+  rm -rf /mnt/sda1/frugal-iot                     # your install directory: this deletes the readings too
+  rm /etc/mosquitto/conf.d/frugal-iot.conf /etc/mosquitto/aclfile; rm -rf /etc/mosquitto/state
+  cp /etc/mosquitto/mosquitto.conf.pre-frugal /etc/mosquitto/mosquitto.conf   # if it exists
+  /etc/init.d/mosquitto restart
+  ```
+
+## Tested on
+
+| Date | Router | OpenWrt | Node | Server | Result |
+| --- | --- | --- | --- | --- | --- |
+| 2026-11 | OpenWrt One (Banana Pi), MediaTek Filogic, aarch64_cortex-a53, 986 MB; USB drive, ext4, mounted at `/mnt/sda1` | 25.12.5 with LibreMesh, `apk`, kernel 6.12.94 | v24.9.0 and npm 11.6.0, unpacked by hand (no node in the aarch64 feed) | 2.0.8 | Installed and running: broker (websockets and dynamic security both present), organization, procd service, dashboard. Took five runs: no USB storage driver; no node package; no `install` command; `frugal-iot-init` failing on `extras/grafana`, which also left `server.yaml` unconfigured and made `/` answer 404. All now handled by the scripts or described above, except the `init` fix, which needs a server release to reach a router. |

package/extras/frugaliot.service

@@ -1,6 +1,6 @@
 # Systemd unit for the Frugal IoT server.
 #
-# As shipped this matches the Raspberry Pi installation in INSTALLATION.md: user "pi", installed
+# As shipped this matches the Raspberry Pi installation in docs/INSTALLATION.md: user "pi", installed
 # with npm into /home/pi/frugal-iot. If that is what you have, install it unedited:
 #   sudo cp extras/frugaliot.service /etc/systemd/system/frugaliot.service
 #   sudo systemctl daemon-reload

package/extras/mosquitto.conf

@@ -6,7 +6,7 @@ protocol websockets
 # Kept under /var/lib/mosquitto, which belongs to the mosquitto user, rather than /etc/mosquitto,
 # which belongs to root. Both the broker and mosquitto_passwd warn unless the file belongs to
 # whoever opened it, and mosquitto_passwd also writes a temporary backup beside it, so the
-# directory has to be writable by that same user. See INSTALLATION.md step A5.
+# directory has to be writable by that same user. See docs/INSTALLATION.md step A5.
 password_file /var/lib/mosquitto/passwords
 
 # Which topics each account may reach. Without this, every account that can log in can publish and

package/frugal-iot-createdb.sql

@@ -108,7 +108,7 @@ CREATE TABLE IF NOT EXISTS `forwarders` (
 CREATE INDEX IF NOT EXISTS `forwarders_org` ON `forwarders` (`org`);
 
 -- A credential for Grafana (or anything else that reads history over HTTP), see lib/grafana.js and
--- GRAFANA.md. Read-only, and scoped to one organization: it can fetch that organization's readings
+-- docs/GRAFANA.md. Read-only, and scoped to one organization: it can fetch that organization's readings
 -- through /grafana/:org/ and nothing else.
 --
 -- Only a SHA-256 of the token is kept, so a copy of this database does not hand out working
@@ -269,3 +269,12 @@ CREATE TABLE IF NOT EXISTS `node_attempts` (
   `count` INTEGER NOT NULL DEFAULT 1,
   UNIQUE(`org`, `nodeid`)
 );
+
+-- Which languages browsers ask for, counted at each sign-in, to choose what to translate next - see
+-- lib/language-requests.js. Codes only: nothing records who asked.
+CREATE TABLE IF NOT EXISTS `language_requests` (
+  `code` TEXT PRIMARY KEY, -- lower-case, as the browser sent it: hi-in, pt-br
+  `count` INTEGER NOT NULL,
+  `last_seen` INTEGER NOT NULL,
+  `supported` INTEGER NOT NULL -- 1 if the client had a file for it at the last count
+);

package/frugal-iot-server.js

@@ -125,6 +125,8 @@ import { APIError } from './lib/api-errors.js';
 import { access, constants, createReadStream, mkdir, readdir, readFile, rm } from 'fs'; // https://nodejs.org/api/fs.html
 import { createGunzip } from 'zlib'; // https://nodejs.org/api/zlib.html - for serving compressed readings
 import { startHousekeeping } from './lib/housekeeping.js';
+import { createTileCache, parseTile, startTileHousekeeping } from './lib/tiles.js';
+import { loggerLocations, startPrefetch } from './lib/tile-prefetch.js';
 import { detectSeries } from 'async'; // https://caolan.github.io/async/v3/docs.html
 import { createMD5 } from 'hash-wasm';
 import multer from 'multer'; // https://www.npmjs.com/package/multer
@@ -140,6 +142,7 @@ import crypto from 'crypto'; /* https://nodejs.org/api/crypto.html */
 import {waterfall, each} from 'async';
 import { mailInit, mailConfigured, sendMail } from './lib/mailer.js';
 import { resetCodeMake, resetCodeCheck, resetRateOk } from './lib/resetcode.js';
+import { countLanguages, parseBrowserLanguages } from './lib/language-requests.js';
 // import { openDB } from 'sqlite-express-package'; /* appContent, appSelect, validateId, validateAlias, tagCloud, atom, rss,*/
 
 export let config; // Live binding - lib/api-routes.js reads the current value at request time
@@ -155,8 +158,16 @@ function loginRedirect(res, { mode = 'signin', message, messagetype, url }) {
   if (message) { q.set('message', message); }
   if (messagetype) { q.set('messagetype', messagetype); }
   if (url) { q.set('url', url); }   // Encoded here: a return url usually has a query string of its own
+  const lang = langOf(url);
+  if (lang) { q.set('lang', lang); }
   res.redirect(`${loginUrl}?${q}`);
 }
+// The language of the page being returned to. login.html reads only its own ?lang=, and without it
+// falls back to the browser's language rather than the one picked before signing in.
+function langOf(url) {
+  if (!url) return null;
+  try { return new URL(url, 'http://localhost').searchParams.get('lang'); } catch (e) { return null; }
+}
 // A password reset link carries its token in the query string, and everything that logs a URL goes
 // to the journal, which outlives the ten minutes the token is good for.
 function redactUrl(url) {
@@ -936,12 +947,8 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
     "Content-Length", "0"]
      */
 
-    // Just log the request for now
-    // noinspection JSCheckFunctionSignatures
-    app.use('/', (req, res, next) => {
-      console.log(redactUrl(req.url));
-      next();
-    })
+    // No request logging of its own: morgan, above, is the one switch for it, and on an SD card
+    // every line is a write.
 
     // Nodes enrol here to get their own broker credential. Deliberately not behind a session: a node
     // has no session, and this is what replaces the shared password compiled into its firmware.
@@ -1241,7 +1248,7 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
         app.use('/api', apiRouter);
         console.log("Mounted API routes at /api");
 
-        // History for Grafana, by bearer token - see lib/grafana.js and GRAFANA.md
+        // History for Grafana, by bearer token - see lib/grafana.js and docs/GRAFANA.md
         const grafana = createGrafanaRouters(db, config.server.datadir, loggerClient, { loggedInOrFail, can_READ });
         app.use('/grafana', grafana.data);
         app.use('/grafana_token', grafana.admin);
@@ -1273,21 +1280,33 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
           (req,res,next) => {
             console.log("Trying to login with redirect to",req.body.url);
             // This is ugly, but I cannot see how to pass the URL to passport.authenticate options
-            // Default the destination rather than trusting every caller to supply one: with no
-            // successRedirect, passport calls next() on success and the request 404s as
-            // "Cannot POST /login" - a confusing way to report a missing form field.
+            // Default the destination rather than trusting every caller to supply one - a missing
+            // form field should still land somewhere, here and in the success handler below.
             const returnTo = req.body.url || '/dashboard';
             const failureQuery = new URLSearchParams({
               mode: 'signin', messagetype: 'error',
               message: 'Incorrect username or password', url: returnTo });
+            const lang = langOf(returnTo);
+            if (lang) { failureQuery.set('lang', lang); }
             passport.authenticate('local', {
               session: true,
               //failWithError: true,
               failureRedirect: `${loginUrl}?${failureQuery}`,
-              successRedirect: returnTo,
+              // No successRedirect: on success passport calls next(), which counts and then redirects
               // In failure case will also be messages in the session which need clearing out TODO-N89
               //failureRedirect: `${loginUrl}?register=false&message=Incorrect+username+or+password&url=${req.body.url}`,
             })(req, res, next);
+          },
+          // Counted only once signed in, so failed attempts cannot pad the numbers. Not waited for.
+          (req, res) => {
+            const codes = parseBrowserLanguages(req.body.browserlangs);
+            if (codes.length) {
+              console.log("Browser languages", codes.join(','));
+              countLanguages(db, config.server.htmldir, codes, (err) => {
+                if (err) console.error("Could not count browser languages:", err.message);
+              });
+            }
+            res.redirect(req.body.url || '/dashboard');
           }
         );
         app.post('/register', (req, res) => {
@@ -1826,7 +1845,6 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
         const routerDashboard = express.Router();
         app.use('/dashboard', routerDashboard);
         routerDashboard.use(
-          (req,res,next) => {console.log("/dashboard handler for", req.url); next(); }, // Log attempt
           shouldIBeLoggedIn, // redirect to ./login.html if not logged in then back here
           //(req,res,next) => {console.log("XXX back to /dashboard handler for", req.url); next(); }, // Log attempt
           //(req,res,next) => {console.log("XXX", config.server.htmldir); next(); }, // Log attempt
@@ -1835,6 +1853,27 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
           express.static(config.server.htmldir, revalidate) // Serve static
         );
 
+        // Map tiles, kept here so the map works offline. Behind the login, or every server would be a
+        // free proxy to OpenStreetMap. Off unless config.d/mapping.yaml turns it on.
+        const mapping = config.mapping || {};
+        if (mapping.cache && mapping.cache.enabled) {
+          const tiles = createTileCache({ ...mapping.cache, sources: mapping.sources });
+          console.log("Serving /tiles from", tiles.dir, "for", Object.keys(tiles.sources).join(", "));
+          startTileHousekeeping(tiles);
+          if (mapping.prefetch && mapping.prefetch.enabled) {
+            startPrefetch(tiles, mapping.prefetch, () => loggerLocations(mqttLogger));
+          }
+          app.get('/tiles/:source/:z/:x/:y', loggedInOrFail, (req, res) => {
+            const t = parseTile(req.params.z, req.params.x, req.params.y);
+            if (!t) { res.sendStatus(404); return; }
+            tiles.get(req.params.source, t, (err, file) => {
+              if (!file) { res.sendStatus(404); return; }
+              res.set('Cache-Control', 'private, max-age=86400');
+              res.sendFile(path.resolve(file));
+            });
+          });
+        }
+
         // Serve frugal-iot-logger data at /data but configure where to get them.
         console.log("Serving /data from", config.server.datadir);
         // Nothing ever removed old readings, so a server left running filled its disk - which on a

package/lib/config-for-user.js

@@ -8,6 +8,8 @@
  * possible.
  */
 
+import { sourcesForBrowser } from './tiles.js';
+
 // Sections of the configuration that are never sent to a browser, whatever permissions the user has.
 // config.d itself is not served - the static mounts are publicdir, htmldir, datadir, privatedir and
 // node_modules, so config.d/schema/devices.yaml is a 404 - and buildConfigFor is the only route by
@@ -92,6 +94,8 @@ export function buildConfigFor(config, user) {
   Object.entries(config).forEach(([key, value]) => {
     if (configSectionsNeverServed.includes(key)) {
       // Deliberately absent from the result
+    } else if (key === 'mapping') {
+      // The server's own settings - the browser's version is built below
     } else if (key === 'organizations') {
       Object.entries(value).forEach(([orgid, org]) => {
         if (hasPermissions(user, orgid, 'READ')) {
@@ -102,6 +106,8 @@ export function buildConfigFor(config, user) {
       oo[key] = value;
     }
   });
+  // The browser gets only what the map draws (M-27), and gets it whether or not mapping.yaml exists
+  oo.mapping = { sources: sourcesForBrowser(config.mapping) };
   return oo;
 }
 

package/lib/grafana.js

@@ -21,7 +21,7 @@
  *   GET  /grafana_token/:org                   list names and when last used (yours; all, for an org-wide administrator)
  *   POST /grafana_token/:org/revoke  {name}   yours; any, for an org-wide administrator
  *
- * See GRAFANA.md for setting up Grafana to use these.
+ * See docs/GRAFANA.md for setting up Grafana to use these.
  */
 
 import crypto from 'crypto';
@@ -398,7 +398,7 @@ export function createGrafanaRouters(db, dataDir, loggerClient, auth) {
       res.json(rows.map(({ user_id, ...t }) => ({ ...t, mine: user_id === Number(req.user.id) })));
     } catch (err) { next(err); }
   });
-  // The starter dashboard, to save and import into Grafana (GRAFANA.md). Nothing in it is secret.
+  // The starter dashboard, to save and import into Grafana (docs/GRAFANA.md). Nothing in it is secret.
   admin.get('/:org/dashboard', auth.loggedInOrFail, auth.can_READ, async (req, res, next) => {
     try {
       const dash = await dashboardFor(req.params.org, String(req.query.server || ''));