frugal-iot-server 2.0.10 → 2.0.12

package/config.d/mapping.yaml

@@ -0,0 +1,56 @@
+# The map: where its tiles come from, and whether this server keeps them.
+
+# Each source is a set of tiles the map can draw. Sources named "Map" are drawn everywhere; any other
+# name (e.g. "Imagery") is drawn over the map where the source has tiles, and the map offers a switch
+# between them. Sources sharing a name switch together. bounds ([[south, west], [north, east]]) and
+# minzoom are optional - outside them nothing is asked for.
+sources:
+  osm:
+    name: Map
+    enabled: true
+    upstream: https://tile.openstreetmap.org/{z}/{x}/{y}.png
+    attribution: '&copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors'
+    maxzoom: 19
+  nsw:
+    name: Imagery
+    # Aerial imagery of New South Wales from NSW Spatial Services, free and with no key.
+    # Licence: Creative Commons Attribution 4.0, though their own information sheet links the
+    # no-derivatives version - https://www.spatial.nsw.gov.au/__data/assets/pdf_file/0005/202559/NSW_Imagery_rebranded.pdf
+    # Showing it with the attribution is fine either way. It is unclear whether caching it here
+    # (cache and prefetch, below) constitutes redistribution, or is just good behaviour.
+    enabled: false
+    upstream: https://maps.six.nsw.gov.au/arcgis/rest/services/public/NSW_Imagery/MapServer/tile/{z}/{y}/{x}
+    attribution: '&copy; State Government of NSW and Spatial Services (DCS)'
+    bounds: [[-38.4, 138.4], [-26.5, 159.3]]
+    minzoom: 15
+    maxzoom: 21
+
+# Keeping tiles on this server. Off, the browser fetches them from each source itself.
+# On, this server keeps every tile anyone looks at and serves it to the map from then on, so a Pi
+# that goes offline still has a map. On a server that is always online it means one fetch from a
+# source per tile rather than one per browser.
+cache:
+  enabled: false
+  # Not under datadir: that is served at /data, and its housekeeping walks every file in it
+  dir: ./tiles
+  # A kept tile is checked with its source after this many days. Never less than 7 - OSM asks that.
+  # A check that finds it unchanged costs no rewrite, and offline the old tile is served regardless.
+  refreshdays: 30
+  # Tiles looked at beyond this are dropped once a day, least recently used first. A tile is 5-50 KB.
+  maxmb: 200
+
+# Fetching, while online, the tiles nobody has looked at yet - for a Pi that will go offline. Needs
+# the cache on. Not needed on a server that is always online. Tiles it fetches are never dropped
+# while still wanted.
+prefetch:
+  enabled: false
+  # Each organization's area, at every zoom while it fits in this many tiles - zoom 3 for the
+  # whole world, about 12 for a district. So zooming out always finds a map. Not for sources with
+  # bounds, which get only the squares around devices inside them.
+  organizationtiles: 64
+  # The same for each project's area
+  projecttiles: 256
+  # A square this many metres around each device, down to devicezoom (19 is OSM's deepest).
+  # About 100 tiles, 1-2 MB, per device and source. Each zoom deeper is four times as many.
+  deviceradius: 250
+  devicezoom: 19

package/config.d/schema/modules.yaml

@@ -339,6 +339,7 @@ frugal_iot:
     - leaf:     id
     - leaf:     name
     - leaf:     description
+    - leaf:     location
     - leaf:     lastseen
 gps:
   name: GPS

package/config.d/schema/topics.yaml

@@ -411,6 +411,16 @@ loadcell:
   rw:       r
   calibrate:  true
   graphable:  true
+# "lat,lon" in one payload, so a pin is never moved half-way; set from the UI, held by the node
+location:
+  leaf:     location
+  slot:     location
+  name:     Location
+  type:     text
+  log:      false
+  display:  text
+  rw:       w
+  retain:   true
 longitude:
   leaf:     longitude
   name:     Longitude

package/docs/INSTALLATION.md

@@ -1164,8 +1164,8 @@ but close enough for sensor data.
 ### Wear and tear on the SD card
 
 SD cards wear out from being written to, and a server recording sensor readings writes all the time.
-We have not yet seen a card fail, but plan for one eventually: keep a copy of `config.d/` and
-`frugal-iot.db` somewhere else, and treat the readings in `data/` as valuable but not irreplaceable.
+We have not yet seen a card fail, but plan for one eventually: see [Backups](#backups), and keep
+the snapshots somewhere other than the card.
 
 A new installation is set up to write as little as it reasonably can, and the settings are worth
 knowing about, because turning one of them back on for debugging and forgetting is easy to do:
@@ -1199,6 +1199,90 @@ and read the **Wear on the SD card** section, which reports how much has been wr
 whether the machine is swapping, and whether each of the settings above is in force. Run it twice a
 few days apart — the interesting number is the rate, not the total.
 
+## Backups
+
+`frugal-iot-backup` takes a snapshot of everything on this machine that cannot be rebuilt: the
+database, `config.d/` (including `secrets.yaml`), the broker's configuration, password file, ACL
+file and TLS certificate, and the server's systemd unit. It finds the broker's files by asking the
+broker's own configuration, so it works however this machine is laid out.
+
+Not in it: `data/` (large, and copied separately - see below), `ota/` (rebuilt at each release),
+and anything the broker rebuilds or is running on.
+
+First see what it would take, and from where - this writes nothing:
+
+```
+cd ~/frugal-iot
+npx --no frugal-iot-backup check
+```
+
+Then take one now, and install a timer that takes one every night:
+
+```
+npx --no frugal-iot-backup
+npx --no frugal-iot-backup install
+```
+
+Both ask for `sudo`, because the broker's files belong to root and to `mosquitto`. Snapshots go to
+`/var/backups/frugal-iot/`, keeping one a day for a week, one a week for a month, one a month for
+three months, one a quarter for a year, and one a year for ever - about 20 small files.
+
+**A snapshot holds every credential this server has** - with it, someone can log in as any node or
+user on the broker. So each one is mode 600, owned by the account the server runs as, in a directory
+nobody else can open. Copy them off the machine the same way: by `rsync` or `scp` as that account,
+to a machine you trust as much as this one.
+
+A snapshot on the same SD card is lost with the card, so it only protects against mistakes. To
+protect against losing the machine, copy `/var/backups/frugal-iot/` and `data/` somewhere else
+regularly - for example from a laptop:
+
+```
+rsync -a pi@frugaliot.local:/var/backups/frugal-iot/ ~/frugal-iot-backups/frugaliot/state/
+rsync -a pi@frugaliot.local:frugal-iot/data/ ~/frugal-iot-backups/frugaliot/data/
+```
+
+Leave out `--delete` on the second: the server may delete old readings to save space, and this way
+the copy keeps them.
+
+`npx --no frugal-iot-diagnostic` reports how old the newest snapshot is, and calls it a problem if
+the timer is installed and nothing has been written for two days.
+
+### Restoring
+
+Restore onto a machine where the server is already installed - after re-flashing a Pi, run the
+installer first, then restore over what it set up. Copy the snapshot onto the machine, then from
+the server's directory see what would happen - this changes nothing:
+
+```
+npx --no frugal-iot-restore <snapshot>
+```
+
+`latest` in place of a file name means the newest one in `/var/backups/frugal-iot/`. Then do it,
+in one of two ways:
+
+* **`state`** - onto a machine its own installer set up, laid out however that is. It restores the
+  database, `config.d/` (including `secrets.yaml`) and the broker's password file, ACL file and
+  account state, each to wherever *this* broker keeps them. It keeps this machine's own broker
+  configuration, TLS certificate, systemd unit, `mqtt.yaml` and `server.yaml`.
+* **`all`** - onto the same machine, or one laid out exactly like it: everything goes back where it
+  came from. Broker configuration files the snapshot does not have are renamed to
+  `<name>.pre-restore`, so the broker does not read both.
+
+```
+npx --no frugal-iot-restore <snapshot> state
+```
+
+If the machine already has logins of its own, add `force` as the last word - it says so rather
+than replacing them silently.
+
+Before changing anything it takes a snapshot of what is there into
+`/var/backups/frugal-iot/pre-restore/`, so a restore can be undone with
+`npx --no frugal-iot-restore <that snapshot> all force`. It stops the server and the broker while it
+works - every node disconnects for a few seconds - and finishes by checking the broker's accounts
+against the database and running the diagnostic.
+
+`data/` is not in a snapshot: copy it back from wherever it is mirrored.
+
 ## Moving the Pi to another WiFi network
 
 This is how to give the Pi a new WiFi network when you cannot log in to it, because it no longer
@@ -1317,7 +1401,8 @@ Two things to know:
 
 If you have not reached step 4 yet, the command does not exist. Copy
 [scripts/diagnostic.zsh](https://github.com/mitra42/frugal-iot-server/blob/main/scripts/diagnostic.zsh)
-to the Pi and run `zsh diagnostic.zsh` instead.
+and [scripts/broker-config.zsh](https://github.com/mitra42/frugal-iot-server/blob/main/scripts/broker-config.zsh),
+which it needs, into the same directory on the Pi and run `zsh diagnostic.zsh` instead.
 
 ### A topic that will not go away
 
@@ -1363,8 +1448,11 @@ sudo systemctl restart frugaliot
 Taking those in turn:
 
 * `npm update` fetches the new version of the server, and of the client and logger it depends on.
+  It may report vulnerabilities, typically in `uuid`. `frugal-iot-init`, next, is what fixes those,
+  so ignore that report and do not run `npm audit fix`.
 * `frugal-iot-init` adds any configuration file or directory the new version expects and you do not
-  have yet. It never changes files you already have, so your settings, database and data are safe.
+  have yet. It never changes your settings, database or data. It also copies the release's
+  dependency `overrides` into your `package.json` and re-runs `npm install` - see below.
 * `systemctl restart frugaliot` is what actually puts the new version into service — without it the
   old one keeps running from memory, and nothing appears to have changed. (Skip this if you are
   running the server by hand rather than as a service; stop it with `Ctrl-C` and start it again.)
@@ -1377,17 +1465,37 @@ journalctl -u frugaliot -n 30
 npm ls frugal-iot-server
 ```
 
-`frugal-iot-init` leaves your own files alone. Two sections of its output are worth reading.
+`frugal-iot-init` leaves your own files alone, with two exceptions, and tells you what differs.
 
-**"These files were left as you have them, but this release ships a different version"** covers the
-sensor schema under `config.d/schema/`, and gives you a `diff` command for each. If you added a
-sensor type of your own, that is exactly what it should say; if you did not, this release changed
-the schema and the difference is worth reading.
+**`package.json`** in the install directory gets the `overrides` section from frugal-iot-server's
+own. These pin transitive dependencies (such as `uuid`, deep below the logger) to versions without
+known vulnerabilities. npm only reads `overrides` from the top-level project, which here is your
+`package.json`, not the package's, so without this step `npm audit` reports problems that a checkout
+of the same release does not. When it adds or changes one it prints `set ./package.json override ...`
+and runs `npm install` to apply it; `npm audit` should then come back clean.
 
-**"Installed elsewhere from extras/, and older than this release"** covers the files that live
-somewhere else — the broker configuration, its ACL, the systemd unit. It compares what is actually
-in `/etc`, not the copy in `extras/`, because that copy is never overwritten and so differs for ever
-once a release changes it. To bring anything it names up to date, re-run the installer:
+**The sensor schema** under `config.d/schema/` is the other. If every line of your copy is still
+in this release's version, all the release did was add to it, so it installs the new one and says
+`updated ... (this release only adds to it)`. If your copy has lines the release does not, it keeps
+yours and shows just those lines: if you added them, that is exactly right; if you did not, this
+release changed or removed them, and it prints the `cp` that takes its version.
+
+**The broker's configuration** is compared setting by setting across every file the broker reads,
+wherever they are - one `frugal-iot.conf` from the installer, or several files in
+`/etc/mosquitto/conf.d/` assembled by hand. Each difference is one line:
+
+```
+  missing:     connection_messages false - not set (mosquitto's default applies)
+  (fine)       password_file - here /etc/mosquitto/mosquitto_passwords (this release's default is /var/lib/mosquitto/passwords)
+```
+
+A file kept at a different path is fine. A `missing:` or `different:` line is something this release
+sets that your broker does not. Add it to any file in `/etc/mosquitto/conf.d/`, then
+`sudo systemctl restart mosquitto` - or, on a Pi set up by `install-pi.sh`, re-run that.
+
+**"Installed elsewhere from extras/, and different from this release"** covers the ACL file and the
+systemd units, found where the broker and systemd say they are. It lists the differing lines,
+marked `here only` and `release only`. To bring them up to date on a Pi, re-run the installer:
 
 ```
 bash node_modules/frugal-iot-server/scripts/install-pi.sh
@@ -1403,29 +1511,18 @@ install these three, because none of them can simply be copied into place:
 | `extras/frugaliot.service` | written for user `pi` in `/home/pi/frugal-iot`; the installer rewrites `User`, `WorkingDirectory` and `ExecStart` for wherever this actually went. |
 
 The setup network's watchdog, `extras/frugaliot-ap-watchdog` and its `.service` and `.timer`, *can* be
-copied as they are; step 8a has the commands. On a Pi installed before the setup network existed,
-re-running the installer adds it, and asks once for the `pi` login's password to give it.
+copied as they are; step 8a has the commands. It is only checked where it is installed. On a Pi
+installed before the setup network existed, re-running the installer adds it, and asks once for the
+`pi` login's password to give it.
 
 The installer will not overwrite an ACL file that has per-bridge rules in it. If this release adds a
 rule to a broker that has them, it says which lines are missing and leaves the file alone for you to
 add them by hand, then `sudo systemctl reload mosquitto`.
 
 > **If your broker's configuration was assembled by hand** — several small files in
-> `/etc/mosquitto/conf.d/` rather than the single `frugal-iot.conf` the installer writes — then
-> `frugal-iot-init` reports those files under "Nothing at the usual place, so these were not
-> compared", and re-running the installer is *not* what you want: it would add `frugal-iot.conf`
-> alongside what you already have, declaring the same listeners and the same plugin twice. Compare
-> by hand instead, and add only what is missing:
->
-> ```
-> sudo grep -hvE '^\s*(#|$)' /etc/mosquitto/conf.d/*.conf | sort -u > /tmp/running.txt
-> grep -hvE '^\s*(#|$)' node_modules/frugal-iot-server/extras/mosquitto.conf | sort -u > /tmp/shipped.txt
-> diff /tmp/running.txt /tmp/shipped.txt
-> ```
->
-> Lines only in the running config are usually this machine's own — TLS `certfile`/`keyfile`, a
-> bridge `listener`, a `password_file` in a different place. Lines only in the shipped one are what
-> this release added.
+> `/etc/mosquitto/conf.d/` rather than the single `frugal-iot.conf` the installer writes — do not
+> re-run the installer: it would add `frugal-iot.conf` alongside what you already have, declaring the
+> same listeners and the same plugin twice. Add the `missing:` and `different:` settings by hand.
 
 > It does not compare `config.yaml`, `config.d/mqtt.yaml`, `config.d/logger.yaml` or
 > `config.d/server.yaml`, because those hold your own settings and would differ every time. If a
@@ -1437,7 +1534,7 @@ which for the settings under [Wear and tear on the SD card](#wear-and-tear-on-th
 goes on writing as often as it did. A server that has been running since before those settings
 existed keeps logging every message and every web request, and writing every reading as it arrives —
 deliberately, since quietening somebody's server without being asked is not a thing an upgrade should
-do. On a Pi, compare the two files above and copy across the settings you want.
+do. On a Pi, compare the files named above and copy across the settings you want.
 
 Your organizations, accounts, database and logged data are untouched by an upgrade — they live in
 this directory, not in `node_modules`.

package/docs/OPENWRT.md

@@ -24,8 +24,8 @@ Run the preflight script (below) and it checks every line of this table for you.
 
 | | Needed | Why |
 | --- | --- | --- |
-| **CPU** | **aarch64 or x86_64.** 32-bit ARM might work. MIPS will not. | OpenWrt's `node` package is only built for some CPUs, and the database module (`sqlite3`) is a compiled binary that has to exist for that CPU and for OpenWrt's C library (musl). Both are most likely on 64-bit. |
-| **Memory** | 512 MB or more (256 MB at the very least) | The server uses about 250 MB once running. |
+| **CPU** | **aarch64 or x86_64.** 32-bit ARM might work. **MIPS will not** (Node.js has no MIPS build at all, so older and cheaper routers - the GL.iNet AR300M, for one - are out). | OpenWrt's `node` package is only built for some CPUs, and the database module (`sqlite3`) is a compiled binary that has to exist for that CPU and for OpenWrt's C library (musl). Both are most likely on 64-bit. |
+| **Memory** | **512 MB or more** | The server alone uses about 250 MB once running, on top of the router's own software. A router with 256 MB cannot do it (a Cudy with 233 MB usable was refused). |
 | **Storage** | **About 400 MB free** on ext4, f2fs or btrfs | The software is ~100 MB; npm needs room to work; readings accumulate. **Most routers' own flash is far too small**, so this means a USB stick or SSD, set up as extroot or just mounted, and `--dir` pointing at it. Not FAT, exFAT or NTFS: the install needs symlinks and Unix permissions. The image may also lack the USB storage driver - see [Preparing the router](#preparing-the-router). |
 | **Node.js** | **20.17 or later**, and npm | From OpenWrt's feed if it has one - **OpenWrt 25.12 has none for aarch64**, in which case it is unpacked by hand from a musl build ([how](#if-the-feed-has-no-node)). The preflight and the installer accept either. |
 | **mosquitto** | A build with **websockets** | The browser dashboard connects over websockets. If the router's mosquitto has none, the dashboard cannot work. |

package/frugal-iot-server.js

@@ -125,6 +125,8 @@ import { APIError } from './lib/api-errors.js';
 import { access, constants, createReadStream, mkdir, readdir, readFile, rm } from 'fs'; // https://nodejs.org/api/fs.html
 import { createGunzip } from 'zlib'; // https://nodejs.org/api/zlib.html - for serving compressed readings
 import { startHousekeeping } from './lib/housekeeping.js';
+import { createTileCache, parseTile, startTileHousekeeping } from './lib/tiles.js';
+import { loggerLocations, startPrefetch } from './lib/tile-prefetch.js';
 import { detectSeries } from 'async'; // https://caolan.github.io/async/v3/docs.html
 import { createMD5 } from 'hash-wasm';
 import multer from 'multer'; // https://www.npmjs.com/package/multer
@@ -945,12 +947,8 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
     "Content-Length", "0"]
      */
 
-    // Just log the request for now
-    // noinspection JSCheckFunctionSignatures
-    app.use('/', (req, res, next) => {
-      console.log(redactUrl(req.url));
-      next();
-    })
+    // No request logging of its own: morgan, above, is the one switch for it, and on an SD card
+    // every line is a write.
 
     // Nodes enrol here to get their own broker credential. Deliberately not behind a session: a node
     // has no session, and this is what replaces the shared password compiled into its firmware.
@@ -1847,7 +1845,6 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
         const routerDashboard = express.Router();
         app.use('/dashboard', routerDashboard);
         routerDashboard.use(
-          (req,res,next) => {console.log("/dashboard handler for", req.url); next(); }, // Log attempt
           shouldIBeLoggedIn, // redirect to ./login.html if not logged in then back here
           //(req,res,next) => {console.log("XXX back to /dashboard handler for", req.url); next(); }, // Log attempt
           //(req,res,next) => {console.log("XXX", config.server.htmldir); next(); }, // Log attempt
@@ -1856,6 +1853,27 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
           express.static(config.server.htmldir, revalidate) // Serve static
         );
 
+        // Map tiles, kept here so the map works offline. Behind the login, or every server would be a
+        // free proxy to OpenStreetMap. Off unless config.d/mapping.yaml turns it on.
+        const mapping = config.mapping || {};
+        if (mapping.cache && mapping.cache.enabled) {
+          const tiles = createTileCache({ ...mapping.cache, sources: mapping.sources });
+          console.log("Serving /tiles from", tiles.dir, "for", Object.keys(tiles.sources).join(", "));
+          startTileHousekeeping(tiles);
+          if (mapping.prefetch && mapping.prefetch.enabled) {
+            startPrefetch(tiles, mapping.prefetch, () => loggerLocations(mqttLogger));
+          }
+          app.get('/tiles/:source/:z/:x/:y', loggedInOrFail, (req, res) => {
+            const t = parseTile(req.params.z, req.params.x, req.params.y);
+            if (!t) { res.sendStatus(404); return; }
+            tiles.get(req.params.source, t, (err, file) => {
+              if (!file) { res.sendStatus(404); return; }
+              res.set('Cache-Control', 'private, max-age=86400');
+              res.sendFile(path.resolve(file));
+            });
+          });
+        }
+
         // Serve frugal-iot-logger data at /data but configure where to get them.
         console.log("Serving /data from", config.server.datadir);
         // Nothing ever removed old readings, so a server left running filled its disk - which on a

package/lib/config-for-user.js

@@ -8,6 +8,8 @@
  * possible.
  */
 
+import { sourcesForBrowser } from './tiles.js';
+
 // Sections of the configuration that are never sent to a browser, whatever permissions the user has.
 // config.d itself is not served - the static mounts are publicdir, htmldir, datadir, privatedir and
 // node_modules, so config.d/schema/devices.yaml is a 404 - and buildConfigFor is the only route by
@@ -92,6 +94,8 @@ export function buildConfigFor(config, user) {
   Object.entries(config).forEach(([key, value]) => {
     if (configSectionsNeverServed.includes(key)) {
       // Deliberately absent from the result
+    } else if (key === 'mapping') {
+      // The server's own settings - the browser's version is built below
     } else if (key === 'organizations') {
       Object.entries(value).forEach(([orgid, org]) => {
         if (hasPermissions(user, orgid, 'READ')) {
@@ -102,6 +106,8 @@ export function buildConfigFor(config, user) {
       oo[key] = value;
     }
   });
+  // The browser gets only what the map draws (M-27), and gets it whether or not mapping.yaml exists
+  oo.mapping = { sources: sourcesForBrowser(config.mapping) };
   return oo;
 }
 

package/lib/tile-prefetch.js

@@ -0,0 +1,158 @@
+/*
+ * Fetching, while online, the tiles a Pi will need once it is not (frugal-iot-client
+ * docs/MAPPING_PLAN.md M-21, M-30). For each source without bounds, three layers:
+ *  - each organization's devices' bounding box, at every zoom while it needs no more than
+ *    organizationtiles - so zooming out always finds a map;
+ *  - the same for each project, with projecttiles;
+ *  - a square deviceradius metres around each device, down to devicezoom.
+ * An organization, or even a project, can span the world, so the boxes are bounded by tiles, not
+ * zoom: the whole world stops at zoom 3, a district goes to about 12.
+ *
+ * A source with bounds - imagery for one state, say - gets only the device squares inside them, from
+ * its minzoom: the box layers are what an unbounded map is for.
+ *
+ * Locations are what the logger last heard - it keeps the current value of every topic, retained
+ * ones included - so this needs no subscription of its own.
+ */
+
+import { eachLimit } from 'async';
+import { tileInSource, tileKey, tileX, tileY } from './tiles.js';
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+const CHECK_MS = 10 * 60 * 1000;          // how soon a moved device's area is fetched
+const FIRST_CHECK_MS = 2 * 60 * 1000;     // retained locations arrive just after the logger connects
+const BOX_MAX_ZOOM = 16;                  // OSM's policy: deep tiles only in the small device squares
+const METRES_PER_DEGREE = 111320;
+
+export const PREFETCH_DEFAULTS = {
+  organizationtiles: 64,
+  projecttiles: 256,
+  deviceradius: 250,
+  devicezoom: 19,
+};
+
+function boxRange(box, z) {
+  return { z, x0: tileX(box.west, z), x1: tileX(box.east, z), y0: tileY(box.north, z), y1: tileY(box.south, z) };
+}
+const rangeCount = (r) => ((r.x1 - r.x0) + 1) * ((r.y1 - r.y0) + 1);
+
+function addRange(tiles, src, r) {
+  for (let x = r.x0; x <= r.x1; x++) {
+    for (let y = r.y0; y <= r.y1; y++) {
+      const t = { z: r.z, x, y };
+      if (tileInSource(t, src)) tiles.set(tileKey(src.id, t), { source: src.id, t });
+    }
+  }
+}
+
+// Each zoom from 0 while the box fits in budget tiles
+export function boxRanges(box, budget, maxzoom = BOX_MAX_ZOOM) {
+  const ranges = [];
+  for (let z = 0; z <= maxzoom; z++) {
+    const r = boxRange(box, z);
+    if (rangeCount(r) > budget) break;
+    ranges.push(r);
+  }
+  return ranges;
+}
+
+export function deviceSquare({ lat, lon }, radius) {
+  const dLat = radius / METRES_PER_DEGREE;
+  const dLon = radius / (METRES_PER_DEGREE * Math.max(Math.cos((lat * Math.PI) / 180), 0.01));
+  return { south: lat - dLat, north: lat + dLat, west: lon - dLon, east: lon + dLon };
+}
+
+function bounds(locs) {
+  return {
+    south: Math.min(...locs.map((l) => l.lat)), north: Math.max(...locs.map((l) => l.lat)),
+    west: Math.min(...locs.map((l) => l.lon)), east: Math.max(...locs.map((l) => l.lon)),
+  };
+}
+
+const insideBounds = (l, b) => !b || ((l.lat >= b[0][0]) && (l.lat <= b[1][0]) && (l.lon >= b[0][1]) && (l.lon <= b[1][1]));
+
+// locations: [{ org, project, node, lat, lon }], sources: enabledSources() ->
+// Map 'source/z/x/y' -> { source, t: { z, x, y } }
+export function wantedTiles(locations, settings, sources) {
+  const s = { ...PREFETCH_DEFAULTS, ...settings };
+  const tiles = new Map();
+  const groups = (keyOf) => locations.reduce((g, l) => { (g[keyOf(l)] = g[keyOf(l)] || []).push(l); return g; }, {});
+  Object.values(sources).forEach((src) => {
+    if (!src.bounds) {
+      Object.values(groups((l) => l.org)).forEach((locs) =>
+        boxRanges(bounds(locs), s.organizationtiles).forEach((r) => addRange(tiles, src, r)));
+      Object.values(groups((l) => `${l.org}/${l.project}`)).forEach((locs) =>
+        boxRanges(bounds(locs), s.projecttiles).forEach((r) => addRange(tiles, src, r)));
+    }
+    locations.filter((l) => insideBounds(l, src.bounds)).forEach((l) => {
+      const square = deviceSquare(l, s.deviceradius);
+      for (let z = src.minzoom; z <= Math.min(s.devicezoom, src.maxzoom); z++) addRange(tiles, src, boxRange(square, z));
+    });
+  });
+  return tiles;
+}
+
+// As the client decides it (core.js MqttTopicNode.location): a GPS with a fix wins over a set location
+export function locationOf({ location, latitude, longitude, satellites }) {
+  let lat; let lon;
+  if ((Number(satellites) > 0) && Number.isFinite(Number(latitude)) && Number.isFinite(Number(longitude))) {
+    lat = Number(latitude); lon = Number(longitude);
+  } else if (typeof location === 'string') {
+    [lat, lon] = location.split(',').map(Number);
+  }
+  return (Number.isFinite(lat) && Number.isFinite(lon) && (Math.abs(lat) <= 90) && (Math.abs(lon) <= 180))
+    ? { lat, lon } : null;
+}
+
+export function loggerLocations(mqttLogger) {
+  const out = [];
+  Object.entries(mqttLogger.reportNodes()).forEach(([org, projects]) => {
+    Object.entries(projects).forEach(([project, nodes]) => {
+      Object.keys(nodes).forEach((node) => {
+        const v = (field) => mqttLogger.getPropertyValue(org, project, node, field);
+        const loc = locationOf({ location: v('frugal_iot/location'), latitude: v('gps/latitude'),
+          longitude: v('gps/longitude'), satellites: v('gps/satellites') });
+        if (loc) out.push({ org, project, node, ...loc });
+      });
+    });
+  });
+  return out;
+}
+
+// locations() is asked at each check. Runs when the locations change, and daily to refresh; a run
+// with failures (offline, most likely) is tried again at the next check.
+export function createPrefetch(cache, settings, locations, now = Date.now) {
+  const s = { ...PREFETCH_DEFAULTS, ...settings };
+  let wanted = new Map();
+  let done = { signature: null, at: 0 };
+  let running = false;
+  cache.setKeep((key) => wanted.has(key));
+
+  function check(cb = () => {}) {
+    if (running) return cb(null, null);
+    const locs = locations();
+    const tiles = wanted = wantedTiles(locs, s, cache.sources);
+    const signature = locs.map((l) => `${l.org}/${l.project}/${l.node}=${l.lat},${l.lon}`).sort().join(';');
+    if ((signature === done.signature) && ((now() - done.at) < DAY_MS)) return cb(null, null);
+    if (!Object.keys(cache.sources).some((id) => cache.online(id))) return cb(null, null);
+    running = true;
+    const counts = { kept: 0, fetched: 0, failed: 0 };
+    // Two at a time, so the stat calls stay few and a view never queues behind a thousand tiles
+    eachLimit([...tiles.values()], 2, (w, cb1) => cache.prefetch(w.source, w.t, (err, result) => { counts[result] += 1; cb1(); }), () => {
+      running = false;
+      if (!counts.failed) done = { signature, at: now() };
+      if (counts.fetched || counts.failed) {
+        console.log('Tile prefetch:', tiles.size, 'wanted,', counts.fetched, 'fetched,', counts.failed, 'failed');
+      }
+      cb(null, counts);
+    });
+  }
+  return { check };
+}
+
+export function startPrefetch(cache, settings, locations) {
+  const p = createPrefetch(cache, settings, locations);
+  setTimeout(() => p.check(), FIRST_CHECK_MS).unref();
+  setInterval(() => p.check(), CHECK_MS).unref();
+  return p;
+}