frugal-iot-server 2.0.10 → 2.0.11

package/config.d/mapping.yaml

@@ -0,0 +1,56 @@
+# The map: where its tiles come from, and whether this server keeps them.
+
+# Each source is a set of tiles the map can draw. Sources named "Map" are drawn everywhere; any other
+# name (e.g. "Imagery") is drawn over the map where the source has tiles, and the map offers a switch
+# between them. Sources sharing a name switch together. bounds ([[south, west], [north, east]]) and
+# minzoom are optional - outside them nothing is asked for.
+sources:
+  osm:
+    name: Map
+    enabled: true
+    upstream: https://tile.openstreetmap.org/{z}/{x}/{y}.png
+    attribution: '&copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors'
+    maxzoom: 19
+  nsw:
+    name: Imagery
+    # Aerial imagery of New South Wales from NSW Spatial Services, free and with no key.
+    # Licence: Creative Commons Attribution 4.0, though their own information sheet links the
+    # no-derivatives version - https://www.spatial.nsw.gov.au/__data/assets/pdf_file/0005/202559/NSW_Imagery_rebranded.pdf
+    # Showing it with the attribution is fine either way. It is unclear whether caching it here
+    # (cache and prefetch, below) constitutes redistribution, or is just good behaviour.
+    enabled: false
+    upstream: https://maps.six.nsw.gov.au/arcgis/rest/services/public/NSW_Imagery/MapServer/tile/{z}/{y}/{x}
+    attribution: '&copy; State Government of NSW and Spatial Services (DCS)'
+    bounds: [[-38.4, 138.4], [-26.5, 159.3]]
+    minzoom: 15
+    maxzoom: 21
+
+# Keeping tiles on this server. Off, the browser fetches them from each source itself.
+# On, this server keeps every tile anyone looks at and serves it to the map from then on, so a Pi
+# that goes offline still has a map. On a server that is always online it means one fetch from a
+# source per tile rather than one per browser.
+cache:
+  enabled: false
+  # Not under datadir: that is served at /data, and its housekeeping walks every file in it
+  dir: ./tiles
+  # A kept tile is checked with its source after this many days. Never less than 7 - OSM asks that.
+  # A check that finds it unchanged costs no rewrite, and offline the old tile is served regardless.
+  refreshdays: 30
+  # Tiles looked at beyond this are dropped once a day, least recently used first. A tile is 5-50 KB.
+  maxmb: 200
+
+# Fetching, while online, the tiles nobody has looked at yet - for a Pi that will go offline. Needs
+# the cache on. Not needed on a server that is always online. Tiles it fetches are never dropped
+# while still wanted.
+prefetch:
+  enabled: false
+  # Each organization's area, at every zoom while it fits in this many tiles - zoom 3 for the
+  # whole world, about 12 for a district. So zooming out always finds a map. Not for sources with
+  # bounds, which get only the squares around devices inside them.
+  organizationtiles: 64
+  # The same for each project's area
+  projecttiles: 256
+  # A square this many metres around each device, down to devicezoom (19 is OSM's deepest).
+  # About 100 tiles, 1-2 MB, per device and source. Each zoom deeper is four times as many.
+  deviceradius: 250
+  devicezoom: 19

package/config.d/schema/modules.yaml

@@ -339,6 +339,7 @@ frugal_iot:
     - leaf:     id
     - leaf:     name
     - leaf:     description
+    - leaf:     location
     - leaf:     lastseen
 gps:
   name: GPS

package/config.d/schema/topics.yaml

@@ -411,6 +411,16 @@ loadcell:
   rw:       r
   calibrate:  true
   graphable:  true
+# "lat,lon" in one payload, so a pin is never moved half-way; set from the UI, held by the node
+location:
+  leaf:     location
+  slot:     location
+  name:     Location
+  type:     text
+  log:      false
+  display:  text
+  rw:       w
+  retain:   true
 longitude:
   leaf:     longitude
   name:     Longitude

package/frugal-iot-server.js

@@ -125,6 +125,8 @@ import { APIError } from './lib/api-errors.js';
 import { access, constants, createReadStream, mkdir, readdir, readFile, rm } from 'fs'; // https://nodejs.org/api/fs.html
 import { createGunzip } from 'zlib'; // https://nodejs.org/api/zlib.html - for serving compressed readings
 import { startHousekeeping } from './lib/housekeeping.js';
+import { createTileCache, parseTile, startTileHousekeeping } from './lib/tiles.js';
+import { loggerLocations, startPrefetch } from './lib/tile-prefetch.js';
 import { detectSeries } from 'async'; // https://caolan.github.io/async/v3/docs.html
 import { createMD5 } from 'hash-wasm';
 import multer from 'multer'; // https://www.npmjs.com/package/multer
@@ -945,12 +947,8 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
     "Content-Length", "0"]
      */
 
-    // Just log the request for now
-    // noinspection JSCheckFunctionSignatures
-    app.use('/', (req, res, next) => {
-      console.log(redactUrl(req.url));
-      next();
-    })
+    // No request logging of its own: morgan, above, is the one switch for it, and on an SD card
+    // every line is a write.
 
     // Nodes enrol here to get their own broker credential. Deliberately not behind a session: a node
     // has no session, and this is what replaces the shared password compiled into its firmware.
@@ -1847,7 +1845,6 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
         const routerDashboard = express.Router();
         app.use('/dashboard', routerDashboard);
         routerDashboard.use(
-          (req,res,next) => {console.log("/dashboard handler for", req.url); next(); }, // Log attempt
           shouldIBeLoggedIn, // redirect to ./login.html if not logged in then back here
           //(req,res,next) => {console.log("XXX back to /dashboard handler for", req.url); next(); }, // Log attempt
           //(req,res,next) => {console.log("XXX", config.server.htmldir); next(); }, // Log attempt
@@ -1856,6 +1853,27 @@ mqttLogger.readYamlConfig('.', (err, configobj) => {
           express.static(config.server.htmldir, revalidate) // Serve static
         );
 
+        // Map tiles, kept here so the map works offline. Behind the login, or every server would be a
+        // free proxy to OpenStreetMap. Off unless config.d/mapping.yaml turns it on.
+        const mapping = config.mapping || {};
+        if (mapping.cache && mapping.cache.enabled) {
+          const tiles = createTileCache({ ...mapping.cache, sources: mapping.sources });
+          console.log("Serving /tiles from", tiles.dir, "for", Object.keys(tiles.sources).join(", "));
+          startTileHousekeeping(tiles);
+          if (mapping.prefetch && mapping.prefetch.enabled) {
+            startPrefetch(tiles, mapping.prefetch, () => loggerLocations(mqttLogger));
+          }
+          app.get('/tiles/:source/:z/:x/:y', loggedInOrFail, (req, res) => {
+            const t = parseTile(req.params.z, req.params.x, req.params.y);
+            if (!t) { res.sendStatus(404); return; }
+            tiles.get(req.params.source, t, (err, file) => {
+              if (!file) { res.sendStatus(404); return; }
+              res.set('Cache-Control', 'private, max-age=86400');
+              res.sendFile(path.resolve(file));
+            });
+          });
+        }
+
         // Serve frugal-iot-logger data at /data but configure where to get them.
         console.log("Serving /data from", config.server.datadir);
         // Nothing ever removed old readings, so a server left running filled its disk - which on a

package/lib/config-for-user.js

@@ -8,6 +8,8 @@
  * possible.
  */
 
+import { sourcesForBrowser } from './tiles.js';
+
 // Sections of the configuration that are never sent to a browser, whatever permissions the user has.
 // config.d itself is not served - the static mounts are publicdir, htmldir, datadir, privatedir and
 // node_modules, so config.d/schema/devices.yaml is a 404 - and buildConfigFor is the only route by
@@ -92,6 +94,8 @@ export function buildConfigFor(config, user) {
   Object.entries(config).forEach(([key, value]) => {
     if (configSectionsNeverServed.includes(key)) {
       // Deliberately absent from the result
+    } else if (key === 'mapping') {
+      // The server's own settings - the browser's version is built below
     } else if (key === 'organizations') {
       Object.entries(value).forEach(([orgid, org]) => {
         if (hasPermissions(user, orgid, 'READ')) {
@@ -102,6 +106,8 @@ export function buildConfigFor(config, user) {
       oo[key] = value;
     }
   });
+  // The browser gets only what the map draws (M-27), and gets it whether or not mapping.yaml exists
+  oo.mapping = { sources: sourcesForBrowser(config.mapping) };
   return oo;
 }
 

package/lib/tile-prefetch.js

@@ -0,0 +1,158 @@
+/*
+ * Fetching, while online, the tiles a Pi will need once it is not (frugal-iot-client
+ * docs/MAPPING_PLAN.md M-21, M-30). For each source without bounds, three layers:
+ *  - each organization's devices' bounding box, at every zoom while it needs no more than
+ *    organizationtiles - so zooming out always finds a map;
+ *  - the same for each project, with projecttiles;
+ *  - a square deviceradius metres around each device, down to devicezoom.
+ * An organization, or even a project, can span the world, so the boxes are bounded by tiles, not
+ * zoom: the whole world stops at zoom 3, a district goes to about 12.
+ *
+ * A source with bounds - imagery for one state, say - gets only the device squares inside them, from
+ * its minzoom: the box layers are what an unbounded map is for.
+ *
+ * Locations are what the logger last heard - it keeps the current value of every topic, retained
+ * ones included - so this needs no subscription of its own.
+ */
+
+import { eachLimit } from 'async';
+import { tileInSource, tileKey, tileX, tileY } from './tiles.js';
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+const CHECK_MS = 10 * 60 * 1000;          // how soon a moved device's area is fetched
+const FIRST_CHECK_MS = 2 * 60 * 1000;     // retained locations arrive just after the logger connects
+const BOX_MAX_ZOOM = 16;                  // OSM's policy: deep tiles only in the small device squares
+const METRES_PER_DEGREE = 111320;
+
+export const PREFETCH_DEFAULTS = {
+  organizationtiles: 64,
+  projecttiles: 256,
+  deviceradius: 250,
+  devicezoom: 19,
+};
+
+function boxRange(box, z) {
+  return { z, x0: tileX(box.west, z), x1: tileX(box.east, z), y0: tileY(box.north, z), y1: tileY(box.south, z) };
+}
+const rangeCount = (r) => ((r.x1 - r.x0) + 1) * ((r.y1 - r.y0) + 1);
+
+function addRange(tiles, src, r) {
+  for (let x = r.x0; x <= r.x1; x++) {
+    for (let y = r.y0; y <= r.y1; y++) {
+      const t = { z: r.z, x, y };
+      if (tileInSource(t, src)) tiles.set(tileKey(src.id, t), { source: src.id, t });
+    }
+  }
+}
+
+// Each zoom from 0 while the box fits in budget tiles
+export function boxRanges(box, budget, maxzoom = BOX_MAX_ZOOM) {
+  const ranges = [];
+  for (let z = 0; z <= maxzoom; z++) {
+    const r = boxRange(box, z);
+    if (rangeCount(r) > budget) break;
+    ranges.push(r);
+  }
+  return ranges;
+}
+
+export function deviceSquare({ lat, lon }, radius) {
+  const dLat = radius / METRES_PER_DEGREE;
+  const dLon = radius / (METRES_PER_DEGREE * Math.max(Math.cos((lat * Math.PI) / 180), 0.01));
+  return { south: lat - dLat, north: lat + dLat, west: lon - dLon, east: lon + dLon };
+}
+
+function bounds(locs) {
+  return {
+    south: Math.min(...locs.map((l) => l.lat)), north: Math.max(...locs.map((l) => l.lat)),
+    west: Math.min(...locs.map((l) => l.lon)), east: Math.max(...locs.map((l) => l.lon)),
+  };
+}
+
+const insideBounds = (l, b) => !b || ((l.lat >= b[0][0]) && (l.lat <= b[1][0]) && (l.lon >= b[0][1]) && (l.lon <= b[1][1]));
+
+// locations: [{ org, project, node, lat, lon }], sources: enabledSources() ->
+// Map 'source/z/x/y' -> { source, t: { z, x, y } }
+export function wantedTiles(locations, settings, sources) {
+  const s = { ...PREFETCH_DEFAULTS, ...settings };
+  const tiles = new Map();
+  const groups = (keyOf) => locations.reduce((g, l) => { (g[keyOf(l)] = g[keyOf(l)] || []).push(l); return g; }, {});
+  Object.values(sources).forEach((src) => {
+    if (!src.bounds) {
+      Object.values(groups((l) => l.org)).forEach((locs) =>
+        boxRanges(bounds(locs), s.organizationtiles).forEach((r) => addRange(tiles, src, r)));
+      Object.values(groups((l) => `${l.org}/${l.project}`)).forEach((locs) =>
+        boxRanges(bounds(locs), s.projecttiles).forEach((r) => addRange(tiles, src, r)));
+    }
+    locations.filter((l) => insideBounds(l, src.bounds)).forEach((l) => {
+      const square = deviceSquare(l, s.deviceradius);
+      for (let z = src.minzoom; z <= Math.min(s.devicezoom, src.maxzoom); z++) addRange(tiles, src, boxRange(square, z));
+    });
+  });
+  return tiles;
+}
+
+// As the client decides it (core.js MqttTopicNode.location): a GPS with a fix wins over a set location
+export function locationOf({ location, latitude, longitude, satellites }) {
+  let lat; let lon;
+  if ((Number(satellites) > 0) && Number.isFinite(Number(latitude)) && Number.isFinite(Number(longitude))) {
+    lat = Number(latitude); lon = Number(longitude);
+  } else if (typeof location === 'string') {
+    [lat, lon] = location.split(',').map(Number);
+  }
+  return (Number.isFinite(lat) && Number.isFinite(lon) && (Math.abs(lat) <= 90) && (Math.abs(lon) <= 180))
+    ? { lat, lon } : null;
+}
+
+export function loggerLocations(mqttLogger) {
+  const out = [];
+  Object.entries(mqttLogger.reportNodes()).forEach(([org, projects]) => {
+    Object.entries(projects).forEach(([project, nodes]) => {
+      Object.keys(nodes).forEach((node) => {
+        const v = (field) => mqttLogger.getPropertyValue(org, project, node, field);
+        const loc = locationOf({ location: v('frugal_iot/location'), latitude: v('gps/latitude'),
+          longitude: v('gps/longitude'), satellites: v('gps/satellites') });
+        if (loc) out.push({ org, project, node, ...loc });
+      });
+    });
+  });
+  return out;
+}
+
+// locations() is asked at each check. Runs when the locations change, and daily to refresh; a run
+// with failures (offline, most likely) is tried again at the next check.
+export function createPrefetch(cache, settings, locations, now = Date.now) {
+  const s = { ...PREFETCH_DEFAULTS, ...settings };
+  let wanted = new Map();
+  let done = { signature: null, at: 0 };
+  let running = false;
+  cache.setKeep((key) => wanted.has(key));
+
+  function check(cb = () => {}) {
+    if (running) return cb(null, null);
+    const locs = locations();
+    const tiles = wanted = wantedTiles(locs, s, cache.sources);
+    const signature = locs.map((l) => `${l.org}/${l.project}/${l.node}=${l.lat},${l.lon}`).sort().join(';');
+    if ((signature === done.signature) && ((now() - done.at) < DAY_MS)) return cb(null, null);
+    if (!Object.keys(cache.sources).some((id) => cache.online(id))) return cb(null, null);
+    running = true;
+    const counts = { kept: 0, fetched: 0, failed: 0 };
+    // Two at a time, so the stat calls stay few and a view never queues behind a thousand tiles
+    eachLimit([...tiles.values()], 2, (w, cb1) => cache.prefetch(w.source, w.t, (err, result) => { counts[result] += 1; cb1(); }), () => {
+      running = false;
+      if (!counts.failed) done = { signature, at: now() };
+      if (counts.fetched || counts.failed) {
+        console.log('Tile prefetch:', tiles.size, 'wanted,', counts.fetched, 'fetched,', counts.failed, 'failed');
+      }
+      cb(null, counts);
+    });
+  }
+  return { check };
+}
+
+export function startPrefetch(cache, settings, locations) {
+  const p = createPrefetch(cache, settings, locations);
+  setTimeout(() => p.check(), FIRST_CHECK_MS).unref();
+  setInterval(() => p.check(), CHECK_MS).unref();
+  return p;
+}

package/lib/tiles.js

@@ -0,0 +1,271 @@
+/*
+ * Map tiles kept on this server (frugal-iot-client docs/MAPPING_PLAN.md, M-19 to M-30).
+ *
+ * The map asks for /tiles/<source>/z/x/y. A tile kept here is served from disk; one that is not is
+ * fetched from that source's upstream while it is reachable, and is a 404 at once while it is not.
+ * So a Pi that goes offline still shows whatever was looked at while it was online.
+ *
+ * A file's mtime is when the tile was last known to be current, so a check that finds it unchanged
+ * costs one metadata write, not a rewrite. Last use is kept in memory, never read from atime: a Pi
+ * is often mounted noatime to spare its card, and where it is not, atime is a write per view.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { priorityQueue } from 'async';
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+// Waiting on a timeout for every tile would freeze the map, so one failure stands for a minute
+const OFFLINE_RETRY_MS = 60 * 1000;
+const FETCH_TIMEOUT_MS = 15 * 1000;
+const CONCURRENT_FETCHES = 2;     // OSM's tile usage policy
+const VIEW = 0;                   // someone is looking - ahead of any prefetch
+const PREFETCH = 1;
+const MIN_REFRESH_DAYS = 7;       // OSM's tile usage policy: never re-ask sooner than this
+const EXTENSIONS = { 'image/png': '.png', 'image/jpeg': '.jpg', 'image/webp': '.webp' };
+const { version } = JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
+
+export const CACHE_DEFAULTS = {
+  dir: './tiles',
+  refreshdays: 30,
+  maxmb: 200,
+  userAgent: `frugal-iot-server/${version} (+https://github.com/mitra42/frugal-iot-server)`,
+};
+// What a mapping.yaml with no sources gets: the map as it always was
+export const DEFAULT_SOURCES = {
+  osm: {
+    name: 'Map',
+    enabled: true,
+    upstream: 'https://tile.openstreetmap.org/{z}/{x}/{y}.png',
+    attribution: '&copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors',
+    maxzoom: 19,
+  },
+};
+
+// The enabled sources, by id. The id becomes a directory and a URL segment, so it is held to letters
+// and digits.
+export function enabledSources(mapping) {
+  const sources = (mapping && mapping.sources) || DEFAULT_SOURCES;
+  const out = {};
+  Object.entries(sources).forEach(([id, src]) => {
+    if (!src || (src.enabled === false) || !/^[a-z0-9_-]{1,32}$/i.test(id) || !src.upstream) return;
+    out[id] = { id, name: src.name || id, minzoom: 0, maxzoom: 19, ...src };
+  });
+  return out;
+}
+
+// What the map is told about each source (M-27): through this server while it keeps tiles,
+// straight from upstream while it does not
+export function sourcesForBrowser(mapping) {
+  const cached = !!(mapping && mapping.cache && mapping.cache.enabled);
+  return Object.values(enabledSources(mapping)).map((s) => ({
+    id: s.id, name: s.name, attribution: s.attribution, minzoom: s.minzoom, maxzoom: s.maxzoom,
+    url: cached ? `/tiles/${s.id}/{z}/{x}/{y}` : s.upstream,
+    ...(s.bounds ? { bounds: s.bounds } : {}),
+  }));
+}
+
+const clamp = (v, z) => Math.min(Math.max(v, 0), (2 ** z) - 1);
+export function tileX(lon, z) {
+  return clamp(Math.floor(((lon + 180) / 360) * (2 ** z)), z);
+}
+export function tileY(lat, z) {
+  const r = (lat * Math.PI) / 180;
+  return clamp(Math.floor(((1 - (Math.log(Math.tan(r) + (1 / Math.cos(r))) / Math.PI)) / 2) * (2 ** z)), z);
+}
+
+// Within the source's zooms, and touching its bounds (bounds: [[south, west], [north, east]])
+export function tileInSource(t, src) {
+  if ((t.z < src.minzoom) || (t.z > src.maxzoom)) return false;
+  if (!src.bounds) return true;
+  const [[south, west], [north, east]] = src.bounds;
+  return (t.x >= tileX(west, t.z)) && (t.x <= tileX(east, t.z))
+    && (t.y >= tileY(north, t.z)) && (t.y <= tileY(south, t.z));
+}
+
+// z, x and y straight from the URL. Digits only, so none of it can name a path elsewhere.
+export function parseTile(z, x, y, maxzoom = 22) {
+  const m = /^(\d{1,7})(\.(png|jpg|jpeg|webp))?$/.exec(String(y));
+  if (!m || !/^\d{1,2}$/.test(String(z)) || !/^\d{1,7}$/.test(String(x))) return null;
+  const t = { z: Number(z), x: Number(x), y: Number(m[1]) };
+  const n = 2 ** t.z;
+  return ((t.z <= maxzoom) && (t.x < n) && (t.y < n)) ? t : null;
+}
+
+export const tileKey = (id, t) => `${id}/${t.z}/${t.x}/${t.y}`;
+
+// The network edge, replaced in tests. cb(err) only when upstream could not be reached at all.
+function fetchUpstream(url, { since, userAgent }, cb) {
+  const headers = { 'User-Agent': userAgent };
+  if (since) headers['If-Modified-Since'] = since.toUTCString();
+  fetch(url, { headers, signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) })
+    .then((r) => (r.ok
+      ? r.arrayBuffer().then((b) => ({ status: r.status, type: r.headers.get('content-type'), body: Buffer.from(b) }))
+      : { status: r.status }))
+    .then((res) => cb(null, res), (err) => cb(err));
+}
+
+export function createTileCache(options = {}) {
+  const o = { ...CACHE_DEFAULTS, ...options };
+  const sources = o.sources ? enabledSources({ sources: o.sources }) : enabledSources(null);
+  const fetchTile = o.fetchTile || fetchUpstream;
+  const now = o.now || Date.now;
+  const refreshMs = Math.max(o.refreshdays, MIN_REFRESH_DAYS) * DAY_MS;
+  let keep = o.keep || (() => false);     // keys the prefetch still wants are never evicted
+  const lastUsed = new Map();             // file -> ms
+  const waiting = new Map();              // key -> [cb], so two views of one tile fetch it once
+  const offlineUntil = {};                // by source: NSW being down does not stop OSM
+  const lastExt = {};                     // by source, so a kept tile is usually found in one stat
+
+  const base = (id, t) => path.join(o.dir, id, String(t.z), String(t.x), String(t.y));
+  const urlOf = (src, t) => src.upstream.replace('{z}', t.z).replace('{x}', t.x).replace('{y}', t.y);
+  const online = (id) => now() >= (offlineUntil[id] || 0);
+
+  const fetches = priorityQueue((job, cb) => fetchTile(job.url, { since: job.since, userAgent: o.userAgent }, cb),
+    CONCURRENT_FETCHES);
+
+  // cb(null, {file, st}) for the kept file, whichever type it was stored as, or cb(null, null)
+  function find(id, t, cb) {
+    const exts = [...new Set([lastExt[id] || '.png', ...Object.values(EXTENSIONS)])];
+    let i = 0;
+    const next = () => {
+      if (i >= exts.length) return cb(null, null);
+      const file = base(id, t) + exts[i++];
+      fs.stat(file, (err, st) => {
+        if (err) return next();
+        lastExt[id] = path.extname(file);
+        cb(null, { file, st });
+      });
+    };
+    next();
+  }
+
+  function store(id, t, res, cb) {
+    const ext = EXTENSIONS[String(res.type).split(';')[0].trim()] || '.png';
+    const file = base(id, t) + ext;
+    const tmp = `${file}.tmp`;   // renamed into place, so a half-written tile is never served
+    fs.mkdir(path.dirname(file), { recursive: true }, (err) => {
+      if (err) return cb(err);
+      fs.writeFile(tmp, res.body, (err1) => (err1 ? cb(err1) : fs.rename(tmp, file, (err2) => {
+        if (!err2) lastExt[id] = ext;
+        cb(err2, file);
+      })));
+    });
+  }
+
+  // cb(null, file) when the tile is now current, cb(null, null) when it is not
+  function refresh(src, t, kept, priority, cb) {
+    const key = tileKey(src.id, t);
+    if (!online(src.id)) return cb(null, null);
+    if (waiting.has(key)) { waiting.get(key).push(cb); return; }
+    waiting.set(key, [cb]);
+    const done = (err, file) => {
+      const cbs = waiting.get(key);
+      waiting.delete(key);
+      if (err) console.log('Tile', key, 'not kept:', err.message);
+      cbs.forEach((c) => c(null, err ? null : file));
+    };
+    fetches.push({ url: urlOf(src, t), since: kept && kept.st.mtime }, priority, (err, res) => {
+      if (err || (res.status === 429) || (res.status >= 500)) {
+        offlineUntil[src.id] = now() + OFFLINE_RETRY_MS;
+        return done(null, null);
+      }
+      if ((res.status === 304) && kept) {
+        const d = new Date(now());
+        return fs.utimes(kept.file, d, d, (err1) => done(err1, kept.file));
+      }
+      if (res.status === 200) return store(src.id, t, res, done);
+      done(null, null);   // e.g. 404: upstream has no such tile
+    });
+  }
+
+  // cb(null, file) to serve, or cb(null, null) for a 404. A kept tile is served at once even when
+  // due a check - the check runs behind it, and offline a stale tile beats none.
+  function get(id, t, cb) {
+    const src = sources[id];
+    if (!src || !tileInSource(t, src)) return cb(null, null);
+    find(id, t, (err, kept) => {
+      if (kept) {
+        lastUsed.set(kept.file, now());
+        if ((now() - kept.st.mtimeMs) > refreshMs) refresh(src, t, kept, VIEW, () => {});
+        return cb(null, kept.file);
+      }
+      refresh(src, t, null, VIEW, (err1, file) => {
+        if (file) lastUsed.set(file, now());
+        cb(null, file);
+      });
+    });
+  }
+
+  // Fetch a tile nobody is looking at yet, if it is missing or due. cb(null, 'kept' | 'fetched' | 'failed')
+  function prefetch(id, t, cb) {
+    const src = sources[id];
+    if (!src || !tileInSource(t, src)) return cb(null, 'failed');
+    find(id, t, (err, kept) => {
+      if (kept && ((now() - kept.st.mtimeMs) <= refreshMs)) return cb(null, 'kept');
+      refresh(src, t, kept, PREFETCH, (err1, file) => cb(null, file ? 'fetched' : 'failed'));
+    });
+  }
+
+  function listFiles(cb) {
+    const found = [];
+    const walk = (d, cb1) => fs.readdir(d, { withFileTypes: true }, (err, entries) => {
+      if (err) return cb1(err.code === 'ENOENT' ? null : err);
+      let i = 0;
+      const next = (err1) => {
+        if (err1 || (i >= entries.length)) return cb1(err1);
+        const e = entries[i++];
+        const full = path.join(d, e.name);
+        if (e.isDirectory()) return walk(full, next);
+        fs.stat(full, (err2, st) => {
+          if (!err2) found.push({ file: full, size: st.size, used: lastUsed.get(full) || st.mtimeMs });
+          next();
+        });
+      };
+      next();
+    });
+    walk(o.dir, (err) => cb(err, found));
+  }
+  // dir/osm/15/26000/16900.png -> osm/15/26000/16900
+  const keyOfFile = (file) => path.relative(o.dir, file).split(path.sep).join('/').replace(/\.[a-z]+$/, '');
+
+  // Least recently used first, until under maxmb, across every source. After a restart, a tile not
+  // yet viewed counts as last used when it was last known current.
+  function evict(cb) {
+    listFiles((err, files) => {
+      if (err) return cb(err);
+      let total = files.reduce((s, f) => s + f.size, 0);
+      const limit = o.maxmb * 1024 * 1024;
+      const victims = [];
+      files.sort((a, b) => a.used - b.used);
+      for (const f of files) {
+        if (total <= limit) break;
+        if (keep(keyOfFile(f.file))) continue;
+        victims.push(f.file);
+        total -= f.size;
+      }
+      let i = 0;
+      const next = () => {
+        if (i >= victims.length) return cb(null, { removed: victims.length, bytes: total });
+        const file = victims[i++];
+        lastUsed.delete(file);
+        fs.unlink(file, () => next());
+      };
+      next();
+    });
+  }
+
+  return { get, prefetch, evict, online, sources, setKeep: (fn) => { keep = fn; }, dir: o.dir };
+}
+
+// Once a day, like the readings' housekeeping - first pass delayed past the busy startup
+export function startTileHousekeeping(cache) {
+  const run = () => cache.evict((err, r) => {
+    if (err) console.error('Tile housekeeping:', err.message);
+    else if (r.removed) console.log('Tile housekeeping:', r.removed, 'tiles dropped');
+  });
+  setTimeout(run, 5 * 60 * 1000).unref();
+  const timer = setInterval(run, DAY_MS);
+  timer.unref();
+  return timer;
+}

package/package.json

@@ -12,8 +12,8 @@
     "cookie-parser": "^1.4.7",
     "express": "^5.2.1",
     "express-session": "^1.19.0",
-    "frugal-iot-client": "^2.0.10",
-    "frugal-iot-logger": "^2.0.10",
+    "frugal-iot-client": "^2.0.11",
+    "frugal-iot-logger": "^2.0.11",
     "hash-wasm": "^4.12.0",
     "js-yaml": "^4.2.0",
     "morgan": "^1.11.0",
@@ -47,6 +47,7 @@
     "config.yaml",
     "config.d/email.yaml",
     "config.d/logger.yaml",
+    "config.d/mapping.yaml",
     "config.d/mqtt.yaml",
     "config.d/opensearch.yaml",
     "config.d/server.yaml",
@@ -94,7 +95,7 @@
     "check-files": "node scripts/check-published-files.js"
   },
   "type": "module",
-  "version": "2.0.10",
+  "version": "2.0.11",
   "overrides": {
     "uuid": "^11.1.1"
   }

package/public/service-worker.js

@@ -6,7 +6,7 @@
 // it, an installed browser keeps serving the files it already cached - and since the list now covers
 // the dashboard's own code, that means old JS behind freshly fetched HTML. Any edit to the list
 // needs this bumped, or it does not reach anyone who already has the old one.
-const CACHE_NAME = 'frugal-iot-cache-2.0.10';
+const CACHE_NAME = 'frugal-iot-cache-2.0.11';
 /*
  * Everything the app needs, so that a phone on a poor or expensive link fetches it once.
  *

package/scripts/init.zsh

@@ -60,7 +60,7 @@ copy_if_missing() {
 
 echo "Configuration:"
 copy_if_missing "${PKG}/config.yaml" "./config.yaml"
-for f in logger.yaml mqtt.yaml server.yaml email.yaml; do
+for f in logger.yaml mqtt.yaml server.yaml email.yaml mapping.yaml; do
   copy_if_missing "${PKG}/config.d/${f}" "./config.d/${f}"
 done
 # The schema describes the sensor types the software understands, so a release changing it matters

package/scripts/install-openwrt.sh

@@ -635,6 +635,9 @@ CODE=""; i=0
 while [ "$i" -lt 20 ]; do
   CODE=$(curl -s -o /dev/null -w '%{http_code}' -m 5 "http://localhost:${PORT}/" || true)
   [ "$CODE" = 200 ] && break
+  # "/" is the static front page; the login page is the thing that proves the dashboard is served
+  CODE2=$(curl -s -o /dev/null -w '%{http_code}' -m 5 "http://localhost:${PORT}/dashboard/login.html" || true)
+  [ "$CODE2" = 200 ] && { CODE=200; break; }
   sleep 2; i=$((i+1))
 done
 if [ "$CODE" = 200 ]; then

package/scripts/openwrt-preflight.sh

@@ -344,7 +344,8 @@ fi
 # ---------------------------------------------------------------- ports
 head_ "Ports"
 for port in 1883 9012 8080; do
-  if netstat -ln 2>/dev/null | grep -q "[:.]${port} "; then warn "port ${port} is already in use here - something else is listening on it"
+  if netstat -lnp 2>/dev/null | grep "[:.]${port} " | grep -q mosquitto; then info "port ${port} is in use by mosquitto itself (fine)"
+  elif netstat -ln 2>/dev/null | grep -q "[:.]${port} "; then warn "port ${port} is already in use here - something else is listening on it"
   else info "port ${port} is free"; fi
 done