frugal-iot-server 2.0.9 → 2.0.11

package/lib/language-requests.js

@@ -0,0 +1,50 @@
+/*
+ * Which languages people's browsers ask for, counted at each sign-in, so the next language to
+ * translate is the one most asked for (frugal-iot-client docs/LANGUAGES_PLAN.md, Phase 3). Codes
+ * only - nothing records who asked.
+ *
+ * To read it: SELECT * FROM language_requests ORDER BY supported, count DESC;
+ */
+import { existsSync } from 'fs';
+import { join } from 'path';
+
+// A browser sends a handful; anything longer is not a browser
+const MAX_CODES = 10;
+const CODE = /^[a-z]{2,3}(-[a-z0-9]{1,8}){0,3}$/;
+// As languageAliases in the client's core.js: browsers call Tagalog "fil", and its file is tl.yaml
+const ALIASES = { fil: 'tl' };
+
+// The form field is untrusted: keep only what looks like a language tag, lower-cased, once each
+export function parseBrowserLanguages(value) {
+  if (typeof value !== 'string') return [];
+  const codes = value.split(',').map((c) => c.trim().toLowerCase()).filter((c) => CODE.test(c));
+  return [...new Set(codes)].slice(0, MAX_CODES);
+}
+
+// The client has a file per language it supports, so whether one exists is the answer, and the
+// server keeps no list of its own. fr-CA is served by fr, as the client matches it; English is the
+// keys themselves, so it has no file.
+export function isSupported(htmldir, code) {
+  const primary = ALIASES[code.split('-')[0]] || code.split('-')[0];
+  if (primary === 'en') return true;
+  return [code, primary].some((c) => existsSync(join(htmldir, 'languages', `${c}.yaml`)));
+}
+
+// One upsert per code. cb(err) - a failure to count must never stand in the way of a login
+export function countLanguages(db, htmldir, codes, cb = () => {}) {
+  const now = Date.now();
+  let pending = codes.length;
+  let failed = null;
+  if (!pending) { cb(null); return; }
+  codes.forEach((code) => {
+    db.run(
+      `INSERT INTO language_requests (code, count, last_seen, supported) VALUES (?, 1, ?, ?)
+       ON CONFLICT(code) DO UPDATE SET count = count + 1, last_seen = excluded.last_seen,
+                                       supported = excluded.supported`,
+      [code, now, isSupported(htmldir, code) ? 1 : 0],
+      (err) => {
+        failed = failed || err;
+        if (--pending === 0) cb(failed);
+      });
+  });
+}

package/lib/tile-prefetch.js

@@ -0,0 +1,158 @@
+/*
+ * Fetching, while online, the tiles a Pi will need once it is not (frugal-iot-client
+ * docs/MAPPING_PLAN.md M-21, M-30). For each source without bounds, three layers:
+ *  - each organization's devices' bounding box, at every zoom while it needs no more than
+ *    organizationtiles - so zooming out always finds a map;
+ *  - the same for each project, with projecttiles;
+ *  - a square deviceradius metres around each device, down to devicezoom.
+ * An organization, or even a project, can span the world, so the boxes are bounded by tiles, not
+ * zoom: the whole world stops at zoom 3, a district goes to about 12.
+ *
+ * A source with bounds - imagery for one state, say - gets only the device squares inside them, from
+ * its minzoom: the box layers are what an unbounded map is for.
+ *
+ * Locations are what the logger last heard - it keeps the current value of every topic, retained
+ * ones included - so this needs no subscription of its own.
+ */
+
+import { eachLimit } from 'async';
+import { tileInSource, tileKey, tileX, tileY } from './tiles.js';
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+const CHECK_MS = 10 * 60 * 1000;          // how soon a moved device's area is fetched
+const FIRST_CHECK_MS = 2 * 60 * 1000;     // retained locations arrive just after the logger connects
+const BOX_MAX_ZOOM = 16;                  // OSM's policy: deep tiles only in the small device squares
+const METRES_PER_DEGREE = 111320;
+
+export const PREFETCH_DEFAULTS = {
+  organizationtiles: 64,
+  projecttiles: 256,
+  deviceradius: 250,
+  devicezoom: 19,
+};
+
+function boxRange(box, z) {
+  return { z, x0: tileX(box.west, z), x1: tileX(box.east, z), y0: tileY(box.north, z), y1: tileY(box.south, z) };
+}
+const rangeCount = (r) => ((r.x1 - r.x0) + 1) * ((r.y1 - r.y0) + 1);
+
+function addRange(tiles, src, r) {
+  for (let x = r.x0; x <= r.x1; x++) {
+    for (let y = r.y0; y <= r.y1; y++) {
+      const t = { z: r.z, x, y };
+      if (tileInSource(t, src)) tiles.set(tileKey(src.id, t), { source: src.id, t });
+    }
+  }
+}
+
+// Each zoom from 0 while the box fits in budget tiles
+export function boxRanges(box, budget, maxzoom = BOX_MAX_ZOOM) {
+  const ranges = [];
+  for (let z = 0; z <= maxzoom; z++) {
+    const r = boxRange(box, z);
+    if (rangeCount(r) > budget) break;
+    ranges.push(r);
+  }
+  return ranges;
+}
+
+export function deviceSquare({ lat, lon }, radius) {
+  const dLat = radius / METRES_PER_DEGREE;
+  const dLon = radius / (METRES_PER_DEGREE * Math.max(Math.cos((lat * Math.PI) / 180), 0.01));
+  return { south: lat - dLat, north: lat + dLat, west: lon - dLon, east: lon + dLon };
+}
+
+function bounds(locs) {
+  return {
+    south: Math.min(...locs.map((l) => l.lat)), north: Math.max(...locs.map((l) => l.lat)),
+    west: Math.min(...locs.map((l) => l.lon)), east: Math.max(...locs.map((l) => l.lon)),
+  };
+}
+
+const insideBounds = (l, b) => !b || ((l.lat >= b[0][0]) && (l.lat <= b[1][0]) && (l.lon >= b[0][1]) && (l.lon <= b[1][1]));
+
+// locations: [{ org, project, node, lat, lon }], sources: enabledSources() ->
+// Map 'source/z/x/y' -> { source, t: { z, x, y } }
+export function wantedTiles(locations, settings, sources) {
+  const s = { ...PREFETCH_DEFAULTS, ...settings };
+  const tiles = new Map();
+  const groups = (keyOf) => locations.reduce((g, l) => { (g[keyOf(l)] = g[keyOf(l)] || []).push(l); return g; }, {});
+  Object.values(sources).forEach((src) => {
+    if (!src.bounds) {
+      Object.values(groups((l) => l.org)).forEach((locs) =>
+        boxRanges(bounds(locs), s.organizationtiles).forEach((r) => addRange(tiles, src, r)));
+      Object.values(groups((l) => `${l.org}/${l.project}`)).forEach((locs) =>
+        boxRanges(bounds(locs), s.projecttiles).forEach((r) => addRange(tiles, src, r)));
+    }
+    locations.filter((l) => insideBounds(l, src.bounds)).forEach((l) => {
+      const square = deviceSquare(l, s.deviceradius);
+      for (let z = src.minzoom; z <= Math.min(s.devicezoom, src.maxzoom); z++) addRange(tiles, src, boxRange(square, z));
+    });
+  });
+  return tiles;
+}
+
+// As the client decides it (core.js MqttTopicNode.location): a GPS with a fix wins over a set location
+export function locationOf({ location, latitude, longitude, satellites }) {
+  let lat; let lon;
+  if ((Number(satellites) > 0) && Number.isFinite(Number(latitude)) && Number.isFinite(Number(longitude))) {
+    lat = Number(latitude); lon = Number(longitude);
+  } else if (typeof location === 'string') {
+    [lat, lon] = location.split(',').map(Number);
+  }
+  return (Number.isFinite(lat) && Number.isFinite(lon) && (Math.abs(lat) <= 90) && (Math.abs(lon) <= 180))
+    ? { lat, lon } : null;
+}
+
+export function loggerLocations(mqttLogger) {
+  const out = [];
+  Object.entries(mqttLogger.reportNodes()).forEach(([org, projects]) => {
+    Object.entries(projects).forEach(([project, nodes]) => {
+      Object.keys(nodes).forEach((node) => {
+        const v = (field) => mqttLogger.getPropertyValue(org, project, node, field);
+        const loc = locationOf({ location: v('frugal_iot/location'), latitude: v('gps/latitude'),
+          longitude: v('gps/longitude'), satellites: v('gps/satellites') });
+        if (loc) out.push({ org, project, node, ...loc });
+      });
+    });
+  });
+  return out;
+}
+
+// locations() is asked at each check. Runs when the locations change, and daily to refresh; a run
+// with failures (offline, most likely) is tried again at the next check.
+export function createPrefetch(cache, settings, locations, now = Date.now) {
+  const s = { ...PREFETCH_DEFAULTS, ...settings };
+  let wanted = new Map();
+  let done = { signature: null, at: 0 };
+  let running = false;
+  cache.setKeep((key) => wanted.has(key));
+
+  function check(cb = () => {}) {
+    if (running) return cb(null, null);
+    const locs = locations();
+    const tiles = wanted = wantedTiles(locs, s, cache.sources);
+    const signature = locs.map((l) => `${l.org}/${l.project}/${l.node}=${l.lat},${l.lon}`).sort().join(';');
+    if ((signature === done.signature) && ((now() - done.at) < DAY_MS)) return cb(null, null);
+    if (!Object.keys(cache.sources).some((id) => cache.online(id))) return cb(null, null);
+    running = true;
+    const counts = { kept: 0, fetched: 0, failed: 0 };
+    // Two at a time, so the stat calls stay few and a view never queues behind a thousand tiles
+    eachLimit([...tiles.values()], 2, (w, cb1) => cache.prefetch(w.source, w.t, (err, result) => { counts[result] += 1; cb1(); }), () => {
+      running = false;
+      if (!counts.failed) done = { signature, at: now() };
+      if (counts.fetched || counts.failed) {
+        console.log('Tile prefetch:', tiles.size, 'wanted,', counts.fetched, 'fetched,', counts.failed, 'failed');
+      }
+      cb(null, counts);
+    });
+  }
+  return { check };
+}
+
+export function startPrefetch(cache, settings, locations) {
+  const p = createPrefetch(cache, settings, locations);
+  setTimeout(() => p.check(), FIRST_CHECK_MS).unref();
+  setInterval(() => p.check(), CHECK_MS).unref();
+  return p;
+}

package/lib/tiles.js

@@ -0,0 +1,271 @@
+/*
+ * Map tiles kept on this server (frugal-iot-client docs/MAPPING_PLAN.md, M-19 to M-30).
+ *
+ * The map asks for /tiles/<source>/z/x/y. A tile kept here is served from disk; one that is not is
+ * fetched from that source's upstream while it is reachable, and is a 404 at once while it is not.
+ * So a Pi that goes offline still shows whatever was looked at while it was online.
+ *
+ * A file's mtime is when the tile was last known to be current, so a check that finds it unchanged
+ * costs one metadata write, not a rewrite. Last use is kept in memory, never read from atime: a Pi
+ * is often mounted noatime to spare its card, and where it is not, atime is a write per view.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { priorityQueue } from 'async';
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+// Waiting on a timeout for every tile would freeze the map, so one failure stands for a minute
+const OFFLINE_RETRY_MS = 60 * 1000;
+const FETCH_TIMEOUT_MS = 15 * 1000;
+const CONCURRENT_FETCHES = 2;     // OSM's tile usage policy
+const VIEW = 0;                   // someone is looking - ahead of any prefetch
+const PREFETCH = 1;
+const MIN_REFRESH_DAYS = 7;       // OSM's tile usage policy: never re-ask sooner than this
+const EXTENSIONS = { 'image/png': '.png', 'image/jpeg': '.jpg', 'image/webp': '.webp' };
+const { version } = JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
+
+export const CACHE_DEFAULTS = {
+  dir: './tiles',
+  refreshdays: 30,
+  maxmb: 200,
+  userAgent: `frugal-iot-server/${version} (+https://github.com/mitra42/frugal-iot-server)`,
+};
+// What a mapping.yaml with no sources gets: the map as it always was
+export const DEFAULT_SOURCES = {
+  osm: {
+    name: 'Map',
+    enabled: true,
+    upstream: 'https://tile.openstreetmap.org/{z}/{x}/{y}.png',
+    attribution: '&copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors',
+    maxzoom: 19,
+  },
+};
+
+// The enabled sources, by id. The id becomes a directory and a URL segment, so it is held to letters
+// and digits.
+export function enabledSources(mapping) {
+  const sources = (mapping && mapping.sources) || DEFAULT_SOURCES;
+  const out = {};
+  Object.entries(sources).forEach(([id, src]) => {
+    if (!src || (src.enabled === false) || !/^[a-z0-9_-]{1,32}$/i.test(id) || !src.upstream) return;
+    out[id] = { id, name: src.name || id, minzoom: 0, maxzoom: 19, ...src };
+  });
+  return out;
+}
+
+// What the map is told about each source (M-27): through this server while it keeps tiles,
+// straight from upstream while it does not
+export function sourcesForBrowser(mapping) {
+  const cached = !!(mapping && mapping.cache && mapping.cache.enabled);
+  return Object.values(enabledSources(mapping)).map((s) => ({
+    id: s.id, name: s.name, attribution: s.attribution, minzoom: s.minzoom, maxzoom: s.maxzoom,
+    url: cached ? `/tiles/${s.id}/{z}/{x}/{y}` : s.upstream,
+    ...(s.bounds ? { bounds: s.bounds } : {}),
+  }));
+}
+
+const clamp = (v, z) => Math.min(Math.max(v, 0), (2 ** z) - 1);
+export function tileX(lon, z) {
+  return clamp(Math.floor(((lon + 180) / 360) * (2 ** z)), z);
+}
+export function tileY(lat, z) {
+  const r = (lat * Math.PI) / 180;
+  return clamp(Math.floor(((1 - (Math.log(Math.tan(r) + (1 / Math.cos(r))) / Math.PI)) / 2) * (2 ** z)), z);
+}
+
+// Within the source's zooms, and touching its bounds (bounds: [[south, west], [north, east]])
+export function tileInSource(t, src) {
+  if ((t.z < src.minzoom) || (t.z > src.maxzoom)) return false;
+  if (!src.bounds) return true;
+  const [[south, west], [north, east]] = src.bounds;
+  return (t.x >= tileX(west, t.z)) && (t.x <= tileX(east, t.z))
+    && (t.y >= tileY(north, t.z)) && (t.y <= tileY(south, t.z));
+}
+
+// z, x and y straight from the URL. Digits only, so none of it can name a path elsewhere.
+export function parseTile(z, x, y, maxzoom = 22) {
+  const m = /^(\d{1,7})(\.(png|jpg|jpeg|webp))?$/.exec(String(y));
+  if (!m || !/^\d{1,2}$/.test(String(z)) || !/^\d{1,7}$/.test(String(x))) return null;
+  const t = { z: Number(z), x: Number(x), y: Number(m[1]) };
+  const n = 2 ** t.z;
+  return ((t.z <= maxzoom) && (t.x < n) && (t.y < n)) ? t : null;
+}
+
+export const tileKey = (id, t) => `${id}/${t.z}/${t.x}/${t.y}`;
+
+// The network edge, replaced in tests. cb(err) only when upstream could not be reached at all.
+function fetchUpstream(url, { since, userAgent }, cb) {
+  const headers = { 'User-Agent': userAgent };
+  if (since) headers['If-Modified-Since'] = since.toUTCString();
+  fetch(url, { headers, signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) })
+    .then((r) => (r.ok
+      ? r.arrayBuffer().then((b) => ({ status: r.status, type: r.headers.get('content-type'), body: Buffer.from(b) }))
+      : { status: r.status }))
+    .then((res) => cb(null, res), (err) => cb(err));
+}
+
+export function createTileCache(options = {}) {
+  const o = { ...CACHE_DEFAULTS, ...options };
+  const sources = o.sources ? enabledSources({ sources: o.sources }) : enabledSources(null);
+  const fetchTile = o.fetchTile || fetchUpstream;
+  const now = o.now || Date.now;
+  const refreshMs = Math.max(o.refreshdays, MIN_REFRESH_DAYS) * DAY_MS;
+  let keep = o.keep || (() => false);     // keys the prefetch still wants are never evicted
+  const lastUsed = new Map();             // file -> ms
+  const waiting = new Map();              // key -> [cb], so two views of one tile fetch it once
+  const offlineUntil = {};                // by source: NSW being down does not stop OSM
+  const lastExt = {};                     // by source, so a kept tile is usually found in one stat
+
+  const base = (id, t) => path.join(o.dir, id, String(t.z), String(t.x), String(t.y));
+  const urlOf = (src, t) => src.upstream.replace('{z}', t.z).replace('{x}', t.x).replace('{y}', t.y);
+  const online = (id) => now() >= (offlineUntil[id] || 0);
+
+  const fetches = priorityQueue((job, cb) => fetchTile(job.url, { since: job.since, userAgent: o.userAgent }, cb),
+    CONCURRENT_FETCHES);
+
+  // cb(null, {file, st}) for the kept file, whichever type it was stored as, or cb(null, null)
+  function find(id, t, cb) {
+    const exts = [...new Set([lastExt[id] || '.png', ...Object.values(EXTENSIONS)])];
+    let i = 0;
+    const next = () => {
+      if (i >= exts.length) return cb(null, null);
+      const file = base(id, t) + exts[i++];
+      fs.stat(file, (err, st) => {
+        if (err) return next();
+        lastExt[id] = path.extname(file);
+        cb(null, { file, st });
+      });
+    };
+    next();
+  }
+
+  function store(id, t, res, cb) {
+    const ext = EXTENSIONS[String(res.type).split(';')[0].trim()] || '.png';
+    const file = base(id, t) + ext;
+    const tmp = `${file}.tmp`;   // renamed into place, so a half-written tile is never served
+    fs.mkdir(path.dirname(file), { recursive: true }, (err) => {
+      if (err) return cb(err);
+      fs.writeFile(tmp, res.body, (err1) => (err1 ? cb(err1) : fs.rename(tmp, file, (err2) => {
+        if (!err2) lastExt[id] = ext;
+        cb(err2, file);
+      })));
+    });
+  }
+
+  // cb(null, file) when the tile is now current, cb(null, null) when it is not
+  function refresh(src, t, kept, priority, cb) {
+    const key = tileKey(src.id, t);
+    if (!online(src.id)) return cb(null, null);
+    if (waiting.has(key)) { waiting.get(key).push(cb); return; }
+    waiting.set(key, [cb]);
+    const done = (err, file) => {
+      const cbs = waiting.get(key);
+      waiting.delete(key);
+      if (err) console.log('Tile', key, 'not kept:', err.message);
+      cbs.forEach((c) => c(null, err ? null : file));
+    };
+    fetches.push({ url: urlOf(src, t), since: kept && kept.st.mtime }, priority, (err, res) => {
+      if (err || (res.status === 429) || (res.status >= 500)) {
+        offlineUntil[src.id] = now() + OFFLINE_RETRY_MS;
+        return done(null, null);
+      }
+      if ((res.status === 304) && kept) {
+        const d = new Date(now());
+        return fs.utimes(kept.file, d, d, (err1) => done(err1, kept.file));
+      }
+      if (res.status === 200) return store(src.id, t, res, done);
+      done(null, null);   // e.g. 404: upstream has no such tile
+    });
+  }
+
+  // cb(null, file) to serve, or cb(null, null) for a 404. A kept tile is served at once even when
+  // due a check - the check runs behind it, and offline a stale tile beats none.
+  function get(id, t, cb) {
+    const src = sources[id];
+    if (!src || !tileInSource(t, src)) return cb(null, null);
+    find(id, t, (err, kept) => {
+      if (kept) {
+        lastUsed.set(kept.file, now());
+        if ((now() - kept.st.mtimeMs) > refreshMs) refresh(src, t, kept, VIEW, () => {});
+        return cb(null, kept.file);
+      }
+      refresh(src, t, null, VIEW, (err1, file) => {
+        if (file) lastUsed.set(file, now());
+        cb(null, file);
+      });
+    });
+  }
+
+  // Fetch a tile nobody is looking at yet, if it is missing or due. cb(null, 'kept' | 'fetched' | 'failed')
+  function prefetch(id, t, cb) {
+    const src = sources[id];
+    if (!src || !tileInSource(t, src)) return cb(null, 'failed');
+    find(id, t, (err, kept) => {
+      if (kept && ((now() - kept.st.mtimeMs) <= refreshMs)) return cb(null, 'kept');
+      refresh(src, t, kept, PREFETCH, (err1, file) => cb(null, file ? 'fetched' : 'failed'));
+    });
+  }
+
+  function listFiles(cb) {
+    const found = [];
+    const walk = (d, cb1) => fs.readdir(d, { withFileTypes: true }, (err, entries) => {
+      if (err) return cb1(err.code === 'ENOENT' ? null : err);
+      let i = 0;
+      const next = (err1) => {
+        if (err1 || (i >= entries.length)) return cb1(err1);
+        const e = entries[i++];
+        const full = path.join(d, e.name);
+        if (e.isDirectory()) return walk(full, next);
+        fs.stat(full, (err2, st) => {
+          if (!err2) found.push({ file: full, size: st.size, used: lastUsed.get(full) || st.mtimeMs });
+          next();
+        });
+      };
+      next();
+    });
+    walk(o.dir, (err) => cb(err, found));
+  }
+  // dir/osm/15/26000/16900.png -> osm/15/26000/16900
+  const keyOfFile = (file) => path.relative(o.dir, file).split(path.sep).join('/').replace(/\.[a-z]+$/, '');
+
+  // Least recently used first, until under maxmb, across every source. After a restart, a tile not
+  // yet viewed counts as last used when it was last known current.
+  function evict(cb) {
+    listFiles((err, files) => {
+      if (err) return cb(err);
+      let total = files.reduce((s, f) => s + f.size, 0);
+      const limit = o.maxmb * 1024 * 1024;
+      const victims = [];
+      files.sort((a, b) => a.used - b.used);
+      for (const f of files) {
+        if (total <= limit) break;
+        if (keep(keyOfFile(f.file))) continue;
+        victims.push(f.file);
+        total -= f.size;
+      }
+      let i = 0;
+      const next = () => {
+        if (i >= victims.length) return cb(null, { removed: victims.length, bytes: total });
+        const file = victims[i++];
+        lastUsed.delete(file);
+        fs.unlink(file, () => next());
+      };
+      next();
+    });
+  }
+
+  return { get, prefetch, evict, online, sources, setKeep: (fn) => { keep = fn; }, dir: o.dir };
+}
+
+// Once a day, like the readings' housekeeping - first pass delayed past the busy startup
+export function startTileHousekeeping(cache) {
+  const run = () => cache.evict((err, r) => {
+    if (err) console.error('Tile housekeeping:', err.message);
+    else if (r.removed) console.log('Tile housekeeping:', r.removed, 'tiles dropped');
+  });
+  setTimeout(run, 5 * 60 * 1000).unref();
+  const timer = setInterval(run, DAY_MS);
+  timer.unref();
+  return timer;
+}

package/package.json

@@ -12,8 +12,8 @@
     "cookie-parser": "^1.4.7",
     "express": "^5.2.1",
     "express-session": "^1.19.0",
-    "frugal-iot-client": "^2.0.9",
-    "frugal-iot-logger": "^2.0.9",
+    "frugal-iot-client": "^2.0.11",
+    "frugal-iot-logger": "^2.0.11",
     "hash-wasm": "^4.12.0",
     "js-yaml": "^4.2.0",
     "morgan": "^1.11.0",
@@ -47,6 +47,7 @@
     "config.yaml",
     "config.d/email.yaml",
     "config.d/logger.yaml",
+    "config.d/mapping.yaml",
     "config.d/mqtt.yaml",
     "config.d/opensearch.yaml",
     "config.d/server.yaml",
@@ -64,9 +65,10 @@
     "extras/frugaliot-ap-watchdog.service",
     "extras/frugaliot-ap-watchdog.timer",
     "API.md",
-    "GRAFANA.md",
-    "INSTALL-OPENSEARCH.md",
-    "INSTALLATION.md"
+    "docs/GRAFANA.md",
+    "docs/INSTALL-OPENSEARCH.md",
+    "docs/INSTALLATION.md",
+    "docs/OPENWRT.md"
   ],
   "devDependencies": {
     "supertest": "^7.2.2",
@@ -93,7 +95,7 @@
     "check-files": "node scripts/check-published-files.js"
   },
   "type": "module",
-  "version": "2.0.9",
+  "version": "2.0.11",
   "overrides": {
     "uuid": "^11.1.1"
   }

package/public/service-worker.js

@@ -6,7 +6,7 @@
 // it, an installed browser keeps serving the files it already cached - and since the list now covers
 // the dashboard's own code, that means old JS behind freshly fetched HTML. Any edit to the list
 // needs this bumped, or it does not reach anyone who already has the old one.
-const CACHE_NAME = 'frugal-iot-cache-2.0.9';
+const CACHE_NAME = 'frugal-iot-cache-2.0.11';
 /*
  * Everything the app needs, so that a phone on a poor or expensive link fetches it once.
  *
@@ -100,6 +100,14 @@ self.addEventListener('fetch', event => {
         // Scoped to CACHE_NAME: a bare caches.match searches every cache in the origin, so a
         // leftover older cache answers first and the version bump achieves nothing
         caches.match(event.request, {cacheName: CACHE_NAME})
-            .then(response => response || fetch(event.request))
+            .then(response => response || fetch(event.request).then(fetched => {
+                // A language is cached when first used, not at install: there are many, and a
+                // phone wants one
+                if (fetched.ok && new URL(event.request.url).pathname.startsWith('/dashboard/languages/')) {
+                    const copy = fetched.clone();
+                    caches.open(CACHE_NAME).then(cache => cache.put(event.request, copy));
+                }
+                return fetched;
+            }))
     );
 });

package/scripts/addbridge-pi.zsh

@@ -5,7 +5,7 @@
 # After this, the Pi's readings also appear on the production server, and its nodes can be
 # controlled from there - while the Pi carries on recording and serving on its own whenever the
 # link is down. Readings taken during an outage stay on the Pi: production gets a gap rather than
-# a backlog delivered later with the wrong timestamps. INSTALLATION.md step 11 explains why.
+# a backlog delivered later with the wrong timestamps. docs/INSTALLATION.md step 11 explains why.
 #
 # The relaying is done by the broker, so the server, the logger and the nodes are untouched.
 #

package/scripts/addbridge-prod.zsh

@@ -24,7 +24,7 @@
 # printed - it is needed once, to type into the Pi.
 #
 # This assumes the server has already been set up to accept bridges at all: a TLS listener on 8883
-# with a certificate. That is a one-off, and INSTALLATION.md step 11 covers it.
+# with a certificate. That is a one-off, and docs/INSTALLATION.md step 11 covers it.
 
 set -euo pipefail
 
@@ -255,7 +255,7 @@ else
   echo ""
   echo "NOTE: this broker has no acl_file, so ${ACCOUNT} can reach every topic on it, not just"
   echo "  ${ORG_ID}/#. That is how the broker already treats every other account, so this is no"
-  echo "  worse than what is there - but see INSTALLATION.md step 11 for restricting it."
+  echo "  worse than what is there - but see docs/INSTALLATION.md step 11 for restricting it."
 fi
 
 # password_file and acl_file are both re-read on a reload, so nothing needs to be disconnected.

package/scripts/addorganization.zsh

@@ -133,11 +133,20 @@ if [[ ! -w "$MOSQUITTO_PASSWD_DIR" || ( -e "$MOSQUITTO_PASSWD_FILE" && ! -w "$MO
     fi
   fi
 fi
+# Run as root there is no sudo to go through (OpenWrt has none, and runs everything as root), and
+# mosquitto_passwd then rewrites the file as root - which the broker, running as its own account,
+# cannot read. So when root is doing it and somebody else owns the file, hand it back afterwards.
+if (( EUID == 0 )) && [[ -n "$MOSQUITTO_PASSWD_OWNER" && "$MOSQUITTO_PASSWD_OWNER" != "root" ]]; then
+  MOSQUITTO_RESTORE_OWNER=1
+fi
 # Called after any successful mosquitto_passwd run below
 restore_passwd_owner() {
   (( MOSQUITTO_RESTORE_OWNER )) || return 0
-  sudo chown "${MOSQUITTO_PASSWD_OWNER}:${MOSQUITTO_PASSWD_GROUP}" "$MOSQUITTO_PASSWD_FILE"
-  [[ -n "$MOSQUITTO_PASSWD_MODE" ]] && sudo chmod "$MOSQUITTO_PASSWD_MODE" "$MOSQUITTO_PASSWD_FILE"
+  local SUDO=(sudo)
+  (( EUID == 0 )) && SUDO=()
+  "${SUDO[@]}" chown "${MOSQUITTO_PASSWD_OWNER}:${MOSQUITTO_PASSWD_GROUP}" "$MOSQUITTO_PASSWD_FILE"
+  [[ -n "$MOSQUITTO_PASSWD_MODE" ]] && "${SUDO[@]}" chmod "$MOSQUITTO_PASSWD_MODE" "$MOSQUITTO_PASSWD_FILE"
+  return 0
 }
 
 # ---- Escaping helpers ----

package/scripts/diagnostic.zsh

@@ -1,6 +1,6 @@
 #!/usr/bin/env zsh
 #
-# Report the state of a Frugal IoT installation, for working out why something in INSTALLATION.md
+# Report the state of a Frugal IoT installation, for working out why something in docs/INSTALLATION.md
 # did not do what it said it would. It only looks - it changes nothing.
 #
 # Run it from the directory the server is installed in (the one holding frugal-iot.db):
@@ -85,7 +85,7 @@ for c in node npm sqlite3 zsh mosquitto mosquitto_passwd mosquitto_sub; do
     esac
   else
     item "$c: NOT INSTALLED"
-    [[ $c == (node|npm|sqlite3) ]] && problem "$c is not installed - see INSTALLATION.md step 3"
+    [[ $c == (node|npm|sqlite3) ]] && problem "$c is not installed - see docs/INSTALLATION.md step 3"
   fi
 done
 
@@ -355,7 +355,7 @@ broker_setting() {  # name -> value from the broker's own configuration, sudo on
 broker_setting acl_file; ACLFILE="$BROKER_SETTING"
 if [[ -z "$ACLFILE" ]]; then
   item "acl_file: NOT SET - every account that can log in can reach every topic, across all organizations"
-  problem "The broker has no acl_file. Re-run the install script, or see INSTALLATION.md 'Turn on access control'"
+  problem "The broker has no acl_file. Re-run the install script, or see docs/INSTALLATION.md 'Turn on access control'"
 else
   item "acl_file: $ACLFILE"
   if [[ -r "$ACLFILE" ]] || { have sudo && sudo -n test -r "$ACLFILE" 2>/dev/null; }; then
@@ -608,7 +608,7 @@ else
 fi
 
 # Swapping to an SD card wears it fast. On a Pi 4 there should be little or none; a Pi Zero W may
-# need some. See INSTALLATION.md, "Wear and tear on the SD card".
+# need some. See docs/INSTALLATION.md, "Wear and tear on the SD card".
 if have free; then
   item "swap:        $(free -h | awk '/^Swap:/ {print $3 " used of " $2}')"
 fi

package/scripts/grafana-token.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 /*
- * Issue, list or revoke the tokens Grafana uses to read an organization's history (GRAFANA.md).
+ * Issue, list or revoke the tokens Grafana uses to read an organization's history (docs/GRAFANA.md).
  *
  *   npx --no frugal-iot-grafana-token add <org> <username> <name>   prints the token - once, it is not kept
  *   npx --no frugal-iot-grafana-token list <org>

package/scripts/init.zsh

@@ -48,7 +48,11 @@ copy_if_missing() {
     fi
   elif [[ -e "$from" ]]; then
     mkdir -p "${to:h}"
-    cp "$from" "$to"
+    # extras/ holds a directory (grafana/) as well as files, and a plain cp refuses a directory -
+    # which under "set -e" ended the whole init, after server.yaml had been created but before it
+    # was pointed at node_modules. The re-run then saw an existing server.yaml and left it alone, so
+    # "/" 404ed. -R for a directory, so a failure here cannot strand a half-initialised install.
+    if [[ -d "$from" ]]; then cp -R "$from" "$to"; else cp "$from" "$to"; fi
     COPIED+=("$to")
     echo "  created ${to}"
   fi
@@ -56,7 +60,7 @@ copy_if_missing() {
 
 echo "Configuration:"
 copy_if_missing "${PKG}/config.yaml" "./config.yaml"
-for f in logger.yaml mqtt.yaml server.yaml email.yaml; do
+for f in logger.yaml mqtt.yaml server.yaml email.yaml mapping.yaml; do
   copy_if_missing "${PKG}/config.d/${f}" "./config.d/${f}"
 done
 # The schema describes the sensor types the software understands, so a release changing it matters