hm2mqtt 3.6.2 → 3.8.0

package/README.md

@@ -76,6 +76,9 @@ the automatic detection.
 
 The addon packages are marked `-beta` until someone has confirmed an install on real hardware.
 
+On [openccu-lite](#openccu-lite) the same package is installed from the system's addon catalogue
+(_Addons → Catalogue_); there is nothing to download by hand.
+
 ## Finding the CCU
 
 ```
@@ -271,6 +274,21 @@ names, rooms and functions from the box's **metadata API** instead of from the R
 RaspberryMatic or OpenCCU answers 404 there and everything stays exactly as before — the same
 configuration works on both, which is what a backup restored on the other kind of box needs.
 
+**Install and start.** On the system itself hm2mqtt is installed from the addon catalogue
+(_Addons → Catalogue_, the same package as on a CCU; its manifest `openccu-lite.json` ships in the
+package). The addon declares the early start: its unit starts at boot before `rfd` and `hmipserver`,
+and hm2mqtt waits for them quietly — an interface that does not answer yet is probed and subscribed
+again after 1, 2, 4 and 8 s, then every 15 s, with one info line and no warning, and one that comes
+up later (or after a restart of its process) is picked up without a restart of hm2mqtt. The system
+can switch the early start off on its Addons page; the unit then starts after both. The broker URL
+is the one setting to make, as on a CCU. The log is in the journal and nowhere else — no
+`var/hm2mqtt.log` on the SD card: `journalctl -u addon-hm2mqtt`, the system's _Log_ page
+(`/system/log?unit=addon-hm2mqtt`), and the settings page's log view, which reads it with the
+addon's own API token (the manifest asks for `logs:read`). The system opens the settings page
+without `?sid=`: the page and its CGIs take the session from the system's `X-Occulite-Session`
+header and confirm it with the system (the manifest declares `ui.session_header`); `?sid=` keeps
+working for a CCU and for an openccu-lite image from before the header.
+
 Names arrive as a snapshot at start and then over the box's event stream (`/api/meta/v1/events/sse`):
 a rename in the box's UI is in the topics about a second later, without a poll and without a
 restart. The store is cached in `meta.json` in the state directory, so a start without the box

package/index.js

@@ -15,7 +15,15 @@ import {Rega} from 'homematic-rega';
 import config from './config.js';
 import pkg from './package.json' with {type: 'json'};
 import {handle as handleInstall} from './lib/install.js';
-import {parseInterfaces, probeInterfaces, interfaceConfig, detectLocal, regaPort} from './lib/interfaces.js';
+import {
+    parseInterfaces,
+    probeInterfaces,
+    watchInterfaces,
+    interfaceConfig,
+    detectLocal,
+    regaPort,
+    INTERFACE_NAMES,
+} from './lib/interfaces.js';
 import {RpcServers, RpcConnection} from './lib/rpc.js';
 import {Metadata} from './lib/metadata.js';
 import {ValueStore, hmBlock, isEvent} from './lib/values.js';
@@ -69,6 +77,8 @@ const STOP_TIMEOUT_MS = 1500;
 const RESOLVE_RETRY_MS = 10000;
 /** how often an unreachable box is probed for the metadata api again (H-47) */
 const REDETECT_MS = 60000;
+/** how long the local-mode probe waits for the interface processes on a loopback address (B-1) */
+const LOCAL_WAIT_MS = 120000;
 
 const here = path.dirname(fileURLToPath(import.meta.url));
 /** the configured CCU address (the `ccu` field of every payload); connections use the resolved ip */
@@ -234,7 +244,16 @@ for (const pattern of listenPatterns) {
  */
 // an explicit --ccu-tls is a tunnel or a proxy by definition - local mode would silently drop the
 // TLS and the ports the user asked for, so it is never probed into, only chosen with --local
-const localMode = config.local === undefined ? !config.ccuTls && (await detectLocal(host)) : Boolean(config.local);
+// on a loopback address where nothing answers yet the interface processes may still be starting
+// (a boot, openccu-lite's early start): wait for them up to two minutes before deciding (B-1)
+const localMode =
+    config.local === undefined
+        ? !config.ccuTls &&
+          (await detectLocal(host, {
+              waitMs: LOCAL_WAIT_MS,
+              onWait: () => log.info('no interface process answers on', host, 'yet - waiting for them'),
+          }))
+        : Boolean(config.local);
 if (localMode) {
     log.info('local mode: BidCos over binrpc (32001/32000), hmipserver on 32010, ReGa on 8183');
 }
@@ -331,6 +350,8 @@ function publishTopic(topic, value, {retain = true, extra, ts, lc} = {}) {
 let indexTimer = null;
 
 let enabled = [];
+/** --interfaces auto: the interfaces the start probe missed, probed on until they answer (B-1) */
+let interfaceWatch = null;
 const connections = {};
 let servers = null;
 let regaOk = false;
@@ -995,10 +1016,16 @@ async function start() {
     } else if (regaSync) {
         regaSync.setHost(ccuIp);
     }
-    enabled =
-        parseInterfaces(config.interfaces) || (await probeInterfaces(ccuIp, {tls: config.ccuTls, local: localMode}));
+    const listed = parseInterfaces(config.interfaces);
+    enabled = listed || (await probeInterfaces(ccuIp, {tls: config.ccuTls, local: localMode}));
     if (enabled.length === 0) {
-        log.error('no interface found on', host, '- check --ccu-address / --interfaces');
+        // nothing answers yet: on the box itself the interface processes are still starting, which is
+        // no fault; the probe goes on (B-1)
+        (localMode ? log.info : log.warn)(
+            'no interface answers on',
+            host,
+            'yet - probing again' + (localMode ? '' : '; check --ccu-address / --interfaces if this stays'),
+        );
     }
     log.info('interfaces:', enabled.join(', ') || '(none)');
     // locally the CCU calls back over loopback, so nothing of ours needs to listen on the LAN
@@ -1009,6 +1036,7 @@ async function start() {
         xmlrpcPort: config.xmlrpcPort,
         binrpcPort: config.binrpcPort,
         log,
+        ownPrefix: `hm2mqtt_${config.name}_`,
     });
     for (const iface of enabled) {
         connections[iface] = createConnection(iface);
@@ -1022,6 +1050,13 @@ async function start() {
     await Promise.all(enabled.map((iface) => connections[iface].start()));
     updateConnected();
     adapter.publishInfo();
+    if (!listed) {
+        interfaceWatch = watchInterfaces(
+            ccuIp,
+            INTERFACE_NAMES.filter((iface) => !enabled.includes(iface)),
+            {tls: config.ccuTls, local: localMode, onFound: addInterface},
+        );
+    }
 
     timers.push(setInterval(publishCounters, COUNTER_INTERVAL_MS));
     if (config.dutyCycleInterval > 0) {
@@ -1031,7 +1066,25 @@ async function start() {
     timers.push(setInterval(() => values.save(), VALUES_SAVE_MS));
 }
 
+/** An interface the start probe missed answers now: subscribe to it like to the others (B-1). */
+function addInterface(iface) {
+    if (enabled.includes(iface)) {
+        return;
+    }
+    enabled = INTERFACE_NAMES.filter((name) => name === iface || enabled.includes(name));
+    log.info('interface', iface, 'answers now - interfaces:', enabled.join(', '));
+    connections[iface] = createConnection(iface);
+    publishItem(`interface/${iface}/connected`, false);
+    rebuildIndex();
+    updateConnected();
+    adapter.publishInfo();
+    connections[iface].start().catch((err) => log.warn('rpc', iface, 'start failed:', err.message));
+}
+
 async function shutdown() {
+    if (interfaceWatch) {
+        interfaceWatch.stop();
+    }
     for (const timer of timers) {
         clearInterval(timer);
     }

package/lib/interfaces.js

@@ -9,6 +9,7 @@
  */
 
 import net from 'node:net';
+import {initRetryDelay} from './rpc.js';
 
 export const INTERFACES = {
     'BidCos-RF': {
@@ -139,12 +140,15 @@ function portOpen(host, port, timeout) {
 /**
  * Probes which interface ports answer on the CCU (--interfaces auto).
  * @param {string} host
- * @param {{tls?: boolean, local?: boolean, timeout?: number, connect?: Function}} [options]
- * @returns {Promise<string[]>} interface names in table order
+ * @param {{tls?: boolean, local?: boolean, timeout?: number, connect?: Function, names?: string[]}} [options]
+ * @returns {Promise<string[]>} interface names in table order (of `names`)
  */
-export async function probeInterfaces(host, {tls = false, local = false, timeout = 2000, connect = portOpen} = {}) {
+export async function probeInterfaces(
+    host,
+    {tls = false, local = false, timeout = 2000, connect = portOpen, names = INTERFACE_NAMES} = {},
+) {
     const found = [];
-    for (const name of INTERFACE_NAMES) {
+    for (const name of names) {
         const {port} = interfaceConfig(name, {tls, local});
         if (await connect(host, port, timeout)) {
             found.push(name);
@@ -153,6 +157,67 @@ export async function probeInterfaces(host, {tls = false, local = false, timeout
     return found;
 }
 
+/**
+ * Probes the interfaces `--interfaces auto` did not find at the start again, on the schedule of a
+ * failed init (1, 2, 4, 8 s, then every 15 s), and hands each one that answers to `onFound` - once;
+ * the others are probed on until they answer or `stop()` is called (B-1). An interface process that
+ * comes up after hm2mqtt (openccu-lite's early start, a restart of hmipserver, a LAN gateway added
+ * later) is picked up this way without a restart.
+ * @param {string} host
+ * @param {string[]} missing interface names
+ * @param {object} o
+ * @param {(name: string) => void} o.onFound
+ * @param {boolean} [o.tls]
+ * @param {boolean} [o.local]
+ * @param {number} [o.timeout] ms per probe
+ * @param {Function} [o.connect] for tests
+ * @param {{setTimeout: Function, clearTimeout: Function}} [o.timers] for tests
+ * @returns {{stop: () => void, missing: () => string[]}}
+ */
+export function watchInterfaces(
+    host,
+    missing,
+    {onFound, tls = false, local = false, timeout = 2000, connect = portOpen, timers = {setTimeout, clearTimeout}},
+) {
+    let left = INTERFACE_NAMES.filter((name) => missing.includes(name));
+    let rounds = 0;
+    let timer = null;
+    let stopped = false;
+    const round = async () => {
+        timer = null;
+        const found = await probeInterfaces(host, {tls, local, timeout, connect, names: left});
+        if (stopped) {
+            return;
+        }
+        left = left.filter((name) => !found.includes(name));
+        for (const name of found) {
+            onFound(name);
+        }
+        schedule();
+    };
+    const schedule = () => {
+        if (stopped || left.length === 0) {
+            return;
+        }
+        rounds += 1;
+        timer = timers.setTimeout(round, initRetryDelay(rounds));
+        if (timer && typeof timer.unref === 'function') {
+            timer.unref();
+        }
+    };
+    schedule();
+    return {
+        stop() {
+            stopped = true;
+            if (timer) {
+                timers.clearTimeout(timer);
+                timer = null;
+            }
+        },
+        missing: () => [...left],
+    };
+}
+
 /**
  * Is this address the machine we run on?
  * @param {string} host
@@ -171,17 +236,52 @@ export function isLocalHost(host) {
  * node-red-contrib-ccu does - that check looks for `"port" => 32001` in
  * /etc/lighttpd/conf.d/proxy.conf, which on firmware 3.8x is a one-line include, so it silently
  * stopped detecting anything while the ports themselves are all still there.
+ *
+ * On a loopback address where nothing answers yet, the processes may still be starting (a boot, or
+ * openccu-lite's early start): with `waitMs` the probe is repeated on the init schedule (1, 2, 4,
+ * 8 s, then every 15 s) until one answers or `waitMs` has passed (B-1); `onWait` is called once,
+ * before the first repetition. When the CCU's usual ports answer instead (a tunnel, a simulator,
+ * lighttpd's proxies without the processes behind them) it is not local mode and nothing waits.
  * @param {string} host
- * @param {{timeout?: number, connect?: Function}} [options]
+ * @param {{timeout?: number, connect?: Function, waitMs?: number, onWait?: Function,
+ *          sleep?: (ms: number) => Promise<void>, now?: () => number}} [options]
  * @returns {Promise<boolean>}
  */
-export async function detectLocal(host, {timeout = 500, connect = portOpen} = {}) {
+export async function detectLocal(
+    host,
+    {
+        timeout = 500,
+        connect = portOpen,
+        waitMs = 0,
+        onWait = () => {},
+        sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+        now = Date.now,
+    } = {},
+) {
     if (!isLocalHost(host)) {
         return false;
     }
     const ports = INTERFACE_NAMES.map((name) => INTERFACES[name].localPort).filter(Boolean);
-    // concurrently: this runs at every startup, and closed-but-filtered ports would otherwise
-    // stack their timeouts
-    const results = await Promise.all([...ports, REGA_LOCAL_PORT].map((port) => connect(host, port, timeout)));
-    return results.some(Boolean);
+    const start = now();
+    for (let attempt = 1; ; attempt += 1) {
+        // concurrently: this runs at every startup, and closed-but-filtered ports would otherwise
+        // stack their timeouts
+        const results = await Promise.all([...ports, REGA_LOCAL_PORT].map((port) => connect(host, port, timeout)));
+        if (results.some(Boolean)) {
+            return true;
+        }
+        const delay = initRetryDelay(attempt);
+        if (now() - start + delay > waitMs) {
+            return false;
+        }
+        const usual = INTERFACE_NAMES.map((name) => INTERFACES[name].port).filter(Boolean);
+        const proxied = await Promise.all([...usual, REGA_PORT].map((port) => connect(host, port, timeout)));
+        if (proxied.some(Boolean)) {
+            return false;
+        }
+        if (attempt === 1) {
+            onWait();
+        }
+        await sleep(delay);
+    }
 }

package/lib/rpc.js

@@ -22,9 +22,9 @@ export const METHODS = [
 ];
 
 const TX_METHODS = new Set(['setValue', 'putParamset', 'activateLinkParamset']);
-/** a failed init is retried after 2, 4, 8, 16 s, then every INIT_RETRY_MS (the cap) */
-const INIT_BACKOFF_MS = [2000, 4000, 8000, 16000];
-const INIT_RETRY_MS = 30000;
+/** a failed init is retried after 1, 2, 4, 8 s, then every INIT_RETRY_MS (the cap; task 21) */
+const INIT_BACKOFF_MS = [1000, 2000, 4000, 8000];
+export const INIT_RETRY_MS = 15000;
 const DEINIT_TIMEOUT_MS = 1500;
 const CALL_TIMEOUT_MS = 30000;
 
@@ -50,6 +50,19 @@ export function batchHints(events) {
     return hints;
 }
 
+/**
+ * Whether an error only says that nothing listens on the port yet - an interface process that is
+ * still starting (openccu-lite starts the addon before rfd and hmipserver), or one that is
+ * restarting. Such a wait is normal at boot and logged as info, not as a warning.
+ * @param {Error|string} err
+ * @returns {boolean}
+ */
+export function notListening(err) {
+    const code = err && err.code;
+    const msg = String((err && err.message) || err || '');
+    return code === 'ECONNREFUSED' || /ECONNREFUSED/.test(msg);
+}
+
 /**
  * The delay before the next init attempt.
  * @param {number} failures failed attempts in a row (1 after the first failure)
@@ -72,8 +85,12 @@ export class RpcServers {
      * @param {number} o.binrpcPort
      * @param {object} o.log
      * @param {{xmlrpc?: object, binrpc?: object}} [o.libs] for tests
+     * @param {string} [o.ownPrefix] the start of this instance's init ids: a call for one of them that
+     *   is not registered (yet) is expected - an interface process that re-announces itself to a
+     *   subscription of the last run before this one subscribed again - and logged as debug
      */
-    constructor({listenAddress, initAddress, xmlrpcPort, binrpcPort, log, libs = {}}) {
+    constructor({listenAddress, initAddress, xmlrpcPort, binrpcPort, log, libs = {}, ownPrefix}) {
+        this.ownPrefix = ownPrefix;
         this.listenAddress = listenAddress;
         this.initAddress = initAddress || listenAddress;
         this.ports = {xmlrpc: xmlrpcPort, binrpc: binrpcPort};
@@ -130,7 +147,8 @@ export class RpcServers {
         const initId = Array.isArray(params) ? params[0] : undefined;
         const connection = this.connections.get(initId);
         if (!connection) {
-            this.log.warn('rpc < call for unknown init id', String(initId));
+            const own = this.ownPrefix && typeof initId === 'string' && initId.startsWith(this.ownPrefix);
+            (own ? this.log.debug : this.log.warn)('rpc < call for unknown init id', String(initId));
         }
         return connection;
     }
@@ -300,6 +318,18 @@ export class RpcConnection extends EventEmitter {
         this.pingTimer = null;
         this.lastInitError = null;
         this.initFailures = 0;
+        this.everInitialized = false;
+    }
+
+    /** Closes the client (binrpc: its socket and reconnect timer); the next call creates a new one. */
+    dropClient() {
+        if (this.protocol !== 'binrpc' || !this.client) {
+            return;
+        }
+        if (typeof this.client.close === 'function') {
+            this.client.close();
+        }
+        this.client = null;
     }
 
     createClient() {
@@ -384,6 +414,7 @@ export class RpcConnection extends EventEmitter {
             this.log.info('rpc', this.name, '> init', url, this.initId);
             await this.methodCall('init', [url, this.initId]);
             this.initialized = true;
+            this.everInitialized = true;
             if (this.initFailures > 0) {
                 this.log.info('rpc', this.name, 'init succeeded after', this.initFailures + 1, 'attempts');
             }
@@ -397,17 +428,19 @@ export class RpcConnection extends EventEmitter {
             const msg = err.message || String(err);
             this.initFailures += 1;
             const delay = initRetryDelay(this.initFailures);
-            if (msg !== this.lastInitError) {
-                this.log.warn(
-                    'rpc',
-                    this.name,
-                    'init failed:',
-                    msg,
-                    `- retrying in ${delay / 1000} s (then up to every ${INIT_RETRY_MS / 1000} s)`,
-                );
+            // a binrpc client whose socket is gone waits for its own reconnect timer (2.5 s doubling
+            // up to 30 s) and fails every call meanwhile; a fresh one connects at the next attempt
+            this.dropClient();
+            const retry = `retrying in ${delay / 1000} s (then up to every ${INIT_RETRY_MS / 1000} s)`;
+            if (msg === this.lastInitError) {
+                this.log.debug('rpc', this.name, 'init failed:', msg, `- next attempt in ${delay / 1000} s`);
+            } else if (!this.everInitialized && notListening(err)) {
+                // the start before the interface process (openccu-lite's early start): a wait, not a fault
+                this.log.info('rpc', this.name, `nothing is listening on ${this.host}:${this.port} yet - ${retry}`);
                 this.lastInitError = msg;
             } else {
-                this.log.debug('rpc', this.name, 'init failed:', msg, `- next attempt in ${delay / 1000} s`);
+                this.log.warn('rpc', this.name, 'init failed:', msg, `- ${retry}`);
+                this.lastInitError = msg;
             }
             this.setConnected(false);
             this.initTimer = this.timers.setTimeout(() => this.init(), delay);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hm2mqtt",
-  "version": "3.6.2",
+  "version": "3.8.0",
   "description": "Interface between Homematic CCU and MQTT",
   "type": "module",
   "main": "index.js",
@@ -67,7 +67,7 @@
     "serviceExtra": []
   },
   "dependencies": {
-    "binrpc": "^3.3.1",
+    "binrpc": "^4.3.0",
     "homematic-rega": "^2.0.0",
     "homematic-xmlrpc": "^2.0.0",
     "mqtt-interfaces-core": "^0.15.2"