hm2mqtt 3.3.0 → 3.4.0

package/README.md

@@ -46,6 +46,34 @@ docker run -d --name hm2mqtt --restart unless-stopped --network host -v hm2mqtt:
 Host networking, or publish 2126/2127 and set `HM2MQTT_INIT_ADDRESS` to the docker host's address
 so the CCU can call back. `--restart unless-stopped` brings it back after `maintenance/set/restart`.
 
+### On the CCU itself (addon package)
+
+For a setup without a server: hm2mqtt also ships as a CCU addon, installed in the WebUI under
+_Systemsteuerung → Zusatzsoftware_. Pick the package for your hardware from the
+[latest release](https://github.com/hobbyquaker/hm2mqtt.js/releases/latest):
+
+| Platform                                         | Package                                |
+| ------------------------------------------------ | -------------------------------------- |
+| CCU3 with the official eQ-3 firmware             | `hm2mqtt-ccu-armv7l-<version>.tar.gz`  |
+| OpenCCU 32-bit (CCU3 hardware, Raspberry Pi 2/3) | `hm2mqtt-ccu-armv7l-<version>.tar.gz`  |
+| OpenCCU 64-bit (Raspberry Pi 4/5)                | `hm2mqtt-ccu-aarch64-<version>.tar.gz` |
+| OpenCCU on x86_64 (debmatic, virtual machines)   | `hm2mqtt-ccu-x86_64-<version>.tar.gz`  |
+
+After the install a **hm2mqtt** button appears in _Systemsteuerung_: it configures every option of
+this README in a form, starts and stops the service and shows the log. The only setting that has to
+be made is the broker URL — a CCU has no MQTT broker of its own, so point hm2mqtt at one on your
+network (or install the Mosquitto addon).
+
+Everything the addon needs lives in `/usr/local/addons/hm2mqtt`, including its own Node.js runtime
+(the CCU3's firmware is far too old to run a current Node, so the addon brings a musl build that
+depends on nothing outside its own directory — another addon's Node.js is neither used nor
+disturbed). On the CCU it talks to the interface processes directly instead of through lighttpd:
+binrpc on 32001/32000 for BidCos, hmipserver on 32010, ReGa on 8183 — no CCU authentication, no
+firewall rules, and nothing of hm2mqtt listening on the network. `--local` / `--no-local` overrides
+the automatic detection.
+
+The addon packages are marked `-beta` until someone has confirmed an install on real hardware.
+
 ## Finding the CCU
 
 ```

package/config.js

@@ -22,6 +22,11 @@ export const OPTIONS = {
         describe: `comma separated interfaces (${INTERFACE_NAMES.join(', ')}) or "auto" (probe the ports)`,
         default: DEFAULT_INTERFACES.join(','),
     },
+    local: {
+        type: 'boolean',
+        describe:
+            'talk to the CCU processes directly (binrpc 32001/32000, hmipserver 32010, ReGa 8183) - default: probe when the address is local',
+    },
     'bidcos-binrpc': {
         type: 'boolean',
         describe: 'talk binrpc instead of xmlrpc to BidCos-RF and BidCos-Wired',

package/index.js

@@ -15,7 +15,7 @@ import {Rega} from 'homematic-rega';
 import config from './config.js';
 import pkg from './package.json' with {type: 'json'};
 import {handle as handleInstall} from './lib/install.js';
-import {parseInterfaces, probeInterfaces, interfaceConfig} from './lib/interfaces.js';
+import {parseInterfaces, probeInterfaces, interfaceConfig, detectLocal, regaPort} from './lib/interfaces.js';
 import {RpcServers, RpcConnection} from './lib/rpc.js';
 import {Metadata} from './lib/metadata.js';
 import {ValueStore, hmBlock, isEvent} from './lib/values.js';
@@ -154,6 +154,18 @@ const adapter = createAdapter({
 });
 const {log, pubStatus} = adapter;
 
+/*
+ * Running on the CCU itself, the interface processes are on loopback and the familiar
+ * 2000/2001/2010/9292/8181 are only lighttpd proxies in front of them: an extra hop, XML over HTTP
+ * for BidCos, and the CCU's authentication - all for nothing. --local/--no-local decides; by
+ * default we probe, because node-red-contrib-ccu's config-file check stopped working on current
+ * firmware (see lib/interfaces.js).
+ */
+const localMode = config.local === undefined ? await detectLocal(host) : Boolean(config.local);
+if (localMode) {
+    log.info('local mode: BidCos over binrpc (32001/32000), hmipserver on 32010, ReGa on 8183');
+}
+
 const metadata = new Metadata({stateDir: config.stateDir, seedFile: path.join(here, 'paramsets.json'), log});
 metadata.load();
 
@@ -161,7 +173,8 @@ let regaSync = null;
 if (config.rega) {
     const rega = new Rega({
         host: ccuIp,
-        tls: config.ccuTls,
+        port: regaPort({tls: config.ccuTls, local: localMode}),
+        tls: config.ccuTls && !localMode,
         insecure: config.ccuInsecure,
         username: config.ccuUsername,
         password: config.ccuPassword,
@@ -615,14 +628,14 @@ function onEvent(iface, event) {
 }
 
 function createConnection(iface) {
-    const ic = interfaceConfig(iface, {tls: config.ccuTls, bidcosBinrpc: config.bidcosBinrpc});
+    const ic = interfaceConfig(iface, {tls: config.ccuTls, bidcosBinrpc: config.bidcosBinrpc, local: localMode});
     const conn = new RpcConnection({
         name: iface,
         host: ccuIp,
         protocol: ic.protocol,
         port: ic.port,
         path: ic.path,
-        tls: config.ccuTls,
+        tls: config.ccuTls && !localMode,
         insecure: config.ccuInsecure,
         username: config.ccuUsername,
         password: config.ccuPassword,
@@ -763,12 +776,14 @@ async function start() {
         regaSync.rega.url = regaSync.rega.url.replace(host, ccuIp);
         regaSync.rega.webUrl = regaSync.rega.webUrl.replace(host, ccuIp);
     }
-    enabled = parseInterfaces(config.interfaces) || (await probeInterfaces(ccuIp, {tls: config.ccuTls}));
+    enabled =
+        parseInterfaces(config.interfaces) || (await probeInterfaces(ccuIp, {tls: config.ccuTls, local: localMode}));
     if (enabled.length === 0) {
         log.error('no interface found on', host, '- check --ccu-address / --interfaces');
     }
     log.info('interfaces:', enabled.join(', ') || '(none)');
-    const listenAddress = config.listenAddress || firstIp();
+    // locally the CCU calls back over loopback, so nothing of ours needs to listen on the LAN
+    const listenAddress = config.listenAddress || (localMode ? '127.0.0.1' : firstIp());
     servers = new RpcServers({
         listenAddress,
         initAddress: config.initAddress,

package/lib/interfaces.js

@@ -1,6 +1,11 @@
 /**
- * The CCU interface processes: names as the CCU uses them, ports (plain / TLS), protocol,
+ * The CCU interface processes: names as the CCU uses them, ports (plain / TLS / local), protocol,
  * whether they want an init() subscription and answer pings.
+ *
+ * `localPort` is what the process itself listens on, as opposed to the familiar 2000/2001/2010/9292
+ * which are lighttpd proxies in front of it. Running on the CCU we talk to the process directly:
+ * one hop less, no XML over HTTP for BidCos (rfd speaks binrpc there), and no CCU authentication -
+ * exactly what node-red-contrib-ccu does when it detects a local connection.
  */
 
 import net from 'node:net';
@@ -9,30 +14,66 @@ export const INTERFACES = {
     'BidCos-RF': {
         port: 2001,
         tlsPort: 42001,
+        localPort: 32001,
+        localBinrpc: true,
         protocol: 'xmlrpc',
         binrpc: true,
         init: true,
         ping: true,
         dutyCycle: true,
     },
-    'BidCos-Wired': {port: 2000, tlsPort: 42000, protocol: 'xmlrpc', binrpc: true, init: true, ping: true},
+    'BidCos-Wired': {
+        port: 2000,
+        tlsPort: 42000,
+        localPort: 32000,
+        localBinrpc: true,
+        protocol: 'xmlrpc',
+        binrpc: true,
+        init: true,
+        ping: true,
+    },
     // https://github.com/eq-3/occu/issues/42 — HmIP-RF answers pings but events are rare, node-red used 600 s
     'HmIP-RF': {
         port: 2010,
         tlsPort: 42010,
+        // hmipserver speaks no binrpc; "direct" here only means past the proxy
+        localPort: 32010,
         protocol: 'xmlrpc',
         init: true,
         ping: true,
         pingTimeout: 600,
         dutyCycle: true,
     },
-    VirtualDevices: {port: 9292, tlsPort: 49292, path: '/groups', protocol: 'xmlrpc', init: true, ping: false},
+    VirtualDevices: {
+        port: 9292,
+        tlsPort: 49292,
+        localPort: 39292,
+        path: '/groups',
+        protocol: 'xmlrpc',
+        init: true,
+        ping: false,
+    },
     CUxD: {port: 8701, protocol: 'binrpc', init: true, ping: true},
 };
 
 export const INTERFACE_NAMES = Object.keys(INTERFACES);
 export const DEFAULT_INTERFACES = ['BidCos-RF', 'HmIP-RF', 'VirtualDevices', 'BidCos-Wired'];
 
+/** ReGa's script port: 8181 through lighttpd, 8183 is ReGaHSS itself (no authentication). */
+export const REGA_PORT = 8181;
+export const REGA_TLS_PORT = 48181;
+export const REGA_LOCAL_PORT = 8183;
+
+/**
+ * The port the ReGa script interface is reached on.
+ * @param {{tls?: boolean, local?: boolean}} [options]
+ * @returns {number}
+ */
+export function regaPort({tls = false, local = false} = {}) {
+    if (local) return REGA_LOCAL_PORT;
+    return tls ? REGA_TLS_PORT : REGA_PORT;
+}
+
 /**
  * Parses the --interfaces option. Returns the list of names, or null for "auto".
  * @param {string} value
@@ -61,16 +102,19 @@ export function parseInterfaces(value) {
  * @returns {{name: string, protocol: 'xmlrpc' | 'binrpc', port: number, path?: string, init: boolean,
  *          ping: boolean, pingTimeout?: number, dutyCycle: boolean}}
  */
-export function interfaceConfig(name, {tls = false, bidcosBinrpc = false} = {}) {
+export function interfaceConfig(name, {tls = false, bidcosBinrpc = false, local = false} = {}) {
     const def = INTERFACES[name];
     if (!def) {
         throw new Error('unknown interface ' + name);
     }
-    const binrpc = def.protocol === 'binrpc' || (def.binrpc && bidcosBinrpc && !tls);
+    // local wins over tls: the process ports carry no TLS, and they need none on loopback
+    const useLocal = local && Boolean(def.localPort);
+    const binrpc =
+        def.protocol === 'binrpc' || (useLocal && def.localBinrpc) || (def.binrpc && bidcosBinrpc && !tls && !useLocal);
     return {
         name,
         protocol: binrpc ? 'binrpc' : 'xmlrpc',
-        port: tls && def.tlsPort ? def.tlsPort : def.port,
+        port: useLocal ? def.localPort : tls && def.tlsPort ? def.tlsPort : def.port,
         path: def.path,
         init: def.init,
         ping: def.ping,
@@ -95,16 +139,51 @@ function portOpen(host, port, timeout) {
 /**
  * Probes which interface ports answer on the CCU (--interfaces auto).
  * @param {string} host
- * @param {{tls?: boolean, timeout?: number, connect?: Function}} [options]
+ * @param {{tls?: boolean, local?: boolean, timeout?: number, connect?: Function}} [options]
  * @returns {Promise<string[]>} interface names in table order
  */
-export async function probeInterfaces(host, {tls = false, timeout = 2000, connect = portOpen} = {}) {
+export async function probeInterfaces(host, {tls = false, local = false, timeout = 2000, connect = portOpen} = {}) {
     const found = [];
     for (const name of INTERFACE_NAMES) {
-        const {port} = interfaceConfig(name, {tls});
+        const {port} = interfaceConfig(name, {tls, local});
         if (await connect(host, port, timeout)) {
             found.push(name);
         }
     }
     return found;
 }
+
+/**
+ * Is this address the machine we run on?
+ * @param {string} host
+ * @returns {boolean}
+ */
+export function isLocalHost(host) {
+    const value = String(host || '')
+        .trim()
+        .toLowerCase();
+    return value === 'localhost' || value === '::1' || value === '[::1]' || /^127\./.test(value);
+}
+
+/**
+ * Whether the CCU's interface processes can be reached directly, i.e. whether hm2mqtt runs on the
+ * CCU itself. Probes the process ports instead of reading a lighttpd config the way
+ * node-red-contrib-ccu does - that check looks for `"port" => 32001` in
+ * /etc/lighttpd/conf.d/proxy.conf, which on firmware 3.8x is a one-line include, so it silently
+ * stopped detecting anything while the ports themselves are all still there.
+ * @param {string} host
+ * @param {{timeout?: number, connect?: Function}} [options]
+ * @returns {Promise<boolean>}
+ */
+export async function detectLocal(host, {timeout = 500, connect = portOpen} = {}) {
+    if (!isLocalHost(host)) {
+        return false;
+    }
+    const ports = INTERFACE_NAMES.map((name) => INTERFACES[name].localPort).filter(Boolean);
+    for (const port of [...ports, REGA_LOCAL_PORT]) {
+        if (await connect(host, port, timeout)) {
+            return true;
+        }
+    }
+    return false;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hm2mqtt",
-  "version": "3.3.0",
+  "version": "3.4.0",
   "description": "Interface between Homematic CCU and MQTT",
   "type": "module",
   "main": "index.js",
@@ -69,7 +69,7 @@
     "binrpc": "^3.3.1",
     "homematic-rega": "^2.0.0",
     "homematic-xmlrpc": "^2.0.0",
-    "mqtt-interfaces-core": "^0.9.0"
+    "mqtt-interfaces-core": "^0.15.1"
   },
   "devDependencies": {
     "@eslint/js": "^9.0.0",

package/scripts/addon-api.js

@@ -0,0 +1,159 @@
+#!/usr/bin/env node
+
+/**
+ * One-shot helpers for the CCU addon's web UI, called through `www/api.cgi`. Everything here is
+ * a short-lived process that prints one JSON object and exits - the addon runs no HTTP server of
+ * its own, and the UI's interactive bits (probe the interfaces, test the broker, preview an item
+ * template) would otherwise need one.
+ *
+ *   node scripts/addon-api.js discover   [--timeout 4000]
+ *   node scripts/addon-api.js probe      --host 127.0.0.1 [--tls]
+ *   node scripts/addon-api.js mqtt-test  --url mqtt://host:1883 [--username u] [--password p]
+ *   node scripts/addon-api.js channels   --host 127.0.0.1 [--limit 20]
+ *   node scripts/addon-api.js preview    --host 127.0.0.1 --template '${channelName|channel}/${datapoint}'
+ *
+ * Errors are JSON too ({"error": "..."}), so the UI never has to parse a stack trace.
+ */
+
+import Rega from 'homematic-rega';
+import {discover} from 'mqtt-interfaces-core';
+import {probeInterfaces, INTERFACE_NAMES} from '../lib/interfaces.js';
+import {discoveryHint, interfacesOf} from '../lib/discovery.js';
+import {compileTemplate, DEFAULT_ITEM_TEMPLATE} from '../lib/topics.js';
+
+const [command, ...rest] = process.argv.slice(2);
+
+/**
+ * Parses `--key value` and `--flag` arguments.
+ * @param {string[]} argv
+ * @returns {Record<string, string | boolean>}
+ */
+function parseArgs(argv) {
+    const args = {};
+    for (let i = 0; i < argv.length; i++) {
+        if (!argv[i].startsWith('--')) continue;
+        const key = argv[i].slice(2);
+        const next = argv[i + 1];
+        if (next === undefined || next.startsWith('--')) {
+            args[key] = true;
+        } else {
+            args[key] = next;
+            i++;
+        }
+    }
+    return args;
+}
+
+const args = parseArgs(rest);
+
+/**
+ * ReGa client for the CCU the addon runs on. Port 8183 is ReGaHSS' own listener, which needs no
+ * authentication - it exists on the CCU itself, so a local addon never has to ask for credentials.
+ * @param {object} options
+ * @returns {Rega}
+ */
+function rega({host = '127.0.0.1', port, username, password} = {}) {
+    const local = host === '127.0.0.1' || host === 'localhost';
+    return new Rega({
+        host,
+        port: port ? Number(port) : local ? 8183 : 8181,
+        username,
+        password,
+        translate: false,
+    });
+}
+
+const commands = {
+    // the same broadcast probe as --discover, for an addon that bridges a CCU elsewhere in the
+    // network rather than the one it runs on
+    async discover() {
+        const found = await discover(discoveryHint({tls: Boolean(args.tls)}), {
+            timeout: Number(args.timeout || 4000),
+            // broadcast only: a subnet sweep can run for minutes, which is not something a web
+            // page should wait for. `hm2mqtt --discover` on a shell still does the full search.
+            sweep: false,
+        });
+        return {
+            ccus: [...found.values()].map((entry) => ({
+                address: entry.address,
+                name: entry.name || entry.serial || '',
+                interfaces: interfacesOf(entry),
+            })),
+        };
+    },
+
+    async probe() {
+        const host = String(args.host || '127.0.0.1');
+        const found = await probeInterfaces(host, {tls: Boolean(args.tls), timeout: 2000});
+        return {host, interfaces: found, known: INTERFACE_NAMES};
+    },
+
+    async 'mqtt-test'() {
+        const url = String(args.url || '');
+        if (!url) throw new Error('--url is required');
+        // mqtt comes with mqtt-interfaces-core; if it cannot be resolved the test degrades to a
+        // plain connect instead of failing outright
+        const {default: mqtt} = await import('mqtt');
+        const started = Date.now();
+        const client = mqtt.connect(url, {
+            username: args.username ? String(args.username) : undefined,
+            password: args.password ? String(args.password) : undefined,
+            clientId: 'hm2mqtt-addon-test-' + Math.random().toString(16).slice(2, 8),
+            connectTimeout: 8000,
+            reconnectPeriod: 0,
+        });
+        try {
+            await new Promise((resolve, reject) => {
+                client.once('connect', resolve);
+                client.once('error', reject);
+                client.once('close', () => reject(new Error('connection closed')));
+            });
+            return {ok: true, ms: Date.now() - started};
+        } finally {
+            client.end(true);
+        }
+    },
+
+    async channels() {
+        const limit = Number(args.limit || 20);
+        const channels = await rega(args).getChannels();
+        return {
+            count: channels.length,
+            channels: channels.slice(0, limit).map(({address, name}) => ({address, name})),
+        };
+    },
+
+    async preview() {
+        const template = String(args.template || DEFAULT_ITEM_TEMPLATE);
+        const render = compileTemplate(template);
+        const limit = Number(args.limit || 5);
+        const channels = await rega(args).getChannels();
+        const datapoints = ['STATE', 'LEVEL', 'TEMPERATURE'];
+        const examples = channels.slice(0, limit).map((ch, index) => {
+            const datapoint = datapoints[index % datapoints.length];
+            const fields = {
+                channel: ch.address,
+                channelName: ch.name,
+                device: String(ch.address).split(':')[0],
+                datapoint,
+            };
+            const {name, changed} = render(fields);
+            return {channel: ch.name || ch.address, datapoint, item: name, sanitized: changed};
+        });
+        return {template, examples};
+    },
+};
+
+if (!command || !commands[command]) {
+    process.stdout.write(JSON.stringify({error: `unknown command "${command || ''}"`}) + '\n');
+    process.exit(1);
+}
+
+try {
+    const result = await commands[command]();
+    process.stdout.write(JSON.stringify(result) + '\n');
+    process.exit(0);
+} catch (error) {
+    process.stdout.write(JSON.stringify({error: error.message || String(error)}) + '\n');
+    process.exit(1);
+}