hm2mqtt 3.5.1 → 3.6.0

package/README.md

@@ -259,7 +259,51 @@ the plain format (the second `ccu-mqtt` node of a flow).
 Names come from the ReGa (devices, channels, rooms, functions), are cached in the state
 directory and re-read with `<name>/set/rega/sync`. A `--name-file` overrides single addresses.
 Without the ReGa (`--no-rega`) topics use addresses. Events of channels without a name use the
-address as well.
+address as well. On a box without ReGaHSS they come from the metadata API instead — see
+[openccu-lite](#openccu-lite).
+
+## openccu-lite
+
+[openccu-lite](https://github.com/hobbyquaker/openccu-lite) is a CCU firmware without ReGaHSS.
+hm2mqtt works there without a changed configuration: at start (and again on reconnect) it asks
+`GET http://<ccu-address>/api/meta/v1/version`, and when that answers it takes device and channel
+names, rooms and functions from the box's **metadata API** instead of from the ReGa. A CCU3,
+RaspberryMatic or OpenCCU answers 404 there and everything stays exactly as before — the same
+configuration works on both, which is what a backup restored on the other kind of box needs.
+
+Names arrive as a snapshot at start and then over the box's event stream (`/api/meta/v1/events/sse`):
+a rename in the box's UI is in the topics about a second later, without a poll and without a
+restart. The store is cached in `meta.json` in the state directory, so a start without the box
+still has the names. `<name>/set/rega/sync` re-reads the snapshot here too.
+
+**The credential.** Everything but the version call needs one:
+
+| where hm2mqtt runs | credential                                                                                            |
+| ------------------ | ----------------------------------------------------------------------------------------------------- |
+| on the box (addon) | the box's own read-only token, read from `/usr/local/etc/occulite/local-token` — nothing to configure |
+| anywhere else      | `--meta-token` (`HM2MQTT_META_TOKEN`): an API token (`olt_…`) created on the box under _Benutzer_     |
+
+Without a valid credential hm2mqtt logs one line and runs on addresses; it picks the names up as
+soon as a token works, without a restart. `--meta-url` overrides the base url when the box is
+behind a proxy or on another port. Note that openccu-lite's interface processes listen on loopback
+only, so hm2mqtt normally runs **on** the box (addon package) or goes through its XML-RPC proxy.
+
+**What has no replacement** on such a box (from openccu-lite's `docs/porting-from-rega.md`):
+
+- **System variables** and **programs**: there is no ReGa DOM. Users who need them run automation
+  in Node-RED (RedMatic), Home Assistant, or whatever the addon is bridging to.
+- **`exec()`** of HM-Script, `dom.GetObject`, `system.GetSessionVarStr` from your own code: gone.
+- **ReGa ids** (`dom.GetObject(1234)`): there are none. Refs (`<interface>.<address>`) are the identity.
+- **Service messages / alarms** (variables 40 and 41): interface-level state only.
+- **The CCU WebUI's JSON-RPC API** (`/api/homematic.cgi`, `Session.login`, `Device.listAll`,
+  `Interface.*`): not present.
+
+In hm2mqtt that means: no `status/<variable>` and `status/<program>` topics and no writes to them,
+and no `--publish-cache` (the value cache was ReGa's `getValues`). `--rega-poll-interval`,
+`--rega-poll-trigger` and the `--topic-sysvar-*`/`--topic-program-*` templates stay accepted so one
+configuration serves both kinds of box; on openccu-lite they do nothing and one log line at start
+says so. Rooms and functions are trees there — a channel in `room/eg/wohnzimmer` is reported as
+`Wohnzimmer`, the parent nodes are not added, so `hm.rooms` looks exactly as it did on a CCU.
 
 ## Home Assistant
 

package/config.js

@@ -72,6 +72,16 @@ export const OPTIONS = {
         type: 'string',
         describe: 'channel.datapoint whose event triggers a variable/program poll, e.g. BidCoS-RF:50.PRESS_SHORT',
     },
+    'meta-token': {
+        type: 'string',
+        describe:
+            'openccu-lite API token (olt_...) for names, rooms and functions - not needed on the box itself, where the local token is read',
+        secret: true,
+    },
+    'meta-url': {
+        type: 'string',
+        describe: "base url of openccu-lite's metadata api (default: http[s]://<ccu-address>)",
+    },
     'ccu-timezone': {type: 'string', describe: "IANA time zone of the CCU (default: this host's time zone)"},
     'name-file': {
         alias: 'm',

package/index.js

@@ -20,6 +20,7 @@ import {RpcServers, RpcConnection} from './lib/rpc.js';
 import {Metadata} from './lib/metadata.js';
 import {ValueStore, hmBlock, isEvent} from './lib/values.js';
 import {RegaSync} from './lib/rega.js';
+import {MetaSync, detectMeta} from './lib/meta.js';
 import {castValue, isWriteable} from './lib/cast.js';
 import {
     sanitizeName,
@@ -66,6 +67,8 @@ const SET_THROTTLE_MS = 500;
 const DEVICES_WAIT_MS = 10000;
 const STOP_TIMEOUT_MS = 1500;
 const RESOLVE_RETRY_MS = 10000;
+/** how often an unreachable box is probed for the metadata api again (H-47) */
+const REDETECT_MS = 60000;
 
 const here = path.dirname(fileURLToPath(import.meta.url));
 /** the configured CCU address (the `ccu` field of every payload); connections use the resolved ip */
@@ -149,11 +152,17 @@ const renderSysvarStatus = compileTemplate(config.topicSysvarStatus);
 const renderSysvarSet = compileTemplate(config.topicSysvarSet);
 const renderProgramStatus = compileTemplate(config.topicProgramStatus);
 const renderProgramSet = compileTemplate(config.topicProgramSet);
-/** what the plain mirror tree calls an item: the status topic without the `<name>/status/` head */
+/** the literal part of the status template - what every rendered status topic starts with */
+const statusLiteral = subscribePattern(config.topicStatus, topicFields).replace(/#$/, '');
+/** what the plain mirror tree calls an item: the status topic without the template's literal head */
 const plainItem = (topic) => {
-    const head = `${config.name}/status/`;
-    return topic.startsWith(head) ? topic.slice(head.length) : topic;
+    const remainder = templateRemainder(config.topicStatus, topic, topicFields);
+    return remainder === null || remainder === '' ? topic : remainder;
 };
+/** the set subscriptions that leave `<name>/set/` and therefore need a listen pattern */
+const listenPatterns = [...new Set([config.topicSet, config.topicSysvarSet, config.topicProgramSet])]
+    .map((template) => subscribePattern(template, topicFields))
+    .filter((pattern) => !pattern.startsWith(`${config.name}/set/`));
 const adapter = createAdapter({
     pkg,
     config,
@@ -180,6 +189,7 @@ const adapter = createAdapter({
         interfaces: enabled,
         devices: metadata.count(),
         rega: Boolean(regaSync),
+        names: regaSync ? regaSync.label : 'addresses',
         payload: payloadFormat,
     }),
     /*
@@ -190,12 +200,7 @@ const adapter = createAdapter({
      * the index is keyed by.
      */
     onSet: (parts, value, topic) => handleSetTopic(topic, value),
-    listen: Object.fromEntries(
-        [...new Set([config.topicSet, config.topicSysvarSet, config.topicProgramSet])]
-            .map((template) => subscribePattern(template, topicFields))
-            .filter((pattern) => !pattern.startsWith(`${config.name}/set/`))
-            .map((pattern) => [pattern, handleSetTopic]),
-    ),
+    listen: Object.fromEntries(listenPatterns.map((pattern) => [pattern, handleSetTopic])),
     subscriptions: {
         'paramset/#': handleParamset,
         ...(config.rpcTopics ? {'rpc/+/+/+': handleRpc} : {}),
@@ -205,6 +210,21 @@ const adapter = createAdapter({
 });
 const {log, pubStatus} = adapter;
 
+for (const warning of config.$warnings || []) {
+    log.warn(warning);
+}
+if (config.topicStatus === config.topicSet) {
+    log.warn('--topic-status and --topic-set are identical: every own publication would come back as a set');
+}
+for (const pattern of listenPatterns) {
+    const literal = pattern.replace(/#$/, '');
+    if (statusLiteral.startsWith(literal)) {
+        log.warn(
+            `the set subscription ${pattern} also matches the status topics under ${statusLiteral} - own publications there are ignored`,
+        );
+    }
+}
+
 /*
  * Running on the CCU itself, the interface processes are on loopback and the familiar
  * 2000/2001/2010/9292/8181 are only lighttpd proxies in front of them: an extra hop, XML over HTTP
@@ -212,7 +232,9 @@ const {log, pubStatus} = adapter;
  * default we probe, because node-red-contrib-ccu's config-file check stopped working on current
  * firmware (see lib/interfaces.js).
  */
-const localMode = config.local === undefined ? await detectLocal(host) : Boolean(config.local);
+// an explicit --ccu-tls is a tunnel or a proxy by definition - local mode would silently drop the
+// TLS and the ports the user asked for, so it is never probed into, only chosen with --local
+const localMode = config.local === undefined ? !config.ccuTls && (await detectLocal(host)) : Boolean(config.local);
 if (localMode) {
     log.info('local mode: BidCos over binrpc (32001/32000), hmipserver on 32010, ReGa on 8183');
 }
@@ -220,8 +242,20 @@ if (localMode) {
 const metadata = new Metadata({stateDir: config.stateDir, seedFile: path.join(here, 'paramsets.json'), log});
 metadata.load();
 
+/*
+ * Where names, rooms and functions come from (H-46): openccu-lite has no ReGaHSS, but a metadata
+ * api that answers `GET /api/meta/v1/version` and nothing else does. That one call is the whole
+ * detection - no new option, no hostname or port heuristics, so a configuration moved between a
+ * CCU and an openccu-lite box keeps working untouched. Explicit --no-rega still means "addresses
+ * only" on both.
+ */
+const metaUrl = config.metaUrl || `${config.ccuTls ? 'https' : 'http'}://${host}`;
+/** true while the box has not answered the probe at all: something may still come up there */
+let redetect = false;
+let redetectTimer = null;
 let regaSync = null;
-if (config.rega) {
+
+function createRega() {
     const rega = new Rega({
         host: ccuIp,
         port: regaPort({tls: config.ccuTls, local: localMode}),
@@ -231,7 +265,37 @@ if (config.rega) {
         password: config.ccuPassword,
         timeZone: config.ccuTimezone,
     });
-    regaSync = new RegaSync({rega, host, metadata, stateDir: config.stateDir, nameFile, log});
+    return new RegaSync({rega, host, metadata, stateDir: config.stateDir, nameFile, log});
+}
+
+function createMeta() {
+    return new MetaSync({
+        url: metaUrl,
+        token: config.metaToken,
+        insecure: config.ccuInsecure,
+        fixedUrl: Boolean(config.metaUrl),
+        metadata,
+        stateDir: config.stateDir,
+        nameFile,
+        log,
+    });
+}
+
+if (config.rega) {
+    const found = await detectMeta(metaUrl, {insecure: config.ccuInsecure});
+    if (found.ok) {
+        log.info(
+            'openccu-lite on',
+            host + ':',
+            found.version.implementation || 'metadata api v' + found.version.version,
+            '- names, rooms and functions from the metadata api',
+        );
+        regaSync = createMeta();
+    } else {
+        log.debug('no metadata api on', host, '(' + found.reason + ') - using ReGa');
+        redetect = !found.reachable;
+        regaSync = createRega();
+    }
     regaSync.load();
 }
 
@@ -560,11 +624,21 @@ async function handleSetTopic(topic, value) {
     }
     let target = topicIndex.get(topic);
     if (!target) {
-        const remainder = templateRemainder(config.topicSet, topic, topicFields);
+        // the reserved namespace stays positional whatever the template says: the address form,
+        // the rega commands and the sysvar/program names below <name>/set/ keep working
+        const head = `${config.name}/set/`;
+        const remainder = topic.startsWith(head)
+            ? topic.slice(head.length)
+            : templateRemainder(config.topicSet, topic, topicFields);
         const positional = remainder === null ? null : remainder.split('/').filter(Boolean);
         target = positional && positional.length > 0 ? resolveSet(positional, lookup) : null;
     }
     if (!target) {
+        // a set subscription that overlaps the status tree (a template whose literal part ends
+        // above it) delivers our own publications here - those are not errors
+        if (statusLiteral && topic.startsWith(statusLiteral)) {
+            return;
+        }
         throw new Error('unknown channel, variable or program');
     }
     switch (target.kind) {
@@ -778,56 +852,113 @@ function createConnection(iface) {
  */
 
 async function startRega() {
+    if (regaSync instanceof MetaSync) {
+        // one line, once: everything a ReGa box could do and this one cannot (porting invariant 4).
+        // The options stay accepted so a configuration works on both kinds of box.
+        log.info(
+            'occulite: no ReGaHSS on this box - system variables, programs and the value cache are',
+            'not available; --rega-poll-interval, --rega-poll-trigger, --publish-cache and the',
+            'sysvar/program topics do nothing here',
+        );
+    }
     try {
         await regaSync.syncNames();
     } catch (err) {
         log.warn(
-            'rega: names not available (' + err.message + '), using',
+            regaSync.label + ': names not available (' + err.message + '), using',
             Object.keys(regaSync.channelNames).length,
             'cached names',
         );
     }
-    const duplicates = Object.entries(
-        Object.values(regaSync.channelNames).reduce((acc, name) => ((acc[name] = (acc[name] || 0) + 1), acc), {}),
-    )
-        .filter(([, n]) => n > 1)
-        .map(([name]) => name);
-    if (duplicates.length > 0) {
-        log.warn(
-            'duplicate channel names share a topic:',
-            duplicates.slice(0, 20).join(', '),
-            duplicates.length > 20 ? '…' : '',
+    warnDuplicateNames();
+    attachProvider();
+    regaSync.startPolling(config.regaPollInterval);
+    if (regaSync.rega) {
+        loadCache().catch((err) => log.warn('rega getValues failed:', err.message));
+    }
+    if (config.regaNamesInterval > 0) {
+        timers.push(setInterval(() => resyncNames(), config.regaNamesInterval * 1000));
+    }
+    if (redetect) {
+        // nothing answered the probe at start (the box was down or still booting): ReGa was the
+        // fallback, and an openccu-lite that comes up later must not need a restart of this
+        // service to be recognised
+        redetectTimer = setInterval(
+            () => redetectProvider().catch((err) => log.debug('re-detection failed:', err.message)),
+            REDETECT_MS,
         );
+        timers.push(redetectTimer);
     }
+}
+
+/** The provider's events, wired the same way whichever provider it is. */
+function attachProvider() {
     regaSync.on('names', scheduleIndex);
     regaSync.on('sysvar', publishRega);
     regaSync.on('program', publishRega);
     regaSync.on('polled', () => {
         if (!regaOk) {
             regaOk = true;
-            log.info('rega connected');
+            log.info(regaSync.label + ' connected');
             updateConnected();
         }
     });
     regaSync.on('error', () => {
         if (regaOk) {
             regaOk = false;
-            log.warn('rega disconnected');
+            log.warn(regaSync.label + ' disconnected');
             updateConnected();
         }
     });
-    regaSync.startPolling(config.regaPollInterval);
-    loadCache().catch((err) => log.warn('rega getValues failed:', err.message));
-    if (config.regaNamesInterval > 0) {
-        timers.push(
-            setInterval(
-                () => regaSync.syncNames().catch((err) => log.warn('rega names re-sync failed:', err.message)),
-                config.regaNamesInterval * 1000,
-            ),
+}
+
+function warnDuplicateNames() {
+    const duplicates = Object.entries(
+        Object.values(regaSync.channelNames).reduce((acc, name) => ((acc[name] = (acc[name] || 0) + 1), acc), {}),
+    )
+        .filter(([, n]) => n > 1)
+        .map(([name]) => name);
+    if (duplicates.length > 0) {
+        log.warn(
+            'duplicate channel names share a topic:',
+            duplicates.slice(0, 20).join(', '),
+            duplicates.length > 20 ? '…' : '',
         );
     }
 }
 
+function resyncNames() {
+    return regaSync.syncNames().catch((err) => log.warn(regaSync.label + ' names re-sync failed:', err.message));
+}
+
+/** Probes a box that did not answer at start; switches to the metadata api when it now does. */
+async function redetectProvider() {
+    const found = await detectMeta(metaUrl, {insecure: config.ccuInsecure});
+    if (!found.reachable) {
+        return;
+    }
+    // something answers now - whatever it is, this question is settled
+    redetect = false;
+    if (redetectTimer) {
+        clearInterval(redetectTimer);
+        redetectTimer = null;
+    }
+    if (!found.ok) {
+        log.debug('no metadata api on', host, '(' + found.reason + ') - staying with ReGa');
+        return;
+    }
+    log.info('openccu-lite answered on', host + ': switching from ReGa to the metadata api');
+    const previous = regaSync;
+    previous.stopPolling();
+    previous.removeAllListeners();
+    regaSync = createMeta();
+    regaSync.load();
+    attachProvider();
+    regaOk = false;
+    await resyncNames();
+    scheduleIndex();
+}
+
 /** The ReGa's copy of every datapoint value: seeds the value store, published with --publish-cache. */
 async function loadCache() {
     const list = await regaSync.rega.getValues();
@@ -857,10 +988,12 @@ async function loadCache() {
 async function start() {
     adapter.start();
     ccuIp = await resolveCcu();
-    if (regaSync) {
+    if (regaSync && regaSync.rega) {
         regaSync.rega.host = ccuIp;
         regaSync.rega.url = regaSync.rega.url.replace(host, ccuIp);
         regaSync.rega.webUrl = regaSync.rega.webUrl.replace(host, ccuIp);
+    } else if (regaSync) {
+        regaSync.setHost(ccuIp);
     }
     enabled =
         parseInterfaces(config.interfaces) || (await probeInterfaces(ccuIp, {tls: config.ccuTls, local: localMode}));

package/lib/hadiscovery.js

@@ -147,17 +147,29 @@ function deviceBlock({iface, device, list, ctx, t, st, cmd, bridgeId, ignored})
 
     for (const ch of channels) {
         const consumed = new Set();
-        const e = (dp, platform, label, more = {}) =>
-            entity({
+        const e = (dp, platform, label, more = {}) => {
+            const built = entity({
                 id,
                 name,
-                item: ctx.statusTopicFor(iface, ch.ADDRESS, dp),
+                item: `${ch.index}/${dp}`,
                 platform,
                 label,
                 uid: `${ch.index}_${dp}`,
                 jsonPayloads: json,
                 ...more,
             });
+            // The topics are whole, rendered from the configured templates. entity() composes
+            // <name>/status|set/<item> for a core-style item, so exactly its composed values are
+            // replaced here; an explicit stat_t/cmd_t from `extra` and the stateless platforms
+            // (which get none) stay as they are.
+            if (built.stat_t === `${name}/status/${ch.index}/${dp}`) {
+                built.stat_t = own(dp);
+            }
+            if (built.cmd_t === `${name}/set/${ch.index}/${dp}`) {
+                built.cmd_t = scmd(dp);
+            }
+            return built;
+        };
         const fallback = ch.index === 0 ? 'Maintenance' : `Channel ${ch.index}`;
         const label = labelOf(ch, deviceName, fallback);
         const has = (dp) => Boolean(ch.description[dp]) && !ignored(iface, ch.ADDRESS, dp);

package/lib/interfaces.js

@@ -180,10 +180,8 @@ export async function detectLocal(host, {timeout = 500, connect = portOpen} = {}
         return false;
     }
     const ports = INTERFACE_NAMES.map((name) => INTERFACES[name].localPort).filter(Boolean);
-    for (const port of [...ports, REGA_LOCAL_PORT]) {
-        if (await connect(host, port, timeout)) {
-            return true;
-        }
-    }
-    return false;
+    // concurrently: this runs at every startup, and closed-but-filtered ports would otherwise
+    // stack their timeouts
+    const results = await Promise.all([...ports, REGA_LOCAL_PORT].map((port) => connect(host, port, timeout)));
+    return results.some(Boolean);
 }