homematic-manager 3.0.0-beta.2 → 3.0.0-beta.21

package/node_modules/@homematic-manager/backend/dist/api/backend.js

@@ -14,7 +14,7 @@
  * main process with it, issue #127).
  */
 import path from 'node:path';
-import { RPC_METHOD_NAMES, RSSI_UNKNOWN, countsAsServiceMessage, isAcknowledgeable, maintenanceAddress, mergeMethodHelp, methodsFor, } from '@homematic-manager/core';
+import { RPC_METHOD_NAMES, RSSI_UNKNOWN, countsAsServiceMessage, isAcknowledgeable, isKnownInterface, maintenanceAddress, mergeMethodHelp, methodsFor, callbackPinOption, normaliseDescription, repairMisdecodedUtf8, } from '@homematic-manager/core';
 import { CacheStore } from '../cache/store.js';
 import { ConfigStore } from '../config/store.js';
 import { LinkTemplateStore } from '../config/linkTemplates.js';
@@ -23,12 +23,14 @@ import { DataFileServer } from '../data/files.js';
 import { installModeCalls } from '../devices/installMode.js';
 import { discoverCcus } from '../discovery/discover.js';
 import { BackendError, configError, connectionError, errorMessage, internalError, isMethodUnsupported, validationError, } from '../errors.js';
-import { InterfaceManager, firstBidcosInterfaceAddress } from '../interfaces/manager.js';
+import { InterfaceManager, LOOPBACK_IP, callbackBindHost, firstBidcosInterfaceAddress, } from '../interfaces/manager.js';
+import { MetaService } from '../meta/service.js';
 import { RegaService } from '../rega/client.js';
 import { listDevicesAnswer } from '../rpc/server.js';
 import { ApiEventEmitter } from '../util/emitter.js';
-import { WriteLog } from '../write/log.js';
-import { isWriteMethod } from '../write/log.js';
+import { describeCallbackAddresses, staticNetwork, systemNetwork } from '../util/net.js';
+import { RpcLog, isWriteMethod } from '../rpc/log.js';
+import { currentOrigin, runWithOrigin } from '../rpc/origin.js';
 import { ParamsetWriter } from '../write/paramset.js';
 import { ConfigRepair } from '../write/repair.js';
 import { WriteQueue } from '../write/queue.js';
@@ -36,13 +38,26 @@ import { WriteQueue } from '../write/queue.js';
 export const SERVICE_MESSAGE_POLL_MS = 300_000;
 /** How long the HmIP `getParamset(:0, VALUES)` sweep waits for the device list to settle. */
 export const HMIP_SWEEP_DELAY_MS = 1000;
+/**
+ * An HmIP interface, by name or by interface type - the same rule the UI's settings dialog uses to
+ * count the messages it asks about (task 34), so the question and the writes agree.
+ */
+function isHmipInterface(name, type) {
+    return /hmip/i.test(name) || /hmip/i.test(type);
+}
+/** Task 38: how a log line names a callback field. */
+const CALLBACK_FIELD_WORDS = {
+    ip: 'callback address',
+    xmlrpcPort: 'XML-RPC callback port',
+    binrpcPort: 'BIN-RPC callback port',
+};
 /** Implements the whole contract. `open()` loads, `start()` connects, `stop()` disconnects. */
 export class Backend {
     events;
     #options;
     #config;
     #queue;
-    #writeLog;
+    #rpcLog;
     #unknownMethodsSeen = new Set();
     #writer;
     #repair;
@@ -54,6 +69,19 @@ export class Backend {
     #caches;
     #manager;
     #rega;
+    /** D-40: the metadata store of this connection - this profile's own, or an openccu-lite box. */
+    #meta;
+    /** D-40: the session a host with a login knows about; writes to the box go out as this one. */
+    #metaSession;
+    /**
+     * D-40: the detection, in flight.
+     *
+     * The probe is **not** awaited by `#connect`: a host that swallows packets on port 80 - a
+     * firewall, a CCU that is off, a WSL loopback - would otherwise hold the whole connection up
+     * for the detection timeout while the interfaces are already there. Everything that needs the
+     * store waits for this promise instead, and the UI hears about it through `meta.changed`.
+     */
+    #metaReady;
     #serviceMessageTimer;
     #hmipSweepTimer;
     #hmipSweepRunning = false;
@@ -65,7 +93,23 @@ export class Backend {
      * table - and it is emptied whenever the connection is rebuilt.
      */
     #noServiceMessages = new Set();
+    /**
+     * B-26 (#158): interfaces whose last `getServiceMessages` failed for a reason that may pass - a
+     * timeout, a process that is restarting. The failure is logged when the interface lands in
+     * here and not again until a call succeeds; before, the five-minute poll logged it every round.
+     */
+    #serviceMessageFailures = new Set();
+    /**
+     * B-26 (#158): the `system.listMethods` answer of a user-defined interface, asked once and
+     * dropped when the interface subscribes again - a re-`init` may be a restarted, updated process.
+     * `undefined` inside means "no usable list", and the method is then tried once, as before.
+     */
+    #listedMethods = new Map();
+    /** B-26: interfaces that do not have `system.listMethods` itself; kept for the connection. */
+    #noListMethods = new Set();
     #idleTimer;
+    /** Task 38: the saved callback values a pinned one replaces are logged by the first `start()` only. */
+    #pinsReported = false;
     constructor(options, config, caches) {
         this.#options = options;
         this.#config = config;
@@ -82,14 +126,14 @@ export class Backend {
         this.#queue = new WriteQueue({
             paceFor: (interfaceName) => writePaceFor(interfaceName, this.#config.connection.writePaceMs),
         });
-        this.#writeLog = new WriteLog({
+        this.#rpcLog = new RpcLog({
             file: config.cacheFile('write-log.json'),
             rpcLogFolder: config.connection.rpcLogFolder,
             onAppended: (entry) => {
-                this.events.emit('writeLog.appended', entry);
+                this.events.emit('rpcLog.appended', entry);
             },
             onError: (error) => {
-                this.#notice('warn', `write log: ${errorMessage(error)}`);
+                this.#notice('warn', `RPC log: ${errorMessage(error)}`);
             },
             ...(options.cacheWriteDelayMs === undefined ? {} : { writeDelayMs: options.cacheWriteDelayMs }),
         });
@@ -123,7 +167,7 @@ export class Backend {
         });
         await caches.load();
         const backend = new Backend(options, config, caches);
-        await backend.#writeLog.load();
+        await backend.#rpcLog.load();
         await backend.#linkTemplates.load();
         return backend;
     }
@@ -137,6 +181,14 @@ export class Backend {
         if (this.#config.importedFromLegacy) {
             this.#notice('info', 'the configuration of Homematic Manager 2.x was imported');
         }
+        if (!this.#pinsReported) {
+            // Task 38: once per process - the value in config.json stays, it only has no effect
+            this.#pinsReported = true;
+            for (const { field, saved, pinned } of this.#config.ignoredCallback) {
+                this.#notice('info', `callback: the saved ${CALLBACK_FIELD_WORDS[field]} ${String(saved)} is ignored, ` +
+                    `${callbackPinOption(field)} sets ${String(pinned)}`);
+            }
+        }
         const problems = validateConnection(this.#config.connection);
         if (problems.length > 0) {
             this.#notice('info', `not connecting: ${problems.join(', ')}`);
@@ -152,8 +204,17 @@ export class Backend {
         await this.#manager?.stop();
         this.#manager = undefined;
         this.#rega = undefined;
+        // D-40: before the caches are flushed, and awaited even when it is only the detection that
+        // is still in flight. A store that finishes loading after this would apply its names, ask
+        // for a cache write and put a file back into a directory the caller has already taken
+        // apart - which is how CI found this, as an ENOTEMPTY in a suite that has nothing to do
+        // with metadata.
+        await this.#metaReady?.catch(() => undefined);
+        this.#metaReady = undefined;
+        await this.#meta?.stop();
+        this.#meta = undefined;
         await this.#caches.flush();
-        await this.#writeLog.flush();
+        await this.#rpcLog.flush();
         this.events.clear();
     }
     /**
@@ -176,7 +237,7 @@ export class Backend {
         }
         if (this.#sessions > 0) {
             if (previous === 0) {
-                void this.#resubscribe();
+                void runWithOrigin('background', () => this.#resubscribe());
             }
             return;
         }
@@ -186,7 +247,7 @@ export class Backend {
         }
         const timer = setTimeout(() => {
             this.#idleTimer = undefined;
-            void this.#unsubscribeIdle();
+            void runWithOrigin('background', () => this.#unsubscribeIdle());
         }, grace);
         if (typeof timer.unref === 'function') {
             timer.unref();
@@ -219,7 +280,9 @@ export class Backend {
      */
     async request(method, ...params) {
         try {
-            return await this.#dispatch(method, params);
+            // Task 48: everything a request awaits is a UI action for the RPC log; the console's
+            // own call is the console's. Whatever runs outside a request is background work.
+            return await runWithOrigin(method === 'rpc.call' ? 'console' : 'ui', () => this.#dispatch(method, params));
         }
         catch (error) {
             throw error instanceof BackendError ? error : internalError(errorMessage(error), error);
@@ -240,11 +303,13 @@ export class Backend {
             case 'config.get':
                 return this.#configWithDetected();
             case 'config.set':
-                return this.#setConfig(p[0]);
+                return this.#setConfig(p[0], params[1]);
             case 'config.discover':
                 return this.#discover();
             case 'config.clearCaches':
                 return this.#clearCaches();
+            case 'config.callbackAddresses':
+                return this.#callbackAddresses(p[0]);
             case 'interfaces.list':
                 return this.#manager?.states() ?? [];
             case 'interfaces.reconnect':
@@ -284,6 +349,51 @@ export class Backend {
                 return this.#caches.names.all();
             case 'names.set':
                 return this.#setNames(p[0]);
+            case 'meta.state':
+                return this.#metaState();
+            case 'meta.get':
+                return this.#metaSnapshot();
+            case 'meta.enums':
+                return (await this.#requireMeta()).enums();
+            case 'meta.objects':
+                return (await this.#requireMeta()).objects();
+            case 'meta.setMembership':
+                await (await this.#requireMeta()).setMembership(p[0]);
+                return null;
+            case 'meta.assign':
+                await (await this.#requireMeta()).assign(p[0], p[1], p[2]);
+                return null;
+            case 'meta.enum.create':
+                await (await this.#requireMeta()).createEnum(p[0], p[1]);
+                return null;
+            case 'meta.enum.update':
+                await (await this.#requireMeta()).updateEnum(p[0], p[1]);
+                return null;
+            case 'meta.enum.delete':
+                await (await this.#requireMeta()).deleteEnum(p[0], p[1] === true);
+                return null;
+            case 'meta.node.create':
+                return (await this.#requireMeta()).createNode(p[0], 
+                // absent means "at the root"; see the contract for why it is not `null` here.
+                // Read from `params` and not from `p`, whose slots are `never`: this is the one
+                // place that has to tell "absent" from a value, and `never` cannot.
+                params[1] ?? null, p[2], params[3] ?? {});
+            case 'meta.node.update':
+                await (await this.#requireMeta()).updateNode(p[0], params[1]);
+                return null;
+            case 'meta.node.delete':
+                await (await this.#requireMeta()).deleteNode(p[0], p[1] === true);
+                return null;
+            case 'meta.refresh': {
+                const meta = await this.#requireMeta();
+                await meta.refresh();
+                return meta.state();
+            }
+            case 'meta.export':
+                return (await this.#requireMeta()).document();
+            case 'meta.import':
+                await (await this.#requireMeta()).import(p[0], params[1] ?? 'replace');
+                return null;
             case 'paramset.get':
                 return this.#getParamset(p[0], p[1], p[2]);
             case 'paramset.description':
@@ -348,6 +458,8 @@ export class Backend {
                 return this.#resetUnreach(p[0], p[1]);
             case 'serviceMessages.list':
                 return this.#serviceMessages(p[0]);
+            case 'serviceMessages.refresh':
+                return this.#refreshAllServiceMessages(p[0]);
             case 'serviceMessages.ack':
                 return this.#acknowledge(p[0], p[1], p[2]);
             case 'events.recent':
@@ -361,10 +473,10 @@ export class Backend {
                 return this.#methods(p[0]);
             case 'write.cancel':
                 return this.#queue.cancel(p[0]);
-            case 'writeLog.list':
-                return this.#writeLog.list(p[0]);
-            case 'writeLog.clear':
-                this.#writeLog.clear();
+            case 'rpcLog.list':
+                return this.#rpcLog.list(p[0]);
+            case 'rpcLog.clear':
+                this.#rpcLog.clear();
                 return null;
             case 'data.file':
                 return this.#files.read(p[0]);
@@ -386,9 +498,20 @@ export class Backend {
         }
         return this.#manager;
     }
-    async #connect() {
+    /**
+     * Task 48: the connection is background work even when a `config.set` from the settings dialog
+     * asks for it - the watchdog, the callback servers and the polling timers it creates would
+     * otherwise inherit that request's origin for the rest of the session.
+     */
+    #connect() {
+        return runWithOrigin('background', () => this.#connectNow());
+    }
+    async #connectNow() {
         const connection = this.#config.connection;
         this.#noServiceMessages.clear();
+        this.#serviceMessageFailures.clear();
+        this.#listedMethods.clear();
+        this.#noListMethods.clear();
         const manager = (this.#options.createInterfaceManager ?? ((options) => new InterfaceManager(options)))({
             connection,
             handler: this.#callbackHandler(),
@@ -398,16 +521,23 @@ export class Backend {
             onNotice: (level, message, interfaceName) => {
                 this.#notice(level, message, interfaceName);
             },
-            onConnected: (interfaceName) => this.#onInterfaceConnected(interfaceName),
+            onConnected: (interfaceName) => runWithOrigin('background', () => this.#onInterfaceConnected(interfaceName)),
             onCall: (record) => {
                 this.#onCall(record);
             },
+            originOf: currentOrigin,
             ...(this.#options.callbackHost === undefined ? {} : { callbackHost: this.#options.callbackHost }),
+            ...(this.#options.defaultCallbackPorts === undefined
+                ? {}
+                : { defaultCallbackPorts: this.#options.defaultCallbackPorts }),
+            ...(this.#config.callbackPins === undefined ? {} : { callbackPins: this.#config.callbackPins }),
             ...(this.#options.rpcTimeoutMs === undefined ? {} : { rpcTimeoutMs: this.#options.rpcTimeoutMs }),
             ...(this.#options.watchdogIntervalMs === undefined
                 ? {}
                 : { watchdogIntervalMs: this.#options.watchdogIntervalMs }),
             ...(this.#options.localAddresses === undefined ? {} : { localAddresses: this.#options.localAddresses }),
+            ...(this.#options.network === undefined ? {} : { network: this.#options.network }),
+            ...(this.#keepsConfiguredCallbackIp() ? { keepConfiguredCallbackIp: true } : {}),
             ...(this.#options.now === undefined ? {} : { now: this.#options.now }),
             ...this.#options.interfaceManagerOptions,
         });
@@ -444,6 +574,7 @@ export class Backend {
             this.#caches.saveNames();
             this.events.emit('names.changed', this.#caches.names.all());
         }
+        this.#metaReady = this.#startMeta(connection);
         this.#startServiceMessagePolling();
     }
     async #disconnect() {
@@ -452,8 +583,104 @@ export class Backend {
         await this.#manager?.stop();
         this.#manager = undefined;
         this.#rega = undefined;
+        // the detection may still be in flight; stopping a provider that is not there yet would
+        // leave its event stream running behind the disconnect
+        await this.#metaReady?.catch(() => undefined);
+        this.#metaReady = undefined;
+        await this.#meta?.stop();
+        this.#meta = undefined;
         this.#methodCache.clear();
     }
+    /**
+     * D-40: picks the metadata provider for this connection and starts it.
+     *
+     * After the interfaces, because the ref of an address (`<interface>.<address>`) is resolved
+     * from the device caches and those are filled by the `listDevices` sweep the interfaces do on
+     * connect. Before that the store's names are still applied - they are keyed by ref, and the
+     * address half of a ref never needs an interface to be readable.
+     */
+    async #startMeta(connection) {
+        try {
+            const meta = await MetaService.create({
+                connection,
+                dataDir: this.#options.dataDir,
+                cacheDir: this.#config.cacheDir,
+                names: this.#caches.names,
+                interfaceOf: (address) => this.#interfaceOf(address),
+                // task 27: ReGa as the store of rooms and functions on a CCU. Read through the
+                // service that is current at call time - a reconnect replaces it.
+                rega: this.#rega === undefined || !connection.rega
+                    ? undefined
+                    : {
+                        available: this.#rega.available,
+                        exec: (script) => this.#requireRega().exec(script),
+                    },
+                onChanged: () => {
+                    this.#onMetaChanged();
+                },
+                onStateChanged: (state) => {
+                    this.events.emit('meta.changed', state);
+                },
+                onNotice: (level, message) => {
+                    this.#notice(level, message);
+                },
+                ...this.#options.metaOptions,
+            });
+            if (this.#stopped) {
+                // stopped while the box was being probed; starting the provider now would open an
+                // event stream nobody will ever close
+                return;
+            }
+            meta.setSessionCredential(this.#metaSession);
+            this.#meta = meta;
+            await meta.start();
+            this.#onMetaChanged();
+        }
+        catch (error) {
+            // D-2 in its metadata shape: a store that cannot be opened costs the taxonomy, never
+            // the application
+            this.#notice('warn', `the metadata store could not be opened: ${errorMessage(error)}`);
+        }
+    }
+    #requireRega() {
+        if (!this.#rega) {
+            throw configError('ReGa is not connected');
+        }
+        return this.#rega;
+    }
+    /** The store changed - locally, or on the box because somebody else edited it. */
+    #onMetaChanged() {
+        const meta = this.#meta;
+        if (!meta || this.#stopped) {
+            return;
+        }
+        this.#caches.saveNames();
+        this.events.emit('names.changed', this.#caches.names.all());
+        this.events.emit('meta.enums.changed', meta.enums());
+        this.events.emit('meta.objects.changed', meta.objects());
+    }
+    /** Which interface reports an address; the ref of the metadata store is built from it. */
+    #interfaceOf(address) {
+        const device = address.split(':')[0] ?? address;
+        for (const interfaceName of this.#caches.devices.interfaces()) {
+            if (this.#caches.devices.get(interfaceName, address) ?? this.#caches.devices.get(interfaceName, device)) {
+                return interfaceName;
+            }
+        }
+        return undefined;
+    }
+    /**
+     * D-40: the session of the person looking at the page, for the writes to an openccu-lite box.
+     *
+     * The shell of the box hands an addon page the user's session as `?sid=@xxxxxxxxxx@`, and that
+     * session is a valid credential for the metadata API. The addon reads with the box's local
+     * token, which is read-only by design, and writes as the user - so nothing renames a device
+     * unless a person asked for it. Hosts without a login never call this.
+     */
+    noteMetaSession(sid) {
+        this.#metaSession = sid;
+        this.#meta?.setSessionCredential(sid);
+    }
     #clearTimers() {
         if (this.#idleTimer !== undefined) {
             clearTimeout(this.#idleTimer);
@@ -470,6 +697,8 @@ export class Backend {
     }
     /** Fills the caches of an interface that has just subscribed. */
     async #onInterfaceConnected(interfaceName) {
+        // B-26: a (re-)`init` may be a restarted process, so its method list is asked again
+        this.#listedMethods.delete(interfaceName);
         try {
             await this.#refreshDevices(interfaceName);
         }
@@ -589,13 +818,16 @@ export class Backend {
      * a device that is unreachable is not going to take a write either, which is the normal case
      * here and not worth an error dialog.
      */
-    #noteUnreach(interfaceName, address, datapoint, value) {
+    #noteUnreach(interfaceName, address, datapoint, value, options = {}) {
         if (!this.#caches.unreach.note(interfaceName, address, datapoint, value, this.#now())) {
             return;
         }
         this.#caches.saveUnreach();
         this.events.emit('unreach.changed', this.#caches.unreach.list());
-        if (datapoint === 'STICKY_UNREACH' && value === true && this.#config.connection.autoAckStickyUnreach === true) {
+        if (options.acknowledge !== false &&
+            datapoint === 'STICKY_UNREACH' &&
+            value === true &&
+            this.#config.connection.autoAckStickyUnreach === true) {
             void this.#acknowledge(interfaceName, address, datapoint).catch((error) => {
                 this.#notice('info', `${address}: STICKY_UNREACH could not be acknowledged automatically: ${errorMessage(error)}`, interfaceName);
             });
@@ -617,7 +849,7 @@ export class Backend {
      * RPC
      */
     #onCall(record) {
-        this.#writeLog.append(record);
+        this.#rpcLog.append(record);
     }
     /** A read: straight to the interface, never queued. */
     async #read(interfaceName, method, params) {
@@ -626,19 +858,45 @@ export class Backend {
     /** A write: through the paced queue of that interface. */
     async #write(interfaceName, method, params) {
         const client = this.#requireManager().client(interfaceName);
-        return this.#queue.enqueue(interfaceName, () => client.call(method, params));
+        // Task 48: the queue runs a task from the timer that drained it, which is the *previous*
+        // task's context; the origin is the one of whoever enqueued, so it is taken here.
+        const origin = currentOrigin();
+        return this.#queue.enqueue(interfaceName, () => client.call(method, params, { origin }));
     }
     /*
      * configuration
      */
     #configWithDetected() {
-        return this.#config.config;
+        return this.#withHostFacts(this.#config.config);
+    }
+    /**
+     * What the host adds to the stored configuration before anybody sees it: task 35's callback
+     * ports a `0` stands for. Added to every `AppConfig` that leaves the backend - `config.get`,
+     * `config.set` and both `config.changed` events - so the dialog never loses the hint on a save.
+     */
+    #withHostFacts(config) {
+        const ports = this.#options.defaultCallbackPorts;
+        const pins = this.#config.callbackPins;
+        // Task 38: a container whose callback servers listen on the loopback only has nothing to publish
+        const bindHost = this.#options.callbackHost ?? callbackBindHost(config.connection);
+        const publish = this.#options.inContainer === true && bindHost !== LOOPBACK_IP;
+        return {
+            ...config,
+            ...(ports === undefined ? {} : { callbackDefaultPorts: { xmlrpc: ports.xmlrpc, binrpc: ports.binrpc } }),
+            ...(pins === undefined ? {} : { callbackPinned: { ...pins } }),
+            ...(publish ? { publishCallbackPorts: true } : {}),
+        };
     }
-    async #setConfig(connection) {
+    async #setConfig(connection, options) {
         const previousHost = this.#config.connection.host;
+        // Task 34 (#147, D-42): the messages the settings dialog asked about, taken before the
+        // reconnect, and only on the save that switches the auto-acknowledge from off to on.
+        const existing = options?.acknowledgeExisting === true && this.#config.connection.autoAckStickyUnreach !== true
+            ? this.#stickyUnreachMessages()
+            : [];
         await this.#disconnect();
-        const config = await this.#config.setConnection(connection);
-        this.#writeLog.setRpcLogFolder(config.connection.rpcLogFolder);
+        const config = this.#withHostFacts(await this.#config.setConnection(connection));
+        this.#rpcLog.setRpcLogFolder(config.connection.rpcLogFolder);
         if (config.connection.host !== previousHost) {
             await this.#caches.flush();
             this.#caches = new CacheStore({
@@ -653,14 +911,82 @@ export class Backend {
         this.events.emit('config.changed', config);
         if (!this.#stopped && validateConnection(config.connection).length === 0) {
             await this.#connect();
+            // after the reconnect, because a write needs the interface; not awaited, so the
+            // settings dialog closes as quickly as ever and the writes show up in the RPC log
+            if (existing.length > 0 &&
+                config.connection.autoAckStickyUnreach === true &&
+                config.connection.host === previousHost) {
+                void this.#acknowledgeExisting(existing);
+            }
         }
         return config;
     }
+    /**
+     * Task 34: the `STICKY_UNREACH` messages in the list, without HmIP's.
+     *
+     * D-42 says nothing is acknowledged on HmIP: hmipserver has no such flag on real hardware, and
+     * where one appears anyway (a simulator, a future firmware) this is not the switch that should
+     * start writing to those devices.
+     */
+    #stickyUnreachMessages() {
+        const states = this.#manager?.states() ?? [];
+        return this.#caches.listServiceMessages().filter((message) => {
+            if (message.datapoint !== 'STICKY_UNREACH' || message.value === false) {
+                return false;
+            }
+            const type = states.find((state) => state.name === message.interfaceName)?.type ?? '';
+            return !isHmipInterface(message.interfaceName, type);
+        });
+    }
+    /**
+     * Task 34 (#147, D-42): acknowledges the messages that were in the list when the option was
+     * switched on, one after another, each with exactly the write of the acknowledge button
+     * (`#acknowledge`, the paced `setValue`). A message that is gone by now - the device came back
+     * and somebody acknowledged it, or the edge above got there first - is skipped rather than
+     * written twice. A failure is a notice and the write log's red line, as for the edge.
+     */
+    async #acknowledgeExisting(messages) {
+        for (const message of messages) {
+            const listed = this.#caches
+                .listServiceMessages(message.interfaceName)
+                .some((entry) => entry.address === message.address && entry.datapoint === message.datapoint);
+            if (!listed || this.#stopped) {
+                continue;
+            }
+            try {
+                await this.#acknowledge(message.interfaceName, message.address, message.datapoint);
+            }
+            catch (error) {
+                this.#notice('info', `${message.address}: STICKY_UNREACH could not be acknowledged: ${errorMessage(error)}`, message.interfaceName);
+            }
+        }
+    }
+    /**
+     * B-53: a callback address set at start (task 38) or inside a container is taken as it is - the
+     * Docker host's address is none of the container's, and it is still the right one.
+     */
+    #keepsConfiguredCallbackIp() {
+        return this.#options.inContainer === true || this.#config.callbackPins?.ip === true;
+    }
+    #network() {
+        if (this.#options.network !== undefined) {
+            return this.#options.network;
+        }
+        const injected = this.#options.localAddresses;
+        return injected === undefined ? systemNetwork() : staticNetwork(injected);
+    }
+    /** B-53: `config.callbackAddresses` - for the host being typed, or the configured one. */
+    async #callbackAddresses(host) {
+        const { connection } = this.#config;
+        const target = typeof host === 'string' ? host.trim() : connection.host;
+        const info = await describeCallbackAddresses(target, this.#network(), connection.local === true);
+        return this.#keepsConfiguredCallbackIp() ? { ...info, keepsConfigured: true } : info;
+    }
     async #discover() {
         const discover = this.#options.discover ?? ((options) => discoverCcus(options));
         const found = await discover({ tls: this.#config.connection.tls });
         this.#config.setDiscovered(found);
-        this.events.emit('config.changed', this.#config.config);
+        this.events.emit('config.changed', this.#withHostFacts(this.#config.config));
         return found;
     }
     async #clearCaches() {
@@ -712,7 +1038,14 @@ export class Backend {
         return null;
     }
     async #setInstallMode(interfaceName, on, options) {
-        for (const call of installModeCalls(on, options ?? {})) {
+        const effective = { ...options };
+        if (interfaceName === 'HmIP-RF') {
+            // Task 28, measured in the lab: hmipserver's third `setInstallMode` parameter is a String,
+            // and `setInstallMode(true, 30, 1)` got an empty HTTP reply and no install mode at all.
+            // BidCos's mode integer never reaches HmIP, whoever asks for it.
+            delete effective.mode;
+        }
+        for (const call of installModeCalls(on, effective)) {
             await this.#write(interfaceName, call.method, call.params);
         }
         return null;
@@ -723,11 +1056,68 @@ export class Backend {
     async #setNames(entries) {
         const written = this.#caches.names.set(entries);
         this.#caches.saveNames();
-        await this.#rega?.rename(written);
+        // task 27: with ReGa as the metadata store the provider writes the rename itself, and a
+        // second `Name()` through the name service would be the same script twice - but only for
+        // the objects the provider holds, and only while ReGa answers it; anything else still goes
+        // the way it always did (found by the e2e suite: a rename lost for good is worse than one
+        // script twice)
+        const meta = this.#meta;
+        const held = meta?.kind === 'rega' && meta.state().reachable ? meta.document().objects : undefined;
+        const throughNames = held === undefined
+            ? written
+            : written.filter((entry) => {
+                const ref = meta?.refFor(entry.address);
+                return ref === undefined || !(ref in held);
+            });
+        if (throughNames.length > 0) {
+            await this.#rega?.rename(throughNames);
+        }
+        // D-40: and into the metadata store, which on an openccu-lite box is the box's own. The
+        // local cache is written first either way, so a store that refuses the write still leaves
+        // the name where the user typed it - and the refusal is reported rather than swallowed.
+        await this.#meta?.setNames(written);
         const names = this.#caches.names.all();
         this.events.emit('names.changed', names);
         return names;
     }
+    /*
+     * the metadata store (D-40)
+     */
+    /** The state, even before a connection exists - the settings dialog asks for it either way. */
+    #metaState() {
+        return (this.#meta?.state() ?? {
+            provider: this.#config.connection.metaProvider === 'occulite' ||
+                this.#config.connection.metaProvider === 'rega'
+                ? this.#config.connection.metaProvider
+                : 'local',
+            reachable: false,
+            writable: false,
+            revision: 0,
+            objects: 0,
+        });
+    }
+    #metaSnapshot() {
+        const meta = this.#meta;
+        return {
+            state: this.#metaState(),
+            enums: meta?.enums() ?? {},
+            objects: meta?.objects() ?? {},
+        };
+    }
+    /**
+     * The store, or a typed refusal.
+     *
+     * There is no connection without one - `#connect` always builds a provider, the `local` one
+     * when there is no box - so this only fires before the first connect, which is exactly when a
+     * UI should be told "not connected" rather than shown an empty taxonomy.
+     */
+    async #requireMeta() {
+        await this.#metaReady?.catch(() => undefined);
+        if (!this.#meta) {
+            throw configError('the metadata store is not open yet: connect to a system first');
+        }
+        return this.#meta;
+    }
     /*
      * paramsets
      */
@@ -800,27 +1190,95 @@ export class Backend {
     #serviceMessages(interfaceName) {
         return this.#caches.listServiceMessages(interfaceName);
     }
+    /**
+     * Issue #146: what the refresh button of the service-message tab asks for.
+     *
+     * The list itself comes out of the cache, which the events and a five-minute poll keep up to
+     * date; pressing refresh re-read that cache and therefore changed nothing at all - "Refresh
+     * Button ohne Funktion". This makes the round trip the button promises: `getServiceMessages`
+     * on every connected interface that has the method, and the `:0` sweep on HmIP, which is where
+     * its messages come from. Failures are notices, as everywhere else here - an interface that
+     * does not answer must not turn the button into an error dialog.
+     */
+    async #refreshAllServiceMessages(interfaceName) {
+        const names = (this.#manager?.names() ?? []).filter((name) => (interfaceName === undefined || name === interfaceName) && this.#manager?.isConnected(name) === true);
+        for (const name of names) {
+            if (name === 'HmIP-RF') {
+                await this.sweepHmip(name);
+            }
+            else {
+                await this.#refreshServiceMessages(name);
+            }
+        }
+        return this.#serviceMessages(interfaceName);
+    }
     /**
      * Does it make sense to ask this interface for its service messages?
      *
      * The built-in answer comes from the core's table: hmipserver has no `getServiceMessages` (the
-     * HmIP sweep below reads the `:0` channels instead) and the group process behind
-     * `VirtualDevices` answers it with invalid XML-RPC. A user-defined interface is asked once and
-     * then remembered for this session - see {@link isMethodUnsupported}.
+     * HmIP sweep below reads the `:0` channels instead), the group process behind `VirtualDevices`
+     * answers it with invalid XML-RPC, and CUxD with a fault and a warning in the CCU's syslog
+     * (B-26, #158). A user-defined interface is judged by its own `system.listMethods` where that
+     * gives a list, and is otherwise asked once and then remembered for this session - see
+     * {@link isMethodUnsupported}.
      */
-    #hasServiceMessages(interfaceName) {
+    async #hasServiceMessages(interfaceName) {
         if (this.#noServiceMessages.has(interfaceName)) {
             return false;
         }
-        return this.#manager?.resolved(interfaceName)?.serviceMessages !== false;
+        if (this.#manager?.resolved(interfaceName)?.serviceMessages === false) {
+            return false;
+        }
+        // the table is measured for the built-in processes; asking rfd for its method list after
+        // every `init` would buy nothing
+        if (isKnownInterface(interfaceName)) {
+            return true;
+        }
+        const listed = await this.#methodList(interfaceName);
+        return listed === undefined || listed.has('getServiceMessages');
+    }
+    /**
+     * B-26 (#158): the methods a user-defined interface lists in `system.listMethods`, or
+     * `undefined` when it gave no usable list.
+     *
+     * Asked once and shared by concurrent callers; the answer is dropped when the interface
+     * subscribes again. A process that does not have `system.listMethods` either is remembered for
+     * the connection, so it is not asked after every re-`init`; any other failure is forgotten, and
+     * the next caller asks again.
+     */
+    #methodList(interfaceName) {
+        if (this.#noListMethods.has(interfaceName)) {
+            return Promise.resolve(undefined);
+        }
+        const cached = this.#listedMethods.get(interfaceName);
+        if (cached) {
+            return cached;
+        }
+        const pending = this.#read(interfaceName, 'system.listMethods', []).then((answer) => {
+            const names = asStrings(answer);
+            return names.length === 0 ? undefined : new Set(names);
+        }, (error) => {
+            if (isMethodUnsupported(error)) {
+                this.#noListMethods.add(interfaceName);
+            }
+            if (this.#listedMethods.get(interfaceName) === pending) {
+                this.#listedMethods.delete(interfaceName);
+            }
+            return undefined;
+        });
+        this.#listedMethods.set(interfaceName, pending);
+        return pending;
     }
     /** Reads the BidCos service messages of one interface into the store. */
     async #refreshServiceMessages(interfaceName) {
-        if (!this.#hasServiceMessages(interfaceName)) {
+        if (!(await this.#hasServiceMessages(interfaceName))) {
             return;
         }
         try {
             const answer = await this.#read(interfaceName, 'getServiceMessages', []);
+            if (this.#serviceMessageFailures.delete(interfaceName)) {
+                this.#notice('info', `${interfaceName}: getServiceMessages answers again`, interfaceName);
+            }
             if (!Array.isArray(answer)) {
                 return;
             }
@@ -847,8 +1305,14 @@ export class Backend {
             // of it on hardware (task 17). Remembered, and never asked again this session.
             if (isMethodUnsupported(error)) {
                 this.#noServiceMessages.add(interfaceName);
+                this.#serviceMessageFailures.delete(interfaceName);
+                return;
+            }
+            // B-26 (#158): once per interface until a call succeeds again, not once per poll
+            if (this.#serviceMessageFailures.has(interfaceName)) {
                 return;
             }
+            this.#serviceMessageFailures.add(interfaceName);
             this.#notice('info', `${interfaceName}: getServiceMessages failed: ${errorMessage(error)}`, interfaceName);
         }
     }
@@ -858,7 +1322,7 @@ export class Backend {
             return;
         }
         const timer = setInterval(() => {
-            void this.pollServiceMessages();
+            void runWithOrigin('background', () => this.pollServiceMessages());
         }, interval);
         if (typeof timer.unref === 'function') {
             timer.unref();
@@ -886,7 +1350,7 @@ export class Backend {
         }
         const timer = setTimeout(() => {
             this.#hmipSweepTimer = undefined;
-            void this.sweepHmip();
+            void runWithOrigin('background', () => this.sweepHmip());
         }, this.#options.hmipSweepDelayMs ?? HMIP_SWEEP_DELAY_MS);
         if (typeof timer.unref === 'function') {
             timer.unref();
@@ -921,11 +1385,27 @@ export class Backend {
             this.events.emit('serviceMessages.changed', this.#caches.listServiceMessages());
         }
     }
-    /** Files the RSSI and the service messages of a `getParamset(<device>:0, VALUES)` answer. */
+    /**
+     * Files the RSSI, the service messages and the unreach state of a `getParamset(<device>:0,
+     * VALUES)` answer.
+     *
+     * Task 34 (B-9): the unreach state goes through `#noteUnreach` like the BidCos poll's, so the
+     * Funk tab counts the outages of HmIP devices that no event reported. `UNREACH` is what says
+     * whether the device is away now; `STICKY_UNREACH` is only read where there is no `UNREACH`,
+     * because a standing sticky flag next to `UNREACH: false` would count the same outage again on
+     * every sweep. Nothing is acknowledged from here: a sweep is a read, and HmIP has nothing to
+     * acknowledge (D-42).
+     */
     #applyHmipMaintenance(interfaceName, address, values) {
         const device = address.split(':')[0] ?? address;
         this.#caches.rssi(interfaceName).applyHmipParamset(device, values);
-        return this.#caches.serviceMessages.applyParamset(interfaceName, address, values);
+        const changed = this.#caches.serviceMessages.applyParamset(interfaceName, address, values);
+        const datapoint = 'UNREACH' in values ? 'UNREACH' : 'STICKY_UNREACH';
+        const value = values[datapoint];
+        if (value !== undefined) {
+            this.#noteUnreach(interfaceName, address, datapoint, value, { acknowledge: false });
+        }
+        return changed;
     }
     /**
      * Acknowledging a service message writes its datapoint. An `ACTION` is confirmed with `true`,
@@ -1048,7 +1528,9 @@ function asDescriptions(value) {
     const descriptions = [];
     for (const entry of value) {
         if (isStruct(entry) && typeof entry['ADDRESS'] === 'string') {
-            descriptions.push(entry);
+            // #143: the list fields of a description are made into lists here, once, for every
+            // interface - the group process sends `PARAMSETS` as a string on some boxes.
+            descriptions.push(normaliseDescription(entry));
         }
     }
     return descriptions;
@@ -1060,17 +1542,30 @@ function asLinks(value) {
     const links = [];
     for (const entry of value) {
         if (isStruct(entry) && typeof entry['SENDER'] === 'string' && typeof entry['RECEIVER'] === 'string') {
-            links.push(entry);
+            links.push(withLinkText(entry));
         }
     }
     return links;
 }
 function asLink(value, sender, receiver) {
     if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
-        return { SENDER: sender, RECEIVER: receiver, ...value };
+        return withLinkText({ SENDER: sender, RECEIVER: receiver, ...value });
     }
     return { SENDER: sender, RECEIVER: receiver };
 }
+/**
+ * B-23 (#156): a link's `NAME` and `DESCRIPTION` are UTF-8 in rfd - the WebUI's default description
+ * "Standardverknüpfung" and our own `setLinkInfo` alike - and the XML-RPC client reads them as
+ * ISO-8859-1. Repaired here, once, for both reads; a BIN-RPC answer (its library decodes UTF-8) and
+ * a real ISO-8859-1 text pass through unchanged.
+ */
+function withLinkText(link) {
+    return {
+        ...link,
+        ...(typeof link.NAME === 'string' ? { NAME: repairMisdecodedUtf8(link.NAME) } : {}),
+        ...(typeof link.DESCRIPTION === 'string' ? { DESCRIPTION: repairMisdecodedUtf8(link.DESCRIPTION) } : {}),
+    };
+}
 function asBidcosInterfaces(value) {
     if (!Array.isArray(value)) {
         return [];
@@ -1110,6 +1605,7 @@ export const API_METHOD_NAMES = [
     'config.set',
     'config.discover',
     'config.clearCaches',
+    'config.callbackAddresses',
     'interfaces.list',
     'interfaces.reconnect',
     'rega.state',
@@ -1128,6 +1624,21 @@ export const API_METHOD_NAMES = [
     'devices.replaceable',
     'names.get',
     'names.set',
+    'meta.state',
+    'meta.get',
+    'meta.enums',
+    'meta.objects',
+    'meta.setMembership',
+    'meta.assign',
+    'meta.enum.create',
+    'meta.enum.update',
+    'meta.enum.delete',
+    'meta.node.create',
+    'meta.node.update',
+    'meta.node.delete',
+    'meta.refresh',
+    'meta.export',
+    'meta.import',
     'paramset.get',
     'paramset.description',
     'paramset.put',
@@ -1153,14 +1664,15 @@ export const API_METHOD_NAMES = [
     'unreach.list',
     'unreach.reset',
     'serviceMessages.list',
+    'serviceMessages.refresh',
     'serviceMessages.ack',
     'events.recent',
     'events.clear',
     'rpc.call',
     'rpc.methods',
     'write.cancel',
-    'writeLog.list',
-    'writeLog.clear',
+    'rpcLog.list',
+    'rpcLog.clear',
     'data.file',
     'session.info',
 ];