iobroker.zigbee 3.0.5 → 3.1.4

package/main.js

@@ -13,8 +13,6 @@ try {
 }
 const originalLogMethod = debug.log;
 
-const zigbeeHerdsmanConvertersUtils = require('zigbee-herdsman-converters/lib/utils');
-
 const safeJsonStringify = require('./lib/json');
 const fs = require('fs');
 const path = require('path');
@@ -37,6 +35,9 @@ const vm = require('vm');
 const util = require('util');
 const dmZigbee  = require('./lib/devicemgmt.js');
 const DeviceDebug = require('./lib/DeviceDebug');
+const dns = require('dns');
+const net = require('net');
+const { getNetAddress } = require('./lib/utils')
 
 const createByteArray = function (hexString) {
     const bytes = [];
@@ -68,6 +69,8 @@ class Zigbee extends utils.Adapter {
             name: 'zigbee',
             systemConfig: true,
         }));
+        this.zhversion = zigbeeHerdsmanPackage ? zigbeeHerdsmanPackage.version : 'unknown';
+        this.zhcversion = zigbeeHerdsmanConvertersPackage ? zigbeeHerdsmanConvertersPackage.version : 'unknown';
         this.on('ready', () => this.onReady());
         this.on('unload', callback => this.onUnload(callback));
         this.on('message', obj => this.onMessage(obj));
@@ -77,14 +80,13 @@ class Zigbee extends utils.Adapter {
 
         this.stController = new StatesController(this);
         this.stController.on('log', this.onLog.bind(this));
-        this.stController.on('changed', this.publishFromState.bind(this));
+        this.stController.on('acknowledge_state', this.acknowledgeState.bind(this));
 
         this.deviceManagement = new dmZigbee(this);
         this.deviceDebug =  new DeviceDebug(this),
         this.deviceDebug.on('log', this.onLog.bind(this));
         this.debugActive = true;
 
-
         this.plugins = [
             new SerialListPlugin(this),
             new CommandsPlugin(this),
@@ -202,7 +204,6 @@ class Zigbee extends utils.Adapter {
             debug.log = this.debugLog.bind(this);
             debug.enable('zigbee-herdsman*');
         }
-
         // external converters
         this.applyExternalConverters();
         // get devices from exposes
@@ -214,6 +215,9 @@ class Zigbee extends utils.Adapter {
         this.setState('info.connection', false, true);
         const zigbeeOptions = this.getZigbeeOptions();
         this.zbController = new ZigbeeController(this);
+
+
+
         this.zbController.on('log', this.onLog.bind(this));
         this.zbController.on('ready', this.onZigbeeAdapterReady.bind(this));
         this.zbController.on('disconnect', this.onZigbeeAdapterDisconnected.bind(this));
@@ -222,7 +226,12 @@ class Zigbee extends utils.Adapter {
         this.zbController.on('pairing', this.onPairing.bind(this));
         this.zbController.on('event', this.stController.onZigbeeEvent.bind(this.stController));
         this.zbController.on('msg', this.stController.onZigbeeEvent.bind(this.stController));
-        this.zbController.on('publish', this.publishToState.bind(this));
+        this.zbController.on('publish', this.stController.publishToState.bind(this.stController));
+        this.stController.on('send_payload', this.zbController.publishPayload.bind(this.zbController));
+        this.stController.on('changed', this.zbController.publishFromState.bind(this.zbController));
+        this.stController.on('device_query', this.zbController.deviceQuery.bind(this.zbController));
+        this.zbController.on('acknowledge_state', this.acknowledgeState.bind(this));
+
         this.zbController.configure(zigbeeOptions);
         this.zbController.debugActive = this.debugActive;
         this.stController.debugActive = this.debugActive;
@@ -248,64 +257,57 @@ class Zigbee extends utils.Adapter {
 
     sandboxAdd(sandbox, item, module) {
         const multipleItems = item.split(',');
-        if (multipleItems.length > 1) {
-            for(const singleItem of multipleItems) {
-                this.log.warn(`trying to add "${singleItem.trim()} = require(${module})[${singleItem.trim()}]" to sandbox`)
-                sandbox[singleItem.trim()] = require(module)[singleItem.trim()];
+        for(const singleItem of multipleItems) {
+            const message = `Adding code from '${module}' as '${singleItem.trim()}' to sandbox `;
+            if (!module.match(new RegExp(`/${sandbox.zhclibBase}/`)))
+                module = module.replace(/zigbee-herdsman-converters\//, `${sandbox.zhclibBase}/`);
+            try {
+                const m = require(module);
+                sandbox[singleItem.trim()] = m;
+                this.log.info(`${message} -- success`);
+            }
+            catch (error) {
+                this.log.warn(`${message} -- failed: ${error && error.message ? error.message : 'no reason given'}`);
             }
-        }
-        else {
-            this.log.warn(`trying to add "${item} = require(${module})" to sandbox`)
-            sandbox[item] = require(module);
         }
     }
 
     SandboxRequire(sandbox, items) {
         if (!items) return true;
-        let converterLoaded = true;
+        //let converterLoaded = true;
         for (const item of items) {
             const modulePath = item[2].replace(/['"]/gm, '');
 
-            let zhcm1 = modulePath.match(/^zigbee-herdsman-converters\//);
-            if (zhcm1) {
-                try {
-                    const i2 = modulePath.replace(/^zigbee-herdsman-converters\//, `../${sandbox.zhclibBase}/`);
-                    this.sandboxAdd(sandbox, item[1], i2);
-                }
-                catch (error) {
-                    this.log.error(`Sandbox error: ${(error && error.message ? error.message : 'no error message given')}`)
-                }
-                continue;
-            }
-            zhcm1 = modulePath.match(/^..\//);
-            if (zhcm1) {
-                const i2 = modulePath.replace(/^..\//, `../${sandbox.zhclibBase}/`);
-                try {
-                    this.sandboxAdd(sandbox, item[1], i2);
-                }
-                catch (error) {
-                    this.log.error(`Sandbox error: ${(error && error.message ? error.message : 'no error message given')}`);
-                    converterLoaded = false;
-                }
+            const ZHCComponentMatch = modulePath.match(/\/(lib|converters|devices)\/(.+)/)
+
+            if (ZHCComponentMatch) {
+                const fullModulePath = '.' + path.sep + path.join('.',sandbox.zhclibBase, ZHCComponentMatch[1], ZHCComponentMatch[2]);
+                this.sandboxAdd(sandbox, item[1], fullModulePath);
                 continue;
             }
-            try {
-                this.sandboxAdd(sandbox, item[1], modulePath);
-            }
-            catch (error) {
-                this.log.error(`Sandbox error: ${(error && error.message ? error.message : 'no error message given')}`);
-                converterLoaded = false;
-            }
+            this.sandboxAdd(sandbox, item[1], modulePath);
 
         }
-        return converterLoaded;
+        return true;
     }
 
+    checkExternalConverterExists(fn) {
+        if (fs.existsSync(fn)) return fn;
+        const fnD = this.expandFileName(fn)
+        if (fs.existsSync(fnD)) return fnD;
+        const fnL = path.join('converters', fn)
+        if (fs.existsSync(fnL)) return fnL;
+        this.log.error(`unable to load ${fn} - checked ${path.resolve(fn)}, ${path.resolve(fnD)} and ${path.resolve(fnL)}`);
+        return false;
+    }
 
     * getExternalDefinition() {
+
         if (this.config.external === undefined) {
             return;
         }
+        const zhcPackageFn = require.resolve('zigbee-herdsman-converters/package.json');
+        const zhcBaseDir = path.relative('.',path.dirname(zhcPackageFn));
         const extfiles = this.config.external.split(';');
         for (const moduleName of extfiles) {
             if (!moduleName) continue;
@@ -313,48 +315,43 @@ class Zigbee extends utils.Adapter {
             const sandbox = {
                 require,
                 module: {},
-                zhclibBase : path.join('zigbee-herdsman-converters',(ZHCP && ZHCP.exports && ZHCP.exports['.'] ? path.dirname(ZHCP.exports['.']) : ''))
+                zhclibBase : path.join(zhcBaseDir,(ZHCP && ZHCP.exports && ZHCP.exports['.'] ? path.dirname(ZHCP.exports['.']) : ''))
             };
 
-            const mN = (fs.existsSync(moduleName) ? moduleName : this.expandFileName(moduleName));
-            if (!fs.existsSync(mN)) {
-                this.log.warn(`External converter not loaded - neither ${moduleName} nor ${mN} exist.`);
-            }
-            else {
+            const mN = this.checkExternalConverterExists(moduleName.trim());
+
+            if (mN) {
                 const converterCode = fs.readFileSync(mN, {encoding: 'utf8'}).toString();
                 let converterLoaded = true;
                 let modifiedCode = converterCode.replace(/\s+\/\/.+/gm, ''); // remove all lines starting with // (with the exception of the first.)
-                //fs.writeFileSync(mN+'.tmp1', modifiedCode)
                 modifiedCode = modifiedCode.replace(/^\/\/.+/gm, ''); // remove the fist line if it starts with //
-                //fs.writeFileSync(mN+'.tmp2', modifiedCode)
 
                 converterLoaded &= this.SandboxRequire(sandbox,[...modifiedCode.matchAll(/import\s+\*\s+as\s+(\S+)\s+from\s+(\S+);/gm)]);
                 modifiedCode = modifiedCode.replace(/import\s+\*\s+as\s+\S+\s+from\s+\S+;/gm, '')
-                //fs.writeFileSync(mN+'.tmp3', modifiedCode)
+
                 converterLoaded &= this.SandboxRequire(sandbox,[...modifiedCode.matchAll(/import\s+\{(.+)\}\s+from\s+(\S+);/gm)]);
                 modifiedCode = modifiedCode.replace(/import\s+\{.+\}\s+from\s+\S+;/gm, '');
 
                 converterLoaded &= this.SandboxRequire(sandbox,[...modifiedCode.matchAll(/import\s+(.+)\s+from\s+(\S+);/gm)]);
                 modifiedCode = modifiedCode.replace(/import\s+.+\s+from\s+\S+;/gm, '');
-                //fs.writeFileSync(mN+'.tmp4', modifiedCode)
+
                 converterLoaded &= this.SandboxRequire(sandbox,[...modifiedCode.matchAll(/const\s+\{(.+)\}\s+=\s+require\((.+)\)/gm)]);
                 modifiedCode = modifiedCode.replace(/const\s+\{.+\}\s+=\s+require\(.+\)/gm, '');
-                //fs.writeFileSync(mN+'.tmp5', modifiedCode)
+
                 converterLoaded &= this.SandboxRequire(sandbox,[...modifiedCode.matchAll(/const\s+(\S+)\s+=\s+require\((.+)\)/gm)]);
                 modifiedCode = modifiedCode.replace(/const\s+\S+\s+=\s+require\(.+\)/gm, '');
-                //mfs.writeFileSync(mN+'.tmp', modifiedCode)
 
                 for(const component of modifiedCode.matchAll(/const (.+):(.+)=/gm)) {
                     modifiedCode = modifiedCode.replace(component[0], `const ${component[1]} = `);
                 }
-                modifiedCode = modifiedCode.replace(/export .+;/gm, '');
+                modifiedCode = modifiedCode.replace(/export .+ +/gm, 'module.exports = ');
 
                 if (modifiedCode.indexOf('module.exports') < 0) {
                     converterLoaded = false;
                     this.log.error(`converter does not export any converter array, please add 'module.exports' statement to ${mN}`);
                 }
 
-                fs.writeFileSync(mN+'.tmp', modifiedCode)
+                //fs.writeFileSync(mN+'.tmp', modifiedCode)
 
                 if (converterLoaded) {
                     try {
@@ -364,6 +361,10 @@ class Zigbee extends utils.Adapter {
 
                         if (Array.isArray(converter)) for (const item of converter) {
                             this.log.info('Model ' + item.model + ' defined in external converter ' + mN);
+                            if (item.hasOwnProperty('icon')) {
+                                if (!item.icon.toLowerCase().startsWith('http') && !item.useadaptericon)
+                                    item.icon = path.join(path.dirname(mN), item.icon);
+                            }
                             yield item;
                         }
                         else {
@@ -412,10 +413,11 @@ class Zigbee extends utils.Adapter {
                 }
                 this.zbController.configure(this.getZigbeeOptions(message.zigbeeOptions));
                 response.status = await this.doConnect(true);
+                if (!response.status) response.error = { message: 'Unable to start the Zigbee Network. Please check the previous messages.'}
                 this.sendTo(from, command, response, callback);
             }
             catch (error) {
-                this.sendTo(from, command, { status:false }, callback);
+                this.sendTo(from, command, { status:false, error }, callback);
             }
         }
         else try {
@@ -423,7 +425,7 @@ class Zigbee extends utils.Adapter {
             //this.logToPairing('herdsman stopped !');
             this.sendTo(from, command, { status:true }, callback);
         } catch (error) {
-            this.sendTo(from, command, { status:false }, callback);
+            this.sendTo(from, command, { status:true, error }, callback);
         }
     }
 
@@ -442,23 +444,16 @@ class Zigbee extends utils.Adapter {
             if (noReconnect) this.logToPairing(`Starting Adapter ${debugversion}`);
             this.log.info(`Starting Adapter ${debugversion}`);
 
-            this.getForeignObject(`system.adapter.${this.namespace}`,async (err, obj) => {
-                try {
-                    if (!err && obj && obj.common.installedFrom && obj.common.installedFrom.includes('://')) {
-                        const instFrom = obj.common.installedFrom;
-                        gitVers = gitVers + instFrom.replace('tarball', 'commit');
-                    } else {
-                        gitVers = obj.common.installedFrom;
-                    }
-                    if (noReconnect) this.logToPairing(`Installed Version: ${gitVers} (Converters ${zigbeeHerdsmanConvertersPackage.version} Herdsman ${zigbeeHerdsmanPackage.version})`);
-                    this.log.info(`Installed Version: ${gitVers} (Converters ${zigbeeHerdsmanConvertersPackage.version} Herdsman ${zigbeeHerdsmanPackage.version})`);
-                    await this.zbController.start(noReconnect);
-                } catch (error) {
-                    this.logToPairing(error && error.message ? error.message : error);
-                    this.log.error(error && error.message ? error.message : error);
-                }
-                return false;
-            });
+            const obj = await this.getForeignObjectAsync(`system.adapter.${this.namespace}`);
+            if (!obj && obj.common.installedFrom && obj.common.installedFrom.includes('://')) {
+                const instFrom = obj.common.installedFrom;
+                gitVers = gitVers + instFrom.replace('tarball', 'commit');
+            } else {
+                gitVers = obj.common.installedFrom;
+            }
+            if (noReconnect) this.logToPairing(`Installed Version: ${gitVers} (Converters ${zigbeeHerdsmanConvertersPackage.version} Herdsman ${zigbeeHerdsmanPackage.version})`);
+            this.log.info(`Installed Version: ${gitVers} (Converters ${zigbeeHerdsmanConvertersPackage.version} Herdsman ${zigbeeHerdsmanPackage.version})`);
+            const result = await this.zbController.start(noReconnect);
         } catch (error) {
             this.setState('info.connection', false, true);
             this.logToPairing(`Failed to start Zigbee: ${error && error.message ? error.message : 'no message given'}`)
@@ -493,8 +488,14 @@ class Zigbee extends utils.Adapter {
         this.tryToReconnect();
     }
 
-    tryToReconnect() {
-        this.reconnectTimer = setTimeout(() => {
+    async tryToReconnect() {
+        this.reconnectTimer = setTimeout(async () => {
+            const result = await this.testConnection(this.config.port)
+            if (result.error) {
+                delete this.herdsman;
+                this.tryToReconnect();
+                return;
+            }
             if (this.config.port.includes('tcp://')) {
                 // Controller connect though Wi-Fi.
                 // Unlikely USB dongle, connection broken may only cause user unplugged the dongle,
@@ -510,6 +511,65 @@ class Zigbee extends utils.Adapter {
         }, 10 * 1000); // every 10 seconds
     }
 
+
+    async testConnection(address, interactive) {
+
+        function InteractivePairingMessage(msg, t) {
+            if (interactive) t.logToPairing(msg);
+            t.log.debug(msg);
+        }
+
+        this.log.debug(`Test connection for ${address}`);
+        const strMsg = '';
+
+        if (address) {
+            const netAddress = getNetAddress(address);
+            if (netAddress && netAddress.host) {
+                const netConnectPromise = new Promise((resolve) => {
+                    InteractivePairingMessage(`attempting dns lookup for ${netAddress.host}`, this);
+                    dns.lookup(netAddress.host, (err, ip, _) => {
+                        if (err) {
+                            resolve({error:`Unable to resolve name: ${err && err.message ? err.message : 'no message'}`});
+                        }
+                        InteractivePairingMessage(`dns lookup for ${address} produced ${ip}`, this );
+                        const client = new net.Socket();
+                        InteractivePairingMessage(`attempting to connect to ${ip} port ${netAddress.port ? netAddress.port : 80}`, this);
+                        client.connect(netAddress.port, ip, () => {
+                            client.destroy()
+                            InteractivePairingMessage(`connected successfully to connect to ${ip} port ${netAddress.port ? netAddress.port : 80}`, this);
+                            resolve({});
+                        })
+                        client.on('error', (error) => {
+                            resolve({error:`unable to connect to ${ip} port ${netAddress.port ? netAddress.port : 80} : ${error && error.message ? error.message : 'no message given'}`});
+                        });
+                    })
+                });
+                return await netConnectPromise;
+            }
+            else
+            {
+                const serialConnectPromise = new Promise((resolve) => {
+                    try {
+                        const port =address.trim();
+                        InteractivePairingMessage(`reading access rights for ${port}`, this);
+                        fs.access(port, fs.constants.R_OK | fs.constants.W_OK, (error) => {
+                            if (error) {
+                                resolve({error:`unable to access ${port} : ${error && error.message ? error.message : 'no message given'}`});
+                            }
+                            InteractivePairingMessage(`read and write access available for ${port}`, this);
+                            resolve({});
+                        });
+                    }
+                    catch (error) {
+                        resolve({error:`File access error: ${error && error.message ? error.message : 'no message given'}`});
+                    }
+                });
+                return await serialConnectPromise;
+            }
+        }
+        return {error: `missing parameter: address`};
+    }
+
     async onZigbeeAdapterReady() {
         this.reconnectTimer && clearTimeout(this.reconnectTimer);
         this.log.info(`Zigbee started`);
@@ -568,54 +628,43 @@ class Zigbee extends utils.Adapter {
 
         await this.setState('info.connection', true, true);
         this.stController.CleanupRequired(false);
+        const devicesFromObjects = (await this.getDevicesAsync()).filter(item => item.native.id.length ==16).map((item) => `0x${item.native.id}`);
         const devicesFromDB = this.zbController.getClientIterator(false);
         for (const device of devicesFromDB) {
             const entity = await this.zbController.resolveEntity(device);
             if (entity) {
                 const model = entity.mapped ? entity.mapped.model : entity.device.modelID;
+                const idx = devicesFromObjects.indexOf(device.ieeeAddr);
+                if (idx > -1) devicesFromObjects.splice(idx, 1);
                 this.stController.updateDev(device.ieeeAddr.substr(2), model, model, () =>
                     this.stController.syncDevStates(device, model));
             }
             else (this.log.warn('resolveEntity returned no entity'));
         }
+        for (const id of devicesFromObjects) {
+            try {
+                this.log.warn(`removing object for device ${id} - it is no longer in the zigbee database`);
+                await this.delObjectAsync(id.substring(2), { recursive:true })
+            }
+            catch {
+                this.log.warn(`error removing ${id}`)
+            }
+        }
         await this.callPluginMethod('start', [this.zbController, this.stController]);
     }
 
+
     async checkIfModelUpdate(entity) {
         const model = entity.mapped ? entity.mapped.model : entity.device.modelID;
         const device = entity.device;
         const devId = device.ieeeAddr.substr(2);
 
-        return new Promise((resolve) => {
-            this.getObject(devId, (err, obj) => {
-                if (obj && obj.common.type !== model) {
-                    // let's change model
-                    this.getStatesOf(devId, (err, states) => {
-                        if (!err && states) {
-                            const chain = [];
-                            states.forEach((state) =>
-                                chain.push(this.deleteStateAsync(devId, null, state._id)));
-
-                            Promise.all(chain)
-                                .then(() =>
-                                    this.stController.deleteObj(devId, () =>
-                                        this.stController.updateDev(devId, model, model, async () => {
-                                            await this.stController.syncDevStates(device, model);
-                                            resolve();
-                                        })));
-                        } else {
-                            resolve();
-                        }
-                    });
-                } else {
-                    resolve();
-                }
-            });
-        });
-    }
-
-    publishToState(devId, model, payload) {
-        this.stController.publishToState(devId, model, payload);
+        const obj = await this.getObjectAsync(devId);
+        if (obj && obj.common.type !== model) {
+            await this.stController.deleteObj(devId);
+            await this.stController.updateDev(devId, model, model);
+            await this.stController.syncDevStates(device, model);
+        }
     }
 
     acknowledgeState(deviceId, model, stateDesc, value) {
@@ -628,402 +677,14 @@ class Zigbee extends utils.Adapter {
         }
     }
 
-    processSyncStatesList(deviceId, model, syncStateList) {
-        syncStateList.forEach((syncState) => {
-            this.acknowledgeState(deviceId, model, syncState.stateDesc, syncState.value);
-        });
-    }
-
-    async publishFromState(deviceId, model, stateModel, stateList, options, debugID) {
-        let isGroup = false;
-        const has_elevated_debug = this.stController.checkDebugDevice(deviceId)
-
-        if (has_elevated_debug)
-        {
-            const stateNames = [];
-            for (const state of stateList) {
-                stateNames.push(state.stateDesc.id);
-            }
-            const message = `Publishing to ${deviceId} of model ${model} with ${stateNames.join(', ')}`;
-            this.emit('device_debug', { ID:debugID, data: { ID: deviceId, flag: '03', IO:false }, message: message});
-        }
-        else
-            if (this.debugActive) this.log.debug(`publishFromState : ${deviceId} ${model} ${safeJsonStringify(stateList)}`);
-        if (model === 'group') {
-            isGroup = true;
-            deviceId = parseInt(deviceId);
-        }
-        try {
-            const entity = await this.zbController.resolveEntity(deviceId);
-            if (this.debugActive) this.log.debug(`entity: ${deviceId} ${model} ${safeJsonStringify(entity)}`);
-            const mappedModel = entity ? entity.mapped : undefined;
-
-            if (!mappedModel) {
-                if (this.debugActive) this.log.debug(`No mapped model for ${model}`);
-                if (has_elevated_debug) {
-                    const message=`No mapped model ${deviceId} (model ${model})`;
-                    this.emit('device_debug', { ID:debugID, data: { error: 'NOMODEL' , IO:false }, message: message});
-                }
-                return;
-            }
-
-            if (!mappedModel.toZigbee)
-            {
-                this.log.error(`No toZigbee in mapped model for ${model}`);
-                return;
-            }
-
-            stateList.forEach(async changedState => {
-                const stateDesc = changedState.stateDesc;
-                const value = changedState.value;
-
-                if (stateDesc.id === 'send_payload') {
-                    try {
-                        const json_value = JSON.parse(value);
-                        const payload = {device: deviceId.replace('0x', ''), payload: json_value, model:model, stateModel:stateModel};
-                        if (has_elevated_debug) this.emit('device_debug', { ID:debugID, data: { flag: '04' ,payload:value ,states:[{id:stateDesc.id, value:json_value, payload:'none'}], IO:false }});
-
-                        const result = await this.sendPayload(payload);
-                        if (result.hasOwnProperty('success') && result.success) {
-                            this.acknowledgeState(deviceId, model, stateDesc, value);
-                        }
-                        else {
-                            this.log.error('Error in SendPayload: '+result.error.message);
-                        }
-                    } catch (error) {
-                        const message = `send_payload: ${value} does not parse as JSON Object : ${error.message}`;
-                        if (has_elevated_debug) this.emit('device_debug', { ID:debugID, data: { error: 'EXSEND' ,states:[{id:stateDesc.id, value:value, payload:error.message}], IO:false }, message:message});
-                        else this.log.error(message);
-                        return;
-                    }
-                    return;
-                }
-
-                if (stateDesc.isOption || stateDesc.compositeState) {
-                    // acknowledge state with given value
-                    if (has_elevated_debug) {
-                        const message = 'changed state: ' + JSON.stringify(changedState);
-                        this.emit('device_debug', { ID:debugID, data: { flag: 'cc', states:[{id:stateDesc.id, value:value, payload:'none (OC State)'}] , IO:false }, message:message});
-                    }
-                    else
-                        if (this.debugActive) this.log.debug('changed composite state: ' + JSON.stringify(changedState));
-
-                    this.acknowledgeState(deviceId, model, stateDesc, value);
-                    if (stateDesc.compositeState && stateDesc.compositeTimeout) {
-                        this.stController.triggerComposite(deviceId, model, stateDesc, changedState.source.includes('.admin.'));
-                    }
-                    // on activation of the 'device_query' state trigger hardware query where possible
-                    if (stateDesc.id === 'device_query') {
-                        if (this.query_device_block.indexOf(deviceId) > -1) {
-                            this.log.info(`Device query for '${entity.device.ieeeAddr}' blocked`);
-                            return;
-                        }
-                        if (mappedModel) {
-                            this.query_device_block.push(deviceId);
-                            if (has_elevated_debug) {
-                                const message  = `Device query for '${entity.device.ieeeAddr}/${entity.device.endpoints[0].ID}' triggered`;
-                                this.emit('device_debug', { ID:debugID, data: { flag: 'qs' ,states:[{id:stateDesc.id, value:value, payload:'none for device query'}], IO:false }, message:message});
-                            }
-                            else
-                                if (this.debugActive) this.log.debug(`Device query for '${entity.device.ieeeAddr}' started`);
-                            for (const converter of mappedModel.toZigbee) {
-                                if (converter.hasOwnProperty('convertGet')) {
-                                    for (const ckey of converter.key) {
-                                        try {
-                                            await converter.convertGet(entity.device.endpoints[0], ckey, {});
-                                        } catch (error) {
-                                            if (has_elevated_debug) {
-                                                const message = `Failed to read state '${JSON.stringify(ckey)}'of '${entity.device.ieeeAddr}/${entity.device.endpoints[0].ID}' from query with '${error && error.message ? error.message : 'no error message'}`;
-                                                this.log.warn(`ELEVATED OE02.1 ${message}`);
-                                                this.emit('device_debug', { ID:debugID, data: { error: 'NOTREAD' , IO:false }, message:message });
-                                            }
-                                            else
-                                                this.log.info(`failed to read state ${JSON.stringify(ckey)} of ${entity.device.ieeeAddr}/${entity.device.endpoints[0].ID} after device query`);
-                                        }
-                                    }
-                                }
-                            }
-                            if (has_elevated_debug) {
-                                const message = `ELEVATED O07: Device query for '${entity.device.ieeeAddr}/${entity.device.endpoints[0].ID}' complete`;
-                                this.emit('device_debug', { ID:debugID, data: { flag: 'qe' , IO:false }, message:message});
-                            }
-                            else
-                                this.log.info(`Device query for '${entity.device.ieeeAddr}' done`);
-                            const idToRemove = deviceId;
-                            setTimeout(() => {
-                                const idx = this.query_device_block.indexOf(idToRemove);
-                                if (idx > -1) {
-                                    this.query_device_block.splice(idx);
-                                }
-                            }, 10000);
-                        }
-                        return;
-                    }
-                    return;
-                }
-
-                let converter = undefined;
-                let msg_counter = 0;
-                for (const c of mappedModel.toZigbee) {
-
-                    if (!c.hasOwnProperty('convertSet')) continue;
-                    if (this.debugActive) this.log.debug(`Type of toZigbee is '${typeof c}', Contains key ${(c.hasOwnProperty('key')?JSON.stringify(c.key):'false ')}`)
-                    if (!c.hasOwnProperty('key'))
-                    {
-                        if (converter === undefined)
-                        {
-                            converter = c;
-                            if (has_elevated_debug) {
-                                const message = `Setting converter to keyless converter for ${deviceId} of type ${model}`;
-                                this.emit('device_debug', { ID:debugID, data: { flag: `s4.${msg_counter}` , IO:false }, message:message});
-                            }
-                            else
-                                if (this.debugActive) this.log.debug(`Setting converter to keyless converter for ${deviceId} of type ${model}`);
-                            msg_counter++;
-                        }
-                        else
-                        {
-                            if (has_elevated_debug)
-                            {
-                                const message = `ignoring keyless converter for ${deviceId} of type ${model}`;
-                                this.emit('device_debug', { ID:debugID, data: { flag: `i4.${msg_counter}` , IO:false} , message:message});
-                            }
-                            else
-                                if (this.debugActive) this.log.debug(`ignoring keyless converter for ${deviceId} of type ${model}`);
-                            msg_counter++;
-                        }
-                        continue;
-                    }
-                    if (c.key.includes(stateDesc.prop) || c.key.includes(stateDesc.setattr) || c.key.includes(stateDesc.id))
-                    {
-                        const message = `${(converter===undefined?'Setting':'Overriding')}' converter to converter with key(s)'${JSON.stringify(c.key)}}`;
-                        if (has_elevated_debug) {
-                            this.emit('device_debugug', { ID:debugID, data: { flag: `${converter===undefined ? 's' : 'o'}4.${msg_counter}` , IO:false }, message:message});
-
-                        }
-                        else
-                            if (this.debugActive) this.log.debug(message);
-                        converter = c;
-                        msg_counter++;
-                    }
-                }
-                if (converter === undefined) {
-                    const message = `No converter available for '${model}' with key '${stateDesc.id}' `;
-                    if (has_elevated_debug) {
-                        this.emit('device_debug', { ID:debugID, data: { error: 'NOCONV',states:[{id:stateDesc.id, value:value, payload:'no converter'}] , IO:false }, message:message});
-                    }
-                    else {
-                        this.log.info(message);
-                    }
-                    return;
-                }
-
-                const preparedValue = (stateDesc.setter) ? stateDesc.setter(value, options) : value;
-                const preparedOptions = (stateDesc.setterOpt) ? stateDesc.setterOpt(value, options) : {};
-
-                let syncStateList = [];
-                if (stateModel && stateModel.syncStates) {
-                    stateModel.syncStates.forEach(syncFunct => {
-                        const res = syncFunct(stateDesc, value, options);
-                        if (res) {
-                            syncStateList = syncStateList.concat(res);
-                        }
-                    });
-                }
-
-                const epName = stateDesc.epname !== undefined ? stateDesc.epname : (stateDesc.prop || stateDesc.id);
-                const key = stateDesc.setattr || stateDesc.prop || stateDesc.id;
-                const message = `convert ${key}, ${safeJsonStringify(preparedValue)}, ${safeJsonStringify(preparedOptions)} for device ${deviceId} with Endpoint ${epName}`;
-                if (has_elevated_debug) {
-                    this.emit('device_debug', { ID:debugID, data: { flag: '04', payload: {key:key, ep: stateDesc.epname, value:preparedValue, options:preparedOptions}, IO:false }, message:message});
-                }
-                else
-                    if (this.debugActive) this.log.debug(message);
-
-                let target;
-                if (model === 'group') {
-                    target = entity.mapped;
-                } else {
-                    target = await this.zbController.resolveEntity(deviceId, epName);
-                    target = target.endpoint;
-                }
-
-                if (this.debugActive) this.log.debug(`target: ${safeJsonStringify(target)}`);
-
-                const meta = {
-                    endpoint_name: epName,
-                    options: preparedOptions,
-                    device: entity.device,
-                    mapped: model === 'group' ? [] : mappedModel,
-                    message: {[key]: preparedValue},
-                    logger: this.log,
-                    state: {},
-                };
-
-                // new toZigbee
-                if (preparedValue !== undefined && Object.keys(meta.message).filter(p => p.startsWith('state')).length > 0) {
-                    if (typeof preparedValue === 'number') {
-                        meta.message.state = preparedValue > 0 ? 'ON' : 'OFF';
-                    } else {
-                        meta.message.state = preparedValue;
-                    }
-                }
-                if (has_elevated_debug) {
-                    this.emit('device_debug', { ID:debugID, data: { states:[{id:stateDesc.id, value:value, payload:preparedValue, ep:stateDesc.epname}] , IO:false }});
-                }
-
-                if (preparedOptions !== undefined) {
-                    if (preparedOptions.hasOwnProperty('state')) {
-                        meta.state = preparedOptions.state;
-                    }
-                }
-
-                try {
-                    const result = await converter.convertSet(target, key, preparedValue, meta);
-                    const message = `convert result ${safeJsonStringify(result)} for device ${deviceId}`;
-                    if (has_elevated_debug) {
-                        this.emit('device_debug', { ID:debugID, data: { flag: 'SUCCESS' , IO:false }, message:message});
-                    }
-                    else
-                        if (this.debugActive) this.log.debug(message);
-                    if (result !== undefined) {
-                        if (stateModel && !isGroup && !stateDesc.noack) {
-                            this.acknowledgeState(deviceId, model, stateDesc, value);
-                        }
-                        // process sync state list
-                        this.processSyncStatesList(deviceId, model, syncStateList);
-                    }
-                    else {
-                        if (has_elevated_debug) {
-                            const message = `Convert does not return a result result for ${key} with ${safeJsonStringify(preparedValue)} on device ${deviceId}.`;
-                            this.emit('device_debug', { ID:debugID, data: { flag: '06' , IO:false }, message:message});
-                        }
-                    }
-                } catch (error) {
-                    if (has_elevated_debug) {
-                        const message = `caught error ${safeJsonStringify(error)} when setting value for device ${deviceId}.`;
-                        this.emit('device_debug', { ID:debugID, data: { error: 'EXSET' , IO:false },message:message});
-                    }
-                    this.filterError(`Error ${error.code} on send command to ${deviceId}.` +
-                        ` Error: ${error.stack}`, `Send command to ${deviceId} failed with`, error);
-                }
-            });
-        } catch (err) {
-            const message = `No entity for ${deviceId} : ${err && err.message ? err.message : 'no error message'}`;
-            this.emit('device_debug', { ID:debugID, data: { error: 'EXPUB' , IO:false }, message:message});
-        }
-    }
-
-
-    extractEP(key, endpoints) {
-        try {
-            if (endpoints) for (const ep of Object.keys(endpoints)) {
-                if (key.endsWith('_'+ep)) return { setattr: key.replace('_'+ep, ''), epname:ep }
-            }
-        }
-        catch {
-            return {};
-        }
-        return {};
-    }
-    // This function is introduced to explicitly allow user level scripts to send Commands
-    // directly to the zigbee device. It utilizes the zigbee-herdsman-converters to generate
-    // the exact zigbee message to be sent and can be used to set device options which are
-    // not exposed as states. It serves as a wrapper function for "publishFromState" with
-    // extended parameter checking
-    //
-    // The payload can either be a JSON object or the string representation of a JSON object
-    // The following keys are supported in the object:
-    // device: name of the device. For a device zigbee.0.0011223344556677 this would be 0011223344556677
-    // payload: The data to send to the device as JSON object (key/Value pairs)
-    // endpoint: optional: the endpoint to send the data to, if supported.
-    //
     async sendPayload(payload) {
-        let payloadObj = {};
-        if (typeof payload === 'string') {
-            try {
-                payloadObj = JSON.parse(payload);
-            } catch (e) {
-                this.log.error(`Unable to parse ${safeJsonStringify(payload)}: ${safeJsonStringify(e)}`);
-                this.sendError(e, `Unable to parse ${safeJsonStringify(payload)}: ${safeJsonStringify(e)}`);
-                return {
-                    success: false,
-                    error: `Unable to parse ${safeJsonStringify(payload)}: ${safeJsonStringify(e)}`
-                };
-            }
-        } else if (typeof payload === 'object') {
-            payloadObj = payload;
-        } else return { success: false, error: 'illegal type of payload: ' + typeof payload};
-
-        if (payloadObj.hasOwnProperty('device') && payloadObj.hasOwnProperty('payload')) {
-            try {
-                const isDevice = !payload.device.includes('group_');
-                const stateList = [];
-                const devID = isDevice ? `0x${payload.device}` : parseInt(payload.device.replace('group_', ''));
-
-                const entity = await this.zbController.resolveEntity(devID);
-                if (!entity) {
-                    this.log.error(`Device ${safeJsonStringify(payloadObj.device)} not found`);
-                    this.sendError(`Device ${safeJsonStringify(payloadObj.device)} not found`);
-                    return {success: false, error: `Device ${safeJsonStringify(payloadObj.device)} not found`};
-                }
-                const mappedModel = entity.mapped;
-                if (!mappedModel) {
-                    this.log.error(`No Model for Device ${safeJsonStringify(payloadObj.device)}`);
-                    this.sendError(`No Model for Device ${safeJsonStringify(payloadObj.device)}`);
-                    return {success: false, error: `No Model for Device ${safeJsonStringify(payloadObj.device)}`};
-                }
-                if (typeof payloadObj.payload !== 'object') {
-                    this.log.error(`Illegal payload type for ${safeJsonStringify(payloadObj.device)}`);
-                    this.sendError(`Illegal payload type for ${safeJsonStringify(payloadObj.device)}`);
-                    return {success: false, error: `Illegal payload type for ${safeJsonStringify(payloadObj.device)}`};
-                }
-                const endpoints = mappedModel && mappedModel.endpoint ? mappedModel.endpoint(entity.device) : null;
-                for (const key in payloadObj.payload) {
-                    if (payloadObj.payload[key] != undefined) {
-                        const datatype = typeof payloadObj.payload[key];
-                        const epobj = this.extractEP(key, endpoints);
-                        if (payloadObj.endpoint) {
-                            epobj.epname = payloadObj.endpoint;
-                            delete epobj.setattr;
-                        }
-                        stateList.push({
-                            stateDesc: {
-                                id: key,
-                                prop: key,
-                                role: 'state',
-                                type: datatype,
-                                noack:true,
-                                epname: epobj.epname,
-                                setattr: epobj.setattr,
-                            },
-                            value: payloadObj.payload[key],
-                            index: 0,
-                            timeout: 0,
-                        });
-                    }
-                }
-                try {
-                    await this.publishFromState(`0x${payload.device}`, payload.model, payload.stateModel, stateList, payload.options, Date.now());
-                    return {success: true};
-                } catch (error) {
-                    this.log.error(`Error ${error.code} on send command to ${payload.device}.` + ` Error: ${error.stack} ` + `Send command to ${payload.device} failed with ` + error);
-                    this.filterError(`Error ${error.code} on send command to ${payload.device}.` + ` Error: ${error.stack}`, `Send command to ${payload.device} failed with`, error);
-                    return {success: false, error};
-                }
-            } catch (e) {
-                return {success: false, error: e};
-            }
-        }
-
-        return {success: false, error: `missing parameter device or payload in message ${JSON.stringify(payload)}`};
+        return await this.zbController.publishPayload(payload);
     }
 
 
     async newDevice(entity) {
 
         if (this.debugActive) this.log.debug(`New device event: ${safeJsonStringify(entity)}`);
-        //this.stController.AddModelFromHerdsman(entity.device, entity.mapped ? entity.mapped.model : entity.device.modelID)
 
         const dev = entity.device;
         const model = (entity.mapped) ? entity.mapped.model : dev.modelID;
@@ -1084,17 +745,19 @@ class Zigbee extends utils.Adapter {
     async onUnload(callback) {
         try {
             this.log.info(`Halting zigbee adapter. Restart delay is at least ${this.ioPack.common.stopTimeout / 1000} seconds.`)
+            this.setState('info.connection', false, true);
+            const chain = [];
             if (this.config.debugHerdsman) {
                 debug.disable();
                 debug.log = originalLogMethod;
             }
-
-            this.log.info('cleaning everything up...');
+            this.log.info('cleaning everything up');
             await this.callPluginMethod('stop');
-            await this.stController.stop();
+            if (this.stController) chain.push(this.stController.stop());
             if (this.zbController) {
-                await this.zbController.stop();
+                chain.push(this.zbController.stop());
             }
+            Promise.all(chain);
             this.log.info('cleanup successful');
             callback();
         } catch (error) {
@@ -1106,8 +769,8 @@ class Zigbee extends utils.Adapter {
         }
     }
 
-    getZigbeeOptions(_overrideOptions) {
-        const override = (_overrideOptions ? _overrideOptions:{});
+    getZigbeeOptions(overrideOptions) {
+        const override = (overrideOptions ? overrideOptions:{});
         // file path for db
         const dbDir = this.expandFileName('');
 
@@ -1153,7 +816,7 @@ class Zigbee extends utils.Adapter {
             dbPath: 'shepherd.db',
             backupPath: 'nvbackup.json',
             disableLed: this.config.disableLed,
-            disablePing: this.config.disablePing,
+            disablePing: (this.config.pingCluster=='off'),
             transmitPower: this.config.transmitPower,
             disableBackup: this.config.disableBackup,
             extPanIdFix: extPanIdFix,