iobroker.zigbee 3.0.5 → 3.1.4

package/lib/zigbeecontroller.js

@@ -11,7 +11,6 @@ const DeviceAvailabilityExt = require('./zbDeviceAvailability');
 const DeviceConfigureExt = require('./zbDeviceConfigure');
 const DeviceEventExt = require('./zbDeviceEvent');
 const DelayedActionExt = require('./zbDelayedAction');
-const Groups = require('./groups');
 const utils = require('./utils');
 
 const { access, constants } =require('node:fs/promises');
@@ -67,7 +66,7 @@ class ZigbeeController extends EventEmitter {
         this.transmitPower = 0;
         this.herdsmanStarted = false;
         this.extensions = [
-            new DeviceAvailabilityExt(this, {}),
+            new DeviceAvailabilityExt(this, {}, adapter.config),
             new DeviceConfigureExt(this, {}),
             new DeviceEventExt(this, {}),
             new DelayedActionExt(this, {}),
@@ -75,6 +74,10 @@ class ZigbeeController extends EventEmitter {
         this.herdsmanTimeoutRegexp = new RegExp(/(\d+)ms/);
         this.herdsmanLogSettings = {};
         this.debugActive = true;
+        this.ListDevicesAtStart = adapter.config.listDevicesAtStart;
+        this.deviceQueryActive = [];
+        this.storedOptions = undefined;
+        this.isConfigured = false;
 
     }
 
@@ -102,7 +105,7 @@ class ZigbeeController extends EventEmitter {
     }
 
     configure(options) {
-
+        this.storedOptions = options;
         if (options.transmitPower != undefined) {
             this.transmitPower = options.transmitPower;
         }
@@ -117,6 +120,8 @@ class ZigbeeController extends EventEmitter {
 
             this.powerText = powerLevels[this.transmitPower] || 'normal';
         }
+        this.readAtAnnounce = this.adapter.config.readAtAnnounce;
+        this.warnOnDeviceAnnouncement = this.adapter.config.warnOnDeviceAnnouncement;
 
         //this.info(`  --> transmitPower : ${this.powerText}`);
 
@@ -156,17 +161,18 @@ class ZigbeeController extends EventEmitter {
             herdsmanSettings.transmitPower = options.transmitPower;
         }
         this.disableLed = options.disableLed;
-        this.warnOnDeviceAnnouncement = options.warnOnDeviceAnnouncement;
         this.herdsmanLogSettings.panID = herdsmanSettings.network.panID;
         this.herdsmanLogSettings.channel = herdsmanSettings.network.channelList[0];
         this.herdsmanLogSettings.extendedPanID = utils.byteArrayToString(herdsmanSettings.network.extendedPanID);
         this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings, this.adapter.log);
         this.callExtensionMethod('setOptions', [{
-            disableActivePing: options.disablePing,
+            pingCluster: this.adapter.config.pingCluster,
+            startReadDelay: this.adapter.config.readAllAtStart ? this.adapter.config.startReadDelay : 0,
             disableForcedPing: false,
             pingTimeout: 300,
             pingCount: 3
         }]);
+        this.isConfigured = true;
     }
 
     async stopHerdsman() {
@@ -179,11 +185,15 @@ class ZigbeeController extends EventEmitter {
         catch (error) {
             this.emit('pairing', `error stopping zigbee-herdsman: ${error && error.message ? error.message : 'no reason given'}`);
         }
+        this.isConfigured = false;
+        delete this.herdsman;
+
     }
 
     // Start controller
     async start() {
         try {
+            if (!this.isConfigured) this.configure(this.storedOptions);
             this.emit('pairing',`Starting zigbee-herdsman...`);
             this.powerText = '';
             if (this.transmitPower !== '0') {
@@ -207,7 +217,7 @@ class ZigbeeController extends EventEmitter {
             this.herdsman.on('permitJoinChanged', this.handlePermitJoinChanged.bind(this));
 
             this.info('Starting Zigbee-Herdsman');
-            await this.herdsman.start();
+            const result = await this.herdsman.start();
             this.herdsmanStarted = true;
             const cv = await this.herdsman.getCoordinatorVersion();
             const MetaSt = `${cv.meta.transportrev ? cv.meta.transportrev : 'X'}-${cv.meta.product ? cv.meta.product : 'X'}.${cv.meta.majorrel ? cv.meta.majorrel : 'X'}.${cv.meta.minorrel ? cv.meta.minorrel : 'X'}.${cv.meta.maintrel ? cv.meta.maintrel : 'X'}`;
@@ -272,8 +282,7 @@ class ZigbeeController extends EventEmitter {
                 // get the model description for the known devices
                 const entity = await this.resolveEntity(device);
                 if (!entity) {
-                    this.warn('failed to resolve Entity for ' + device.ieeeAddr);
-                    //this.emit('pairing','failed to resolve Entity for ' + device.ieeeAddr)
+                    this.debug('failed to resolve Entity for ' + device.ieeeAddr);
                     continue;
                 }
                 //await this.adapter.stController.AddModelFromHerdsman(device, entity.mapped.model);
@@ -294,19 +303,20 @@ class ZigbeeController extends EventEmitter {
                     ` (addr ${entity.device.networkAddress}): ` +
                     (entity.mapped ? `${entity.mapped.model} - ${entity.mapped.vendor} ${entity.mapped.description} ` : `Unsupported (model ${entity.device.modelID})`) +
                     `(${entity.device.type})`);
-                //this.emit('pairing',msg);
-                this.info(msg);
+                if (this.ListDevicesAtStart) this.info(msg);
+            }
+
+            const Groups = await this.getGroups();
+            for (const group of Groups) {
+                if (this.ListDevicesAtStart) this.info(`${group.stateName} (addr ${group.id}): Group with ${group.size} members.`);
             }
 
             // Log zigbee clients on startup
             // const devices = await this.getClients();
-            if (deviceCount > 0) {
-                this.info(`Currently ${deviceCount} devices are joined:`);
-                this.emit('pairing',`Currently ${deviceCount} devices are joined:`)
-            } else {
-                this.info(`No devices are currently joined.`);
-                this.emit('pairing',`No devices are currently joined.`);
-            }
+            const gm = Groups.length > 0 ? `and ${Groups.length} group${Groups.length > 1?'s':''} ` : '';
+            const m = deviceCount > 0 ? `${deviceCount} devices ${gm}are part of the network` : `No devices ${gm}`;
+            this.info(m);
+            this.emit('pairing',m)
             this.callExtensionMethod('onZigbeeStarted', []);
         }
         catch (error) {
@@ -395,6 +405,15 @@ class ZigbeeController extends EventEmitter {
         }
     }
 
+    getGroupIterator() {
+        if (this.herdsman.database) {
+            this.herdsman.getGroupsIterator()
+        }
+        else {
+            return[].values();
+        }
+    }
+
     async getGroups() {
         try {
             if (this.herdsman) {
@@ -495,6 +514,7 @@ class ZigbeeController extends EventEmitter {
                             ieee: device.ieeeAddr,
                             model: device.modelID,
                             epid,
+                            deviceNetworkAddress: nwk,
                             ep: member
                         });
                     }
@@ -513,9 +533,10 @@ class ZigbeeController extends EventEmitter {
         return members;
     }
 
-    getDevice(key) {
+    async getDevice(key) {
         try {
-            return this.herdsman.getDeviceByIeeeAddr(key);
+            const dev = await this.herdsman.getDeviceByIeeeAddr(key);
+            return dev;
         }
         catch (error) {
             this.error(`getDeviceByIeeeAddr: ${(error && error.message ? error.message : 'no error message')}`);
@@ -549,7 +570,9 @@ class ZigbeeController extends EventEmitter {
             key: key,
             message: 'success'
         }
-        if (typeof key === 'object') return rv;
+        if (typeof key === 'object') {
+            return rv;
+        }
         if (typeof key === 'number') {
             rv.kind = 'group';
             return rv;
@@ -579,8 +602,11 @@ class ZigbeeController extends EventEmitter {
         return rv;
     }
 
+    async getGroup(id) {
+        return await this.herdsman.getGroupByID(id);
+    }
+
     async resolveEntity(key, ep) {
-        // this.warn('resolve entity with key of tyoe ' + typeof (key));
         try {
             const _key = await this.analyzeKey(key);
             if (_key.message !== 'success') return undefined;
@@ -592,6 +618,7 @@ class ZigbeeController extends EventEmitter {
                     device: coordinator,
                     endpoint: coordinator.getEndpoint(1),
                     name: 'Coordinator',
+                    options:{}
                 };
             }
             if (_key.kind === 'group') {
@@ -602,16 +629,39 @@ class ZigbeeController extends EventEmitter {
                 return {
                     type: 'group',
                     mapped: group,
-                    group,
+                    device: group,
+                    endpoint: group,
+                    //group,
                     name: `Group ${_key.key}`,
+                    options: {},
                 };
 
             }
-            if (_key.kind === 'ieee') _key.key = await this.herdsman.getDeviceByIeeeAddr(_key.key);
-            const device = _key.key;
+            //if (_key.kind === 'ieee')
+            const device = (_key.kind === 'ieee' ? this.herdsman.getDeviceByIeeeAddr(_key.key) : key);
+            if (device && device.model === 'group') {
+                return {
+                    type: 'group',
+                    mapped: device,
+                    device,
+                    endpoint: device,
+                    name: `Group ${device.groupID}`,
+                    options:{}
+                }
+            }
             if (device) {
                 const t = Date.now();
-                const mapped = await zigbeeHerdsmanConverters.findByDevice(device);
+                const mapped = await zigbeeHerdsmanConverters.findByDevice(device, false);
+                if (!mapped) {
+                    if (device.type === 'Coordinator')
+                        return {
+                            type: 'device',
+                            device: device,
+                            endpoint: device.getEndpoint(1),
+                            name: 'Coordinator',
+                        };
+                    this.warn(`Resolve Entity did not manage to find a mapped device for ${device.ieeeAddr} of type ${device.modelID}`);
+                }
                 const endpoints = mapped && mapped.endpoint ? mapped.endpoint(device) : null;
                 let endpoint;
                 if (endpoints && ep != undefined && endpoints[ep]) {
@@ -626,6 +676,7 @@ class ZigbeeController extends EventEmitter {
                         endpoint = device.endpoints[0];
                     }
                 }
+                const options = this.adapter.stController.localConfig.getOptions(device.ieeeAddr, mapped.model);
                 return {
                     type: 'device',
                     device,
@@ -633,8 +684,12 @@ class ZigbeeController extends EventEmitter {
                     endpoint,
                     endpoints: device.endpoints,
                     name: device._ieeeAddr,
+                    options:options
                 };
             }
+            else {
+                this.debug(`resolve_entity failed for ${JSON.stringify(_key)}`);
+            }
         }
         catch (error)
         {
@@ -676,10 +731,13 @@ class ZigbeeController extends EventEmitter {
         }
 
         try {
-            if (this.HerdsmanStarted) await this.permitJoin(0);
-            await this.herdsman.stop();
-            this.HerdsmanStarted = false;
-            this.info('zigbecontroller stopped successfully');
+            if (this.HerdsmanStarted) {
+                await this.permitJoin(0);
+                await this.herdsman.stop();
+                this.HerdsmanStarted = false;
+                this.info('zigbecontroller stopped successfully');
+            }
+            this.info('zigbecontroller stopped successfully - ZH was not running');
         } catch (error) {
             this.sendError(error);
             if (this.herdsmanStarted) {
@@ -700,14 +758,18 @@ class ZigbeeController extends EventEmitter {
     }
 
     // Permit join
-    async permitJoin(permitTime, devid, failure) {
+    async permitJoin(permitTime, devid) {
         try {
             this._permitJoinTime = permitTime;
             await this.herdsman.permitJoin(permitTime);
+            this._permitJoinDevice = (permitTime > 0 && devid ? devid : '');
         } catch (e) {
+            this._permitJoinTime = 0;
             this.sendError(e);
             this.error(`Failed to open the network: ${e.stack}`);
+            return false;
         }
+        return true;
     }
 
     async handlePermitJoinChanged(data)
@@ -730,6 +792,7 @@ class ZigbeeController extends EventEmitter {
                 this._permitJoinInterval = null;
                 // this.emit('pairing', 'Pairing time left', 0);
                 this.emit('pairing', 'Closing network.',0);
+                this._permitJoinDevice = '';
             }
         }
         catch (error) {
@@ -774,8 +837,16 @@ class ZigbeeController extends EventEmitter {
             }
         } catch (error) {
             this.sendError(error);
-            this.error(`Failed to remove ${error.stack}`);
-            callback && callback(`Failed to remove ${error.stack}`);
+            let message = 'no reason given';
+            if (error && error.message) {
+                if (error.message.includes('AREQ - ZDO - mgmtLeaveRsp after'))
+                    message = `No response to mgmtLeaveRequest from the device - device may be offline.`;
+                else
+                    message = error.message;
+
+            }
+            this.warn(`Failed to remove ${deviceID ? 'device ' + deviceID : 'unspecified device'}: ${message}`);
+            callback && callback(`Failed to remove ${deviceID ? 'device ' + deviceID : 'unspecified device'}: ${message}`);
         }
     }
 
@@ -806,41 +877,52 @@ class ZigbeeController extends EventEmitter {
         if (this.debugActive) this.debug('handleDeviceAnnounce', message);
         const entity = await this.resolveEntity(message.device || message.ieeeAddr);
         const friendlyName = entity.name;
+
+
+        this.emit('pairing', `Device '${friendlyName}' announced itself`);
         if (this.adapter.stController.checkDebugDevice(friendlyName)) {
             this.emit('device_debug', {ID: Date.now(), data: {flag:'da', states:[{id: '--', value:'--', payload:message}] , IO:true} ,message:`Device '${friendlyName}' announced itself`});
         }
-        else if (this.warnOnDeviceAnnouncement) {
-            this.warn(`Device '${friendlyName}' announced itself`);
+        if (this.warnOnDeviceAnnouncement) {
+            this.warn(`Device '${friendlyName}' announced itself${this.readAtAnnounce ? ', trying to read its status' : ''}`);
         } else {
-            this.info(`Device '${friendlyName}' announced itself`);
+            this.info(`Device '${friendlyName}' announced itself${this.readAtAnnounce ? ', trying to read its status' : ''}`);
+        }
+
+        if (entity.device && entity.device.modelID && entity.device.interviewState != 'SUCCESSFUL') {
+            this.info(`ignoring device announcement for ${entity.device.modelID} due to interview state ${entity.device.interviewState}`);
+            this.emit('pairing', `device interview state is ${entity.device.interviewState}`)
+            return;
         }
 
+        const networkOpen = this.herdsman.getPermitJoin();
+        /*
+        if (networkOpen && entity.device && entity.device.modelID && entity.device.interviewState != 'IN_PROGRESS')
+        {
+            //entity.device.modelID = entity.device._modelID;
+            //this.emit('new', entity);
+            return;
+        }
+        */
         try {
             if (entity && entity.mapped) {
                 this.callExtensionMethod(
                     'onZigbeeEvent',
-                    [{'device': message.device, 'type': 'deviceAnnounce'}, entity ? entity.mapped : null]);
+                    [{'device': message.device, 'type': 'deviceAnnounce', options: entity.options || {}}, entity ? entity.mapped : null]);
+                this.callExtensionMethod('registerDevicePing', [message.device, entity]);
+                if (this.readAtAnnounce) await this.doDeviceQuery(message.device || message.ieeeAddr, Date.now(), false);
             }
         } catch (error) {
             this.sendError(error);
-            this.error(`Failed to handleDeviceLeave ${error.stack}`);
-        }
-
-        this.emit('pairing', `Device '${friendlyName}' announced itself`);
-        if (!this.herdsman.getPermitJoin()) {
-            this.callExtensionMethod('registerDevicePing', [message.device, entity]);
-        }
-        // if has modelID so can create device
-        if (entity.device && entity.device._modelID) {
-            entity.device.modelID = entity.device._modelID;
-            this.emit('new', entity);
+            this.error(`Failed to handleDeviceAnnounce ${error.stack}`);
         }
     }
 
     async handleDeviceJoined(message) {
         if (this.debugActive) this.debug('handleDeviceJoined', message);
         //const entity = await this.resolveEntity(message.device || message.ieeeAddr);
-        //this.emit('new', entity);
+        // this.emit('new', entity);
+        //if (entity && entity.mapped) this.callExtensionMethod([message, entity.mapped]);
     }
 
     async handleDeviceInterview(message) {
@@ -864,13 +946,13 @@ class ZigbeeController extends EventEmitter {
 
                     const log = {friendly_name: friendlyName, model, vendor, description, supported: true};
                     this.emit('pairing', 'Interview successful', JSON.stringify(log));
-                    entity.device.modelID = entity.device._modelID;
-                    this.emit('new', entity);
-                    // send to extensions again (for configure)
+                    //entity.device.modelID = entity.device._modelID;
                     this.callExtensionMethod(
                         'onZigbeeEvent',
-                        [message, entity ? entity.mapped : null],
+                        [{...message,type:'deviceInterview', options: entity.options || {}}, entity.mapped],
                     );
+                    this.emit('new', entity);
+                    // send to extensions again (for configure)
                 } else {
                     if (this.debugActive) this.debug(
                         `Device '${friendlyName}' with Zigbee model '${message.device.modelID}' is NOT supported, ` +
@@ -878,15 +960,23 @@ class ZigbeeController extends EventEmitter {
                     );
                     const frName = {friendly_name: friendlyName, supported: false};
                     this.emit('pairing', 'Interview successful', JSON.stringify(frName));
-                    entity.device.modelID = entity.device._modelID;
+                    //entity.device.modelID = entity.device._modelID;
                     this.emit('new', entity);
                 }
             } else if (message.status === 'failed') {
                 this.error(`Failed to interview '${friendlyName}', device has not successfully been paired. Try again !!!!!!!!!! `);
                 //this.error(`Failed to interview '${friendlyName}', device has not successfully been paired. Try again !!!!!!!!!! ${message.error}`);
                 this.emit('pairing', 'Interview failed', friendlyName);
+                this.callExtensionMethod(
+                    'onZigbeeEvent',
+                    [message, entity ? entity.mapped : null],
+                );
             } else {
                 if (message.status === 'started') {
+                    this.callExtensionMethod(
+                        'onZigbeeEvent',
+                        [message, entity ? entity.mapped : null],
+                    );
                     this.info(`Starting interview of '${friendlyName}'`);
                     this.emit('pairing', 'Interview started', friendlyName);
                 }
@@ -899,6 +989,11 @@ class ZigbeeController extends EventEmitter {
 
     async handleMessage(data) {
         if (this.debugActive) this.debug(`handleMessage`, data);
+
+        const is = data.device.interviewState;
+        if (is != 'SUCCESSFUL' && is != 'FAILED') {
+            this.info(`message ${JSON.stringify(data)} received during interview.`)
+        }
         const entity = await this.resolveEntity(data.device || data.ieeeAddr);
         const name = (entity && entity._modelID) ? entity._modelID : data.device.ieeeAddr;
         if (this.debugActive) this.debug(
@@ -907,11 +1002,10 @@ class ZigbeeController extends EventEmitter {
             (data.hasOwnProperty('groupID') ? ` with groupID ${data.groupID}` : ``)
         );
         this.event(data.type, entity, data);
-
         // Call extensions
         this.callExtensionMethod(
             'onZigbeeEvent',
-            [data, entity ? entity.mapped : null],
+            [{...data, options:entity.options || {}}, entity ? entity.mapped : null],
         );
     }
 
@@ -1000,7 +1094,14 @@ class ZigbeeController extends EventEmitter {
         }
     }
 
+    processSyncStatesList(deviceId, model, syncStateList) {
+        syncStateList.forEach((syncState) => {
+            this.emit('acknowledge_state',deviceId, model, syncState.stateDesc, syncState.value);
+        });
+    }
 
+    // publishing to the zigbee network
+    // publish raw zigbee data
     async publish(deviceID, cid, cmd, zclData, cfg, ep, type, callback, zclSeqNum) {
         const entity = await this.resolveEntity(deviceID, ep);
         const device = entity.device;
@@ -1024,37 +1125,413 @@ class ZigbeeController extends EventEmitter {
             cfg = {};
         }
 
+        //        try { NO Try/Catach here, this is ONLY called from the developer tab and error handling is done there
+
+        if (type === 'foundation') {
+            cfg.disableDefaultResponse = true;
+            let result;
+            if (cmd === 'configReport') {
+                result = await endpoint.configureReporting(cid, zclData, cfg);
+            } else {
+                if (cmd === 'read' && !Array.isArray(zclData))
+                    result = await endpoint[cmd](cid, Object.keys(zclData), cfg);
+                else
+                    result = await endpoint[cmd](cid, zclData, cfg);
+            }
+            callback && callback(undefined, result);
+        } else if (type === 'functionalResp') {
+            cfg.disableDefaultResponse = false;
+            const result = await endpoint.commandResponse(cid, cmd, zclData, cfg, zclSeqNum);
+            callback && callback(undefined, result);
+        } else {
+            cfg.disableDefaultResponse = false;
+            const result = await endpoint.command(cid, cmd, zclData, cfg);
+            callback && callback(undefined, result);
+        }
+        /*        }
+        catch (error)
+        {
+            //this.error(`error sending ${type} ${cmd} to endpoint: ${(error && error.message ? error.message : 'no error message')} ${(error && error.stack ? error.stack : 'no call stack')}`)
+        }
+        */
+    }
+    // publish via converter
+    //
+    async publishFromState(deviceId, model, stateModel, stateList, options, debugID, has_elevated_debug) {
+        let isGroup = false;
+        //const has_elevated_debug = this.stController.checkDebugDevice(deviceId)
+
+        if (has_elevated_debug)
+        {
+            const stateNames = stateList.map((state) => state.stateDesc.id);
+            const message = `Publishing to ${deviceId} of model ${model} with ${stateNames.join(', ')}`;
+            this.emit('device_debug', { ID:debugID, data: { ID: deviceId, flag: '03', IO:false }, message: message});
+        }
+        else
+            if (this.debugActive) this.debug(`main publishFromState : ${deviceId} ${model} ${safeJsonStringify(stateList)}`);
+        if (model === 'group') {
+            isGroup = true;
+            deviceId = parseInt(deviceId);
+        }
         try {
+            const entity = await this.resolveEntity(deviceId);
+            if (this.debugActive) this.debug(`entity: ${deviceId} ${model} ${safeJsonStringify(entity)}`);
+            const mappedModel = entity ? entity.mapped : undefined;
+
+            if (!mappedModel) {
+                if (this.debugActive) this.debug(`No mapped model for ${model}`);
+                if (has_elevated_debug) {
+                    const message=`No mapped model ${deviceId} (model ${model})`;
+                    this.emit('device_debug', { ID:debugID, data: { error: 'NOMODEL' , IO:false }, message: message});
+                }
+                return;
+            }
 
-            if (type === 'foundation') {
-                cfg.disableDefaultResponse = true;
-                let result;
-                if (cmd === 'configReport') {
-                    result = await endpoint.configureReporting(cid, zclData, cfg);
+            if (!mappedModel.toZigbee)
+            {
+                this.error(`No toZigbee in mapped model for ${model}`);
+                return;
+            }
+
+            stateList.forEach(async changedState => {
+                const stateDesc = changedState.stateDesc;
+                const value = changedState.value;
+
+                let converter = undefined;
+                let msg_counter = 0;
+                if (stateDesc.isOption) {
+                    if (has_elevated_debug) {
+                        const message = `No converter needed on option state for ${deviceId} of type ${model}`;
+                        this.emit('device_debug', { ID:debugID, data: { flag: `SUCCESS` , IO:false }, message:message});
+                    }
+                    else
+                        if (this.debugActive) this.debug(`No converter needed on option state for ${deviceId} of type ${model}`);
+
+                    return;
+                }
+                for (const c of mappedModel.toZigbee) {
+
+                    if (!c.hasOwnProperty('convertSet')) continue;
+                    if (this.debugActive) this.debug(`Type of toZigbee is '${typeof c}', Contains key ${(c.hasOwnProperty('key')?JSON.stringify(c.key):'false ')}`)
+                    if (!c.hasOwnProperty('key'))
+                    {
+                        if (converter === undefined)
+                        {
+                            converter = c;
+                            if (has_elevated_debug) {
+                                const message = `Setting converter to keyless converter for ${deviceId} of type ${model}`;
+                                this.emit('device_debug', { ID:debugID, data: { flag: `s4.${msg_counter}` , IO:false }, message:message});
+                            }
+                            else
+                                if (this.debugActive) this.debug(`Setting converter to keyless converter for ${deviceId} of type ${model}`);
+                            msg_counter++;
+                        }
+                        else
+                        {
+                            if (has_elevated_debug)
+                            {
+                                const message = `ignoring keyless converter for ${deviceId} of type ${model}`;
+                                this.emit('device_debug', { ID:debugID, data: { flag: `i4.${msg_counter}` , IO:false} , message:message});
+                            }
+                            else
+                                if (this.debugActive) this.debug(`ignoring keyless converter for ${deviceId} of type ${model}`);
+                            msg_counter++;
+                        }
+                        continue;
+                    }
+                    if (c.key.includes(stateDesc.prop) || c.key.includes(stateDesc.setattr) || c.key.includes(stateDesc.id))
+                    {
+                        const message = `${(converter===undefined?'Setting':'Overriding')}' converter to converter with key(s)'${JSON.stringify(c.key)}}`;
+                        if (has_elevated_debug) {
+                            this.emit('device_debugug', { ID:debugID, data: { flag: `${converter===undefined ? 's' : 'o'}4.${msg_counter}` , IO:false }, message:message});
+
+                        }
+                        else
+                            if (this.debugActive) this.debug(message);
+                        converter = c;
+                        msg_counter++;
+                    }
+                }
+                if (converter === undefined) {
+                    const message = `No converter available for '${model}' with key '${stateDesc.id}' `;
+                    if (has_elevated_debug) {
+                        this.emit('device_debug', { ID:debugID, data: { error: 'NOCONV',states:[{id:stateDesc.id, value:value, payload:'no converter'}] , IO:false }, message:message});
+                    }
+                    else {
+                        this.info(message);
+                    }
+                    return;
+                }
+
+                const preparedValue = (stateDesc.setter) ? stateDesc.setter(value, options) : value;
+                const preparedOptions = (stateDesc.setterOpt) ? stateDesc.setterOpt(value, options) : {};
+
+                let syncStateList = [];
+                if (stateModel && stateModel.syncStates) {
+                    stateModel.syncStates.forEach(syncFunct => {
+                        const res = syncFunct(stateDesc, value, options);
+                        if (res) {
+                            syncStateList = syncStateList.concat(res);
+                        }
+                    });
+                }
+
+                const epName = stateDesc.epname !== undefined ? stateDesc.epname : (stateDesc.prop || stateDesc.id);
+                const key = stateDesc.setattr || stateDesc.prop || stateDesc.id;
+                const message = `convert ${key} with value ${safeJsonStringify(preparedValue)} and options ${safeJsonStringify(preparedOptions)} for device ${deviceId} with Endpoint ${epName}`;
+                if (has_elevated_debug) {
+                    this.emit('device_debug', { ID:debugID, data: { flag: '04', payload: {key:key, ep: stateDesc.epname, value:preparedValue, options:preparedOptions}, IO:false }, message:message});
+                }
+                else
+                    if (this.debugActive) this.debug(message);
+
+                let target;
+                if (model === 'group') {
+                    target = entity.mapped;
                 } else {
-                    if (cmd === 'read' && !Array.isArray(zclData))
-                        result = await endpoint[cmd](cid, Object.keys(zclData), cfg);
+                    target = await this.resolveEntity(deviceId, epName);
+                    target = target.endpoint;
+                }
+
+                if (this.debugActive) this.debug(`target: ${safeJsonStringify(target)}`);
+
+                const meta = {
+                    endpoint_name: epName,
+                    options: preparedOptions,
+                    device: entity.device,
+                    mapped: model === 'group' ? [] : mappedModel,
+                    message: {[key]: preparedValue},
+                    logger: this,
+                    state: {},
+                };
+
+                // new toZigbee
+                if (preparedValue !== undefined && Object.keys(meta.message).filter(p => p.startsWith('state')).length > 0) {
+                    if (typeof preparedValue === 'number') {
+                        meta.message.state = preparedValue > 0 ? 'ON' : 'OFF';
+                    } else {
+                        meta.message.state = preparedValue;
+                    }
+                }
+                if (has_elevated_debug) {
+                    this.emit('device_debug', { ID:debugID, data: { states:[{id:stateDesc.id, value:value, payload:preparedValue, ep:stateDesc.epname}] , IO:false }});
+                }
+
+                if (preparedOptions !== undefined) {
+                    if (preparedOptions.hasOwnProperty('state')) {
+                        meta.state = preparedOptions.state;
+                    }
+                }
+
+                try {
+                    const result = await converter.convertSet(target, key, preparedValue, meta);
+                    const message = `convert result ${safeJsonStringify(result)} for device ${deviceId}`;
+                    if (isGroup)
+                        this.emit('published', deviceId, model, stateModel, stateList, options, debugID, has_elevated_debug );
+                    if (has_elevated_debug) {
+                        this.emit('device_debug', { ID:debugID, data: { flag: 'SUCCESS' , IO:false }, message:message});
+                    }
                     else
-                        result = await endpoint[cmd](cid, zclData, cfg);
+                        if (this.debugActive) this.debug(message);
+                    if (result !== undefined) {
+                        if (stateModel && !isGroup && !stateDesc.noack) {
+                            this.emit('acknowledge_state', deviceId, model, stateDesc, value );
+                        }
+                        // process sync state list
+                        this.processSyncStatesList(deviceId, model, syncStateList);
+                    }
+                    else {
+                        if (has_elevated_debug) {
+                            const message = `Convert does not return a result result for ${key} with ${safeJsonStringify(preparedValue)} on device ${deviceId}.`;
+                            this.emit('device_debug', { ID:debugID, data: { flag: '06' , IO:false }, message:message});
+                        }
+                    }
+                } catch (error) {
+                    if (has_elevated_debug) {
+                        const message = `caught error ${error && error.message ? error.message : 'no reason given'} when setting value for device ${deviceId}.`;
+                        this.emit('device_debug', { ID:debugID, data: { error: 'EXSET' , IO:false },message:message});
+                    }
+                    this.adapter.filterError(`Error ${error.code} on send command to ${deviceId}.` +
+                        ` Error: ${error.stack}`, `Send command to ${deviceId} failed with`, error);
                 }
-                callback && callback(undefined, result);
-            } else if (type === 'functionalResp') {
-                cfg.disableDefaultResponse = false;
-                const result = await endpoint.commandResponse(cid, cmd, zclData, cfg, zclSeqNum);
-                callback && callback(undefined, result);
-            } else {
-                cfg.disableDefaultResponse = false;
-                const result = await endpoint.command(cid, cmd, zclData, cfg);
-                callback && callback(undefined, result);
+            });
+        } catch (err) {
+            const message = `No entity for ${deviceId} : ${err && err.message ? err.message : 'no error message'}`;
+            this.emit('device_debug', { ID:debugID, data: { error: 'EXPUB' , IO:false }, message:message});
+        }
+    }
+
+    extractEP(key, endpoints) {
+        try {
+            if (endpoints) for (const ep of Object.keys(endpoints)) {
+                if (key.endsWith('_'+ep)) return { setattr: key.replace('_'+ep, ''), epname:ep }
             }
         }
-        catch (error)
-        {
-            this.log.error(`error sending ${type} ${cmd} to endpoint: ${(error && error.message ? error.message : 'no error message')} ${(error && error.stack ? error.stack : 'no call stack')}`)
+        catch {
+            return {};
+        }
+        return {};
+    }
+
+    // publish via payload
+    //
+    // This function is introduced to explicitly allow user level scripts to send Commands
+    // directly to the zigbee device. It utilizes the zigbee-herdsman-converters to generate
+    // the exact zigbee message to be sent and can be used to set device options which are
+    // not exposed as states. It serves as a wrapper function for "publishFromState" with
+    // extended parameter checking
+    //
+    // The payload can either be a JSON object or the string representation of a JSON object
+    // The following keys are supported in the object:
+    // device: name of the device. For a device zigbee.0.0011223344556677 this would be 0011223344556677
+    // payload: The data to send to the device as JSON object (key/Value pairs)
+    // endpoint: optional: the endpoint to send the data to, if supported.
+    //
+    async publishPayload(payload, debugID, has_elevated_debug) {
+        let payloadObj = {};
+        if (typeof payload === 'string') {
+            try {
+                payloadObj = JSON.parse(payload);
+            } catch (e) {
+                this.log.error(`Unable to parse ${safeJsonStringify(payload)}: ${safeJsonStringify(e)}`);
+                return {
+                    success: false,
+                    error: `Unable to parse ${safeJsonStringify(payload)}: ${safeJsonStringify(e)}`
+                };
+            }
+        } else if (typeof payload === 'object') {
+            payloadObj = payload;
+        } else return { success: false, error: 'illegal type of payload: ' + typeof payload};
+
+        if (payloadObj.hasOwnProperty('device') && payloadObj.hasOwnProperty('payload')) {
+            try {
+                const isDevice = !payload.device.includes('group_');
+                const stateList = [];
+                const devID = isDevice ? `0x${payload.device}` : parseInt(payload.device.replace('group_', ''));
+
+                const entity = await this.resolveEntity(devID);
+                if (!entity) {
+                    this.log.error(`Device ${safeJsonStringify(payloadObj.device)} not found`);
+                    return {success: false, error: `Device ${safeJsonStringify(payloadObj.device)} not found`};
+                }
+                const mappedModel = entity.mapped;
+                if (!mappedModel) {
+                    this.log.error(`No Model for Device ${safeJsonStringify(payloadObj.device)}`);
+                    return {success: false, error: `No Model for Device ${safeJsonStringify(payloadObj.device)}`};
+                }
+                if (typeof payloadObj.payload !== 'object') {
+                    this.log.error(`Illegal payload type for ${safeJsonStringify(payloadObj.device)}`);
+                    return {success: false, error: `Illegal payload type for ${safeJsonStringify(payloadObj.device)}`};
+                }
+                const endpoints = mappedModel && mappedModel.endpoint ? mappedModel.endpoint(entity.device) : null;
+                for (const key in payloadObj.payload) {
+                    if (payloadObj.payload[key] != undefined) {
+                        const datatype = typeof payloadObj.payload[key];
+                        const epobj = this.extractEP(key, endpoints);
+                        if (payloadObj.endpoint) {
+                            epobj.epname = payloadObj.endpoint;
+                            delete epobj.setattr;
+                        }
+                        stateList.push({
+                            stateDesc: {
+                                id: key,
+                                prop: key,
+                                role: 'state',
+                                type: datatype,
+                                noack:true,
+                                epname: epobj.epname,
+                                setattr: epobj.setattr,
+                            },
+                            value: payloadObj.payload[key],
+                            index: 0,
+                            timeout: 0,
+                        });
+                    }
+                }
+                try {
+                    await this.publishFromState(`0x${payload.device}`, payload.model, payload.stateModel, stateList, payload.options, debugID, has_elevated_debug);
+                    return {success: true};
+                } catch (error) {
+                    this.log.error(`Error ${error.code} on send command to ${payload.device}.` + ` Error: ${error.stack} ` + `Send command to ${payload.device} failed with ` + error);
+                    this.adapter.filterError(`Error ${error.code} on send command to ${payload.device}.` + ` Error: ${error.stack}`, `Send command to ${payload.device} failed with`, error);
+                    return {success: false, error};
+                }
+            } catch (e) {
+                return {success: false, error: e};
+            }
         }
 
+        return {success: false, error: `missing parameter device or payload in message ${JSON.stringify(payload)}`};
     }
 
+    async doDeviceQuery(deviceId, debugID, elevated) {
+        const entity = await this.resolveEntity(deviceId);
+        if (this.debugActive) this.debug(`doDeviceQuery: resolveEntity for entity: ${deviceId} is ${safeJsonStringify(entity)}`);
+        const mappedModel = entity ? entity.mapped : undefined;
+        if (mappedModel) {
+            const epmap = mappedModel.endpoint ? mappedModel.endpoint() : [];
+            if (elevated) {
+                const message  = `Device query for '${entity.device.ieeeAddr}' triggered`;
+                this.emit('device_debug', { ID:debugID, data: { flag: 'qs' ,states:[{id:'device_query', value:true, payload:'device_query'}], IO:false }, message:message});
+            }
+            else
+                if (this.debugActive) this.debug(`Device query for '${entity.device.ieeeAddr}' started`);
+                else this.info(`Device query for '${entity.device.ieeeAddr}' started`);
+
+            for (const converter of mappedModel.toZigbee) {
+                if (converter.hasOwnProperty('convertGet')) {
+                    const sources = [];
+                    if (converter.endpoints && epmap) {
+                        for (const epname of converter.endpoints) {
+                            const source = entity.device.endpoints.find((id) => id.ID == epmap[epname]);
+                            if (source) sources.push(source);
+                        }
+                    }
+                    if (sources.length == 0) sources.push(entity.device.endpoints[0]);
+                    for (const source of sources) {
+                        try {
+                            await converter.convertGet(source, '', {device:entity.device});
+                            this.debug(`read for state${converter.key.length ? '' : 's'} '${converter.key.join(',')}' of '${entity.device.ieeeAddr}/${source.ID}' after device query`);
+                        } catch (error) {
+                            if (elevated) {
+                                const message = `Failed to read for state${converter.key.length ? '' : 's'} '${converter.key.join(',')}' of '${source.ID}' from query with '${error && error.message ? error.message : 'no error message'}`;
+                                this.warn(`ELEVATED OE02.1 ${message}`);
+                                this.emit('device_debug', { ID:debugID, data: { error: 'NOTREAD' , IO:false }, message:message });
+                            }
+                            else
+                                this.debug(`failed to read for state${converter.key.length ? '' : 's'} '${converter.key.join(',')}' of '${source.ID}'after device query`);
+                        }
+                    }
+                }
+            }
+            if (elevated) {
+                const message = `ELEVATED O07: Device query for '${entity.device.ieeeAddr}}' complete`;
+                this.emit('device_debug', { ID:debugID, data: { flag: 'qe' , IO:false }, message:message});
+            }
+            else
+                this.info(`Device query for '${entity.device.ieeeAddr}' complete`);
+        }
+    }
+
+    async deviceQuery(deviceId, debugID, elevated, callback) {
+        if (this.deviceQueryActive.includes (deviceId)) {
+            this.info(`Device query for ${deviceId} is still active.`);
+            return;
+        }
+        this.deviceQueryActive.push(deviceId);
+        try {
+            await this.doDeviceQuery(deviceId, debugID, elevated);
+        }
+        catch (e) {
+            this.warn('error in doDeviceQuery')
+        }
+        const idx = this.deviceQueryActive.indexOf(deviceId)
+        if (idx > -1)
+            this.deviceQueryActive.splice(idx, 1);
+        if (callback) callback(deviceId);
+    }
+
+
     async addDevToGroup(devId, groupId, epid) {
         try {
             if (this.debugActive) this.debug(`called addDevToGroup with ${devId}, ${groupId}, ${epid}`);