@iobroker/db-objects-jsonl 4.0.0-alpha.8-20210909-001a711c → 4.0.3

package/README.md

@@ -4,4 +4,4 @@ The Library contains the JSONL Database classes for File based objects database
 ## License
 The MIT License (MIT)
 
-Copyright 2018-2021 bluefox <dogafox@gmail.com>
+Copyright 2018-2022 bluefox <dogafox@gmail.com>

package/lib/objects/objectsInMemJsonlDB.js

@@ -1,7 +1,7 @@
 /**
  *      Object DB in memory - Server
  *
- *      Copyright 2013-2020 bluefox <dogafox@gmail.com>
+ *      Copyright 2013-2022 bluefox <dogafox@gmail.com>
  *
  *      MIT License
  *
@@ -13,49 +13,50 @@
 /* jshint -W061 */
 'use strict';
 
-const ObjectsInMemoryFileDB        = require('@iobroker/db-objects-file').ObjectsInMemoryFileDB;
+const ObjectsInMemoryFileDB = require('@iobroker/db-objects-file').ObjectsInMemoryFileDB;
 const { JsonlDB } = require('@alcalzone/jsonl-db');
 const path = require('path');
+const fs = require('fs');
+const os = require('os');
+const { tools } = require('@iobroker/js-controller-common');
 
 /**
  * This class inherits InMemoryFileDB class and adds all relevant logic for objects
  * including the available methods for use by js-controller directly
  **/
 class ObjectsInMemoryJsonlDB extends ObjectsInMemoryFileDB {
-
     constructor(settings) {
         settings = settings || {};
         settings.fileDB = {
             fileName: 'objects.json',
             backupDirName: 'backup-objects'
         };
+        super(settings);
+
         /** @type {import("@alcalzone/jsonl-db").JsonlDBOptions<any>} */
-        const jsonlOptions = {
+        const jsonlOptions = this.settings.connection.jsonlOptions || {
             autoCompress: {
                 sizeFactor: 2,
-                sizeFactorMinimumSize: 1000
+                sizeFactorMinimumSize: 25000
             },
             ignoreReadErrors: true,
             throttleFS: {
                 intervalMs: 60000,
-                maxBufferedCommands: 100
+                maxBufferedCommands: 1000
             }
         };
         settings.jsonlDB = {
-            fileName: 'objects.jsonl',
-            options: jsonlOptions
+            fileName: 'objects.jsonl'
         };
-        super(settings);
 
         /** @type {JsonlDB<any>} */
-        this._db = new JsonlDB(
-            path.join(this.dataDir, settings.jsonlDB.fileName),
-            jsonlOptions
-        );
+        this._db = new JsonlDB(path.join(this.dataDir, settings.jsonlDB.fileName), jsonlOptions);
     }
 
     async open() {
-        await this._db.open();
+        if (!(await this._maybeMigrateFileDB())) {
+            await this._db.open();
+        }
 
         // Create an object-like wrapper around the internal Map
         this.dataset = new Proxy(this._db, {
@@ -92,18 +93,128 @@ class ObjectsInMemoryJsonlDB extends ObjectsInMemoryFileDB {
                 };
             }
         });
+
+        if (this.settings.backup && this.settings.backup.period && !this.settings.backup.disabled) {
+            this._backupInterval = setInterval(() => {
+                this.saveBackup();
+            }, this.settings.backup.period);
+        }
+    }
+
+    /**
+     * Checks if an existing file DB should be migrated to JSONL
+     * @returns {Promise<boolean>} true if the file DB was migrated. false if not.
+     * If this returns true, the jsonl DB was opened and doesn't need to be opened again.
+     */
+    async _maybeMigrateFileDB() {
+        const jsonlFileName = path.join(this.dataDir, this.settings.jsonlDB.fileName);
+        const jsonFileName = path.join(this.dataDir, this.settings.fileDB.fileName);
+        const bakFileName = path.join(this.dataDir, this.settings.fileDB.fileName + '.bak');
+
+        // Check the timestamps of each file, defaulting to 0 if they don't exist
+        let jsonlTimeStamp = 0;
+        let jsonTimeStamp = 0;
+        let bakTimeStamp = 0;
+        try {
+            const stat = fs.statSync(jsonlFileName);
+            if (stat.isFile()) {
+                jsonlTimeStamp = stat.mtime;
+            }
+        } catch {
+            // ignore
+        }
+        try {
+            const stat = fs.statSync(jsonFileName);
+            if (stat.isFile()) {
+                jsonTimeStamp = stat.mtime;
+            }
+        } catch {
+            // ignore
+        }
+        try {
+            const stat = fs.statSync(bakFileName);
+            if (stat.isFile()) {
+                bakTimeStamp = stat.mtime;
+            }
+        } catch {
+            // ignore
+        }
+
+        // Figure out which file needs to be imported
+        /** @type {string} */
+        let importFilename;
+        if (jsonTimeStamp > 0 && jsonTimeStamp >= bakTimeStamp && jsonTimeStamp >= jsonlTimeStamp) {
+            importFilename = jsonFileName;
+        } else if (bakTimeStamp > 0 && bakTimeStamp >= jsonTimeStamp && bakTimeStamp >= jsonlTimeStamp) {
+            importFilename = bakFileName;
+        } else {
+            // None of the File DB files are newer than the JSONL file
+            // There is nothing to restore, we're done
+            return false;
+        }
+
+        await this._db.open();
+        this._db.clear();
+        await this._db.importJson(importFilename);
+
+        // And rename the existing files to avoid redoing the work next time
+        if (fs.existsSync(jsonFileName)) {
+            try {
+                fs.renameSync(jsonFileName, `${jsonFileName}.migrated`);
+            } catch {
+                // ignore
+            }
+        }
+        if (fs.existsSync(bakFileName)) {
+            try {
+                fs.renameSync(bakFileName, `${bakFileName}.migrated`);
+            } catch {
+                // ignore
+            }
+        }
+
+        // Signal to the caller that the DB is already open
+        return true;
     }
 
     async saveState() {
         // Nothing to do, the DB saves behind the scenes
     }
 
+    // Is regularly called and stores a compressed backup of the DB
+    async saveBackup() {
+        const now = Date.now();
+        const tmpBackupFileName = path.join(os.tmpdir(), `${this.getTimeStr(now)}_${this.settings.jsonlDB.fileName}`);
+        const backupFileName = path.join(
+            this.backupDir,
+            `${this.getTimeStr(now)}_${this.settings.jsonlDB.fileName}.gz`
+        );
+        try {
+            if (fs.existsSync(backupFileName)) {
+                return;
+            }
+
+            // Create a DB dump
+            await this._db.dump(tmpBackupFileName);
+            // and zip it
+            await tools.compressFileGZip(tmpBackupFileName, backupFileName, { deleteInput: true });
+            // figure out if older files need to be deleted
+            this.deleteOldBackupFiles(this.settings.jsonlDB.fileName);
+        } catch (e) {
+            this.log.error(`${this.namespace} Cannot save backup ${backupFileName}: ${e.message}`);
+        }
+    }
+
     async destroy() {
+        await super.destroy();
+
         if (this._db) {
             await this._db.close();
         }
+        if (this._backupInterval) {
+            clearInterval(this._backupInterval);
+        }
     }
-
 }
 
 module.exports = ObjectsInMemoryJsonlDB;

package/lib/objects/objectsInMemServerClass.js

@@ -1,7 +1,7 @@
 /**
  *      States DB in memory - Server with Redis protocol
  *
- *      Copyright 2013-2020 bluefox <dogafox@gmail.com>
+ *      Copyright 2013-2022 bluefox <dogafox@gmail.com>
  *
  *      MIT License
  *
@@ -18,28 +18,12 @@ const ObjectsInRedisClient = require('@iobroker/db-objects-redis').Client;
 const ObjectsInMemServer = require('./objectsInMemServerRedis');
 
 class ObjectsInMemoryServerClass extends ObjectsInRedisClient {
-
     constructor(settings) {
         settings.autoConnect = false; // delay Client connection to when we need it
-
-        // hack around testing problem where subscribe was called before connect
-        // Should be removed for a later release
-        const origConnected = settings.connected;
-        settings.connected = () => {
-            this.clientConnected = true;
-            if (Array.isArray(this.storedSubscribes) && this.storedSubscribes.length) {
-                this.log.warn(`${this.namespace} Replay ${this.storedSubscribes.length} subscription calls for Objects Server that were done before the client was connected initially`);
-                this.storedSubscribes.forEach((s => this.subscribe(s.pattern, s.options, s.callback)));
-                this.storedSubscribes = [];
-            }
-            origConnected();
-        };
         super(settings);
-        this.clientConnected = false;
-        this.storedSubscribes = [];
 
         const serverSettings = {
-            namespace:  settings.namespace ? `${settings.namespace}-Server` : 'Server',
+            namespace: settings.namespace ? `${settings.namespace}-Server` : 'Server',
             connection: settings.connection,
             logger: settings.logger,
             hostname: settings.hostname,
@@ -66,13 +50,5 @@ class ObjectsInMemoryServerClass extends ObjectsInRedisClient {
     dirExists(id, name) {
         return this.objectsServer.dirExists(id, name);
     }
-
-    subscribe(pattern, options, callback) {
-        if (!this.clientConnected) {
-            this.storedSubscribes.push({pattern, options, callback}); // we ignore the promise return because not used for this testing issue we work around here
-        } else {
-            return super.subscribe(pattern, options, callback);
-        }
-    }
 }
 module.exports = ObjectsInMemoryServerClass;

package/lib/objects/objectsInMemServerRedis.js

@@ -13,14 +13,14 @@
 /* jshint strict:false */
 /* jslint node: true */
 'use strict';
-const net    = require('net');
-const fs     = require('fs-extra');
-const path   = require('path');
+const net = require('net');
+const fs = require('fs-extra');
+const path = require('path');
 const crypto = require('crypto');
-const utils  = require('@iobroker/db-objects-redis').objectsUtils;
-const tools  = require('@iobroker/db-base').tools;
+const utils = require('@iobroker/db-objects-redis').objectsUtils;
+const tools = require('@iobroker/db-base').tools;
 
-const RedisHandler          = require('@iobroker/db-base').redisHandler;
+const RedisHandler = require('@iobroker/db-base').redisHandler;
 const ObjectsInMemoryJsonlDB = require('./objectsInMemJsonlDB');
 
 // settings = {
@@ -57,30 +57,48 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
         super(settings);
 
         this.serverConnections = {};
-        this.namespaceObjects    = (this.settings.redisNamespace || (settings.connection && settings.connection.redisNamespace) || 'cfg') + '.';
-        this.namespaceFile       = this.namespaceObjects + 'f.';
-        this.namespaceObj        = this.namespaceObjects + 'o.';
+        this.namespaceObjects =
+            (this.settings.redisNamespace || (settings.connection && settings.connection.redisNamespace) || 'cfg') +
+            '.';
+        this.namespaceFile = this.namespaceObjects + 'f.';
+        this.namespaceObj = this.namespaceObjects + 'o.';
+        this.namespaceSet = this.namespaceObjects + 's.';
+        this.namespaceSetLen = this.namespaceSet.length;
+
         // this.namespaceObjectsLen   = this.namespaceObjects.length;
-        this.namespaceFileLen    = this.namespaceFile.length;
-        this.namespaceObjLen     = this.namespaceObj.length;
+        this.namespaceFileLen = this.namespaceFile.length;
+        this.namespaceObjLen = this.namespaceObj.length;
+        this.metaNamespace = (this.settings.metaNamespace || 'meta') + '.';
+        this.metaNamespaceLen = this.metaNamespace.length;
 
         this.knownScripts = {};
 
         this.normalizeFileRegex1 = new RegExp('^(.*)\\$%\\$(.*)\\$%\\$(meta|data)$');
         this.normalizeFileRegex2 = new RegExp('^(.*)\\$%\\$(.*)\\/?\\*$');
 
-        this.open().then(() => {
-            return this._initRedisServer(this.settings.connection);
-        }).then(() => {
-            this.log.debug(this.namespace + ' ' + (settings.secure ? 'Secure ' : '') + ' Redis inMem-objects listening on port ' + (settings.port || 9001));
+        this.open()
+            .then(() => {
+                return this._initRedisServer(this.settings.connection);
+            })
+            .then(() => {
+                this.log.debug(
+                    this.namespace +
+                        ' ' +
+                        (settings.secure ? 'Secure ' : '') +
+                        ' Redis inMem-objects listening on port ' +
+                        (settings.port || 9001)
+                );
 
-            if (typeof this.settings.connected === 'function') {
-                setImmediate(() => this.settings.connected());
-            }
-        }).catch(e => {
-            this.log.error(this.namespace + ' Cannot start inMem-objects on port ' + (settings.port || 9001) + ': ' + e.message);
-            process.exit(24); // todo: replace it with exitcode
-        });
+                if (typeof this.settings.connected === 'function') {
+                    setImmediate(() => this.settings.connected());
+                }
+            })
+            .catch(e => {
+                this.log.error(
+                    this.namespace + ' Cannot start inMem-objects on port ' + (settings.port || 9001) + ': ' + e.message
+                );
+                process.exit(24); // todo: replace it with exitcode
+            });
     }
 
     /**
@@ -98,7 +116,7 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
         if (Array.isArray(idWithNamespace)) {
             const ids = [];
             idWithNamespace.forEach(el => {
-                const {id, namespace} = this._normalizeId(el);
+                const { id, namespace } = this._normalizeId(el);
                 ids.push(id);
                 ns = namespace; // we ignore the pot. case from arrays with different namespaces
             });
@@ -111,6 +129,8 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                     idx = this.namespaceObjLen;
                 } else if (idWithNamespace.startsWith(this.namespaceFile)) {
                     idx = this.namespaceFileLen;
+                } else if (idWithNamespace.startsWith(this.namespaceSet)) {
+                    idx = this.namespaceSetLen;
                 }
                 if (idx !== -1) {
                     ns = idWithNamespace.substr(0, idx);
@@ -121,7 +141,7 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                     if (fileIdDetails) {
                         id = fileIdDetails[1];
                         name = fileIdDetails[2] || '';
-                        isMeta = (fileIdDetails[3] === 'meta');
+                        isMeta = fileIdDetails[3] === 'meta';
                     } else {
                         fileIdDetails = id.match(this.normalizeFileRegex2);
                         if (fileIdDetails) {
@@ -134,9 +154,15 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                         }
                     }
                 }
+            } else if (idWithNamespace.startsWith(this.metaNamespace)) {
+                const idx = this.metaNamespaceLen;
+                if (idx !== -1) {
+                    ns = idWithNamespace.substr(0, idx);
+                    id = idWithNamespace.substr(idx);
+                }
             }
         }
-        return {id, namespace: ns, name, isMeta};
+        return { id, namespace: ns, name, isMeta };
     }
 
     /**
@@ -155,11 +181,18 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
 
         const found = s.find(sub => sub.regex.test(id));
         if (found) {
-            const objString = JSON.stringify(obj);
-            this.log.silly(this.namespace + ' Redis Publish Object ' + id + '=' + objString);
-            const sendPattern = (type === 'objects' ? '' : this.namespaceObjects) + found.pattern;
-            const sendId = (type === 'objects' ? this.namespaceObj : this.namespaceObjects) + id;
-            client.sendArray(null,['pmessage', sendPattern, sendId, objString]);
+            if (type === 'meta') {
+                this.log.silly(`${this.namespace} Redis Publish Meta ${id}=${obj}`);
+                const sendPattern = this.metaNamespace + found.pattern;
+                const sendId = this.metaNamespace + id;
+                client.sendArray(null, ['pmessage', sendPattern, sendId, obj]);
+            } else {
+                const objString = JSON.stringify(obj);
+                this.log.silly(this.namespace + ' Redis Publish Object ' + id + '=' + objString);
+                const sendPattern = (type === 'objects' ? '' : this.namespaceObjects) + found.pattern;
+                const sendId = (type === 'objects' ? this.namespaceObj : this.namespaceObjects) + id;
+                client.sendArray(null, ['pmessage', sendPattern, sendId, objString]);
+            }
             return 1;
         }
         return 0;
@@ -174,12 +207,11 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
      */
     getFileId(id, name, isMeta) {
         // e.g. ekey.admin and admin/ekey.png
-        if (id.match(/\.admin$/)) {
-            if (name.match(/^admin\//)) {
+        if (id.endsWith('.admin')) {
+            if (name.startsWith('admin/')) {
                 name = name.replace(/^admin\//, '');
-            } else
-            // e.g. ekey.admin and iobroker.ekey/admin/ekey.png
-            if (name.match(/^iobroker.[-\d\w]\/admin\//i)) {
+            } else if (name.match(/^iobroker.[-\d\w]\/admin\//i)) {
+                // e.g. ekey.admin and iobroker.ekey/admin/ekey.png
                 name = name.replace(/^iobroker.[-\d\w]\/admin\//i, '');
             }
         }
@@ -243,22 +275,41 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                         search = scriptSearch[1];
                     }
 
-                    this.knownScripts[scriptChecksum] = {design: design, search: search};
+                    this.knownScripts[scriptChecksum] = { design: design, search: search };
                     if (this.settings.connection.enhancedLogging) {
-                        this.log.silly(`${namespaceLog} Register View LUA Script: ${scriptChecksum} = ${JSON.stringify(this.knownScripts[scriptChecksum])}`);
+                        this.log.silly(
+                            `${namespaceLog} Register View LUA Script: ${scriptChecksum} = ${JSON.stringify(
+                                this.knownScripts[scriptChecksum]
+                            )}`
+                        );
                     }
                     handler.sendBulk(responseId, scriptChecksum);
                 } else if (scriptFunc && scriptFunc[1]) {
-                    this.knownScripts[scriptChecksum] = {func: scriptFunc[1]};
+                    this.knownScripts[scriptChecksum] = { func: scriptFunc[1] };
                     if (this.settings.connection.enhancedLogging) {
-                        this.log.silly(`${namespaceLog} Register Func LUA Script: ${scriptChecksum} = ${JSON.stringify(this.knownScripts[scriptChecksum])}`);
+                        this.log.silly(
+                            `${namespaceLog} Register Func LUA Script: ${scriptChecksum} = ${JSON.stringify(
+                                this.knownScripts[scriptChecksum]
+                            )}`
+                        );
+                    }
+                    handler.sendBulk(responseId, scriptChecksum);
+                } else if (data[1].includes('-- REDLOCK SCRIPT')) {
+                    // redlock scripts are currently not needed for Simulator
+                    this.knownScripts[scriptChecksum] = { redlock: true };
+                    if (this.settings.connection.enhancedLogging) {
+                        this.log.silly(
+                            `${namespaceLog} Register Func LUA Script: ${scriptChecksum} = ${JSON.stringify(
+                                this.knownScripts[scriptChecksum]
+                            )}`
+                        );
                     }
                     handler.sendBulk(responseId, scriptChecksum);
                 } else {
-                    handler.sendError(responseId, new Error('Unknown LUA script ' + data[0]));
+                    handler.sendError(responseId, new Error(`Unknown LUA script ${data[1]}`));
                 }
             } else {
-                handler.sendError(responseId, new Error('Unsupported Script command ' + data[0]));
+                handler.sendError(responseId, new Error(`Unsupported Script command ${data[0]}`));
             }
         });
 
@@ -269,7 +320,7 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
             }
             if (this.knownScripts[data[0]].design) {
                 const scriptDesign = this.knownScripts[data[0]].design;
-                if (data[2] === this.namespaceObj && data.length > 4) {
+                if (typeof data[2] === 'string' && data[2].startsWith(this.namespaceObj) && data.length > 4) {
                     let scriptSearch = this.knownScripts[data[0]].search;
                     if (scriptDesign === 'system' && !scriptSearch && data[5]) {
                         scriptSearch = data[5];
@@ -278,7 +329,9 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                         scriptSearch = 'state';
                     }
                     if (this.settings.connection.enhancedLogging) {
-                        this.log.silly(`${namespaceLog} Script transformed into getObjectView: design=${scriptDesign}, search=${scriptSearch}`);
+                        this.log.silly(
+                            `${namespaceLog} Script transformed into getObjectView: design=${scriptDesign}, search=${scriptSearch}`
+                        );
                     }
                     let objs;
                     try {
@@ -288,13 +341,17 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                             include_docs: true
                         });
                     } catch (err) {
-                        return void handler.sendError(responseId, new Error('_getObjectView Error for ' + scriptDesign + '/' + scriptSearch + ': ' + err.message));
+                        return void handler.sendError(
+                            responseId,
+                            new Error(`_getObjectView Error for ${scriptDesign}/${scriptSearch}: ${err.message}`)
+                        );
                     }
+
                     const res = objs.rows.map(obj => JSON.stringify(this.dataset[obj.value._id || obj.id]));
                     handler.sendArray(responseId, res);
                 }
             } else if (this.knownScripts[data[0]].func && data.length > 4) {
-                const scriptFunc = {map: this.knownScripts[data[0]].func.replace('%1', data[5])};
+                const scriptFunc = { map: this.knownScripts[data[0]].func.replace('%1', data[5]) };
                 if (this.settings.connection.enhancedLogging) {
                     this.log.silly(`${namespaceLog} Script transformed into _applyView: func=${scriptFunc.map}`);
                 }
@@ -306,16 +363,20 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                 const res = objs.rows.map(obj => JSON.stringify(this.dataset[obj.value._id || obj.id]));
 
                 return void handler.sendArray(responseId, res);
+            } else if (this.knownScripts[data[0]].redlock) {
+                // just return a dummy
+                return void handler.sendArray(responseId, [0]);
             } else {
-                handler.sendError(responseId, new Error('Unknown LUA script eval call ' + JSON.stringify(data)));
+                handler.sendError(responseId, new Error(`Unknown LUA script eval call ${JSON.stringify(data)}`));
             }
         });
 
         // Handle Redis "PUBLISH" request
         handler.on('publish', (data, responseId) => {
-            const {id, namespace} = this._normalizeId(data[0]);
+            const { id, namespace } = this._normalizeId(data[0]);
 
-            if (namespace === this.namespaceObj) { // a "set" always comes afterwards, so do not publish
+            if (namespace === this.namespaceObj || namespace === this.metaNamespace) {
+                // a "set" always comes afterwards, so do not publish
                 return void handler.sendInteger(responseId, 0); // do not publish for now
             }
             const publishCount = this.publishAll(namespace.substr(0, namespace.length - 1), id, JSON.parse(data[1]));
@@ -327,15 +388,17 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
             if (!data || !data.length) {
                 return void handler.sendArray(responseId, []);
             }
-            const {namespace, isMeta} = this._normalizeId(data[0]);
+            const { namespace, isMeta } = this._normalizeId(data[0]);
 
             if (namespace === this.namespaceObj) {
                 const keys = [];
                 data.forEach(dataId => {
-                    const {id, namespace} = this._normalizeId(dataId);
+                    const { id, namespace } = this._normalizeId(dataId);
                     if (namespace !== this.namespaceObj) {
                         keys.push(null);
-                        this.log.warn(`${namespaceLog} Got MGET request for non Object-ID in Objects-ID chunk for ${namespace} / ${dataId}`);
+                        this.log.warn(
+                            `${namespaceLog} Got MGET request for non Object-ID in Objects-ID chunk for ${namespace} / ${dataId}`
+                        );
                         return;
                     }
                     keys.push(id);
@@ -346,17 +409,19 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                 } catch (err) {
                     return void handler.sendError(responseId, new Error('ERROR _getObjects: ' + err.message));
                 }
-                result = result.map(el => el ? JSON.stringify(el) : null);
+                result = result.map(el => (el ? JSON.stringify(el) : null));
                 handler.sendArray(responseId, result);
             } else if (namespace === this.namespaceFile) {
                 // Handle request for Meta data for files
                 if (isMeta) {
                     const response = [];
                     data.forEach(dataId => {
-                        const {id, namespace, name} = this._normalizeId(dataId);
+                        const { id, namespace, name } = this._normalizeId(dataId);
                         if (namespace !== this.namespaceFile) {
                             response.push(null);
-                            this.log.warn(`${namespaceLog} Got MGET request for non File ID in File-ID chunk for ${dataId}`);
+                            this.log.warn(
+                                `${namespaceLog} Got MGET request for non File ID in File-ID chunk for ${dataId}`
+                            );
                             return;
                         }
                         this._loadFileSettings(id);
@@ -370,7 +435,9 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                             obj.stats = fs.statSync(path.join(this.objectsDir, id, name));
                         } catch (err) {
                             if (!name.endsWith('/_data.json')) {
-                                this.log.warn(`${namespaceLog} Got MGET request for non existing file ${dataId}, err: ${err.message}`);
+                                this.log.warn(
+                                    `${namespaceLog} Got MGET request for non existing file ${dataId}, err: ${err.message}`
+                                );
                             }
                             response.push(null);
                             return;
@@ -383,13 +450,16 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                     handler.sendError(responseId, new Error('MGET-UNSUPPORTED for file data'));
                 }
             } else {
-                handler.sendError(responseId, new Error('MGET-UNSUPPORTED for namespace ' + namespace + ': Data=' + JSON.stringify(data)));
+                handler.sendError(
+                    responseId,
+                    new Error('MGET-UNSUPPORTED for namespace ' + namespace + ': Data=' + JSON.stringify(data))
+                );
             }
         });
 
         // Handle Redis "GET" requests
         handler.on('get', (data, responseId) => {
-            const {id, namespace, name, isMeta} = this._normalizeId(data[0]);
+            const { id, namespace, name, isMeta } = this._normalizeId(data[0]);
 
             if (namespace === this.namespaceObj) {
                 const result = this._getObject(id);
@@ -408,11 +478,14 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                         return void handler.sendNull(responseId);
                     }
                     if (stats.isDirectory()) {
-                        return void handler.sendBulk(responseId, JSON.stringify({
-                            file: name,
-                            stats: {},
-                            isDir: true
-                        }));
+                        return void handler.sendBulk(
+                            responseId,
+                            JSON.stringify({
+                                file: name,
+                                stats: {},
+                                isDir: true
+                            })
+                        );
                     }
                     this._loadFileSettings(id);
                     if (!this.fileOptions[id] || !this.fileOptions[id][name]) {
@@ -424,9 +497,16 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                         obj = {
                             mimeType: obj,
                             acl: {
-                                owner: (this.defaultNewAcl && this.defaultNewAcl.owner) || utils.CONSTS.SYSTEM_ADMIN_USER,
-                                ownerGroup: (this.defaultNewAcl && this.defaultNewAcl.ownerGroup) || utils.CONSTS.SYSTEM_ADMIN_GROUP,
-                                permissions: (this.defaultNewAcl && this.defaultNewAcl.file.permissions) || (utils.CONSTS.ACCESS_USER_ALL | utils.CONSTS.ACCESS_GROUP_ALL | utils.CONSTS.ACCESS_EVERY_ALL) // 777
+                                owner:
+                                    (this.defaultNewAcl && this.defaultNewAcl.owner) || utils.CONSTS.SYSTEM_ADMIN_USER,
+                                ownerGroup:
+                                    (this.defaultNewAcl && this.defaultNewAcl.ownerGroup) ||
+                                    utils.CONSTS.SYSTEM_ADMIN_GROUP,
+                                permissions:
+                                    (this.defaultNewAcl && this.defaultNewAcl.file.permissions) ||
+                                    utils.CONSTS.ACCESS_USER_ALL |
+                                        utils.CONSTS.ACCESS_GROUP_ALL |
+                                        utils.CONSTS.ACCESS_EVERY_ALL // 777
                             }
                         };
                     }
@@ -447,18 +527,36 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                     if (!Buffer.isBuffer(fileData) && tools.isObject(fileData)) {
                         // if its an invalid object, stringify it and log warning
                         fileData = JSON.stringify(fileData);
-                        this.log.warn(`${namespaceLog} Data of "${id}/${name}" has invalid structure at file data request: ${fileData}`);
+                        this.log.warn(
+                            `${namespaceLog} Data of "${id}/${name}" has invalid structure at file data request: ${fileData}`
+                        );
                     }
                     handler.sendBufBulk(responseId, Buffer.from(fileData));
                 }
+            } else if (namespace === this.metaNamespace) {
+                // special handling for the primaryHost
+                if (id === 'objects.primaryHost') {
+                    // we are the server -> we are primary
+                    handler.sendString(this.settings.hostname);
+                } else {
+                    const result = this.getMeta(id);
+                    if (result === undefined || result === null) {
+                        handler.sendNull(responseId);
+                    } else {
+                        handler.sendBulk(responseId, result);
+                    }
+                }
             } else {
-                handler.sendError(responseId, new Error('GET-UNSUPPORTED for namespace ' + namespace + ': Data=' + JSON.stringify(data)));
+                handler.sendError(
+                    responseId,
+                    new Error('GET-UNSUPPORTED for namespace ' + namespace + ': Data=' + JSON.stringify(data))
+                );
             }
         });
 
         // Handle Redis "SET" requests
         handler.on('set', (data, responseId) => {
-            const {id, namespace, name, isMeta} = this._normalizeId(data[0]);
+            const { id, namespace, name, isMeta } = this._normalizeId(data[0]);
 
             if (namespace === this.namespaceObj) {
                 try {
@@ -480,10 +578,16 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                         // only set if the meta object is already/still existing
                         if (this.fileOptions[id]) {
                             this.fileOptions[id][name] = JSON.parse(data[1].toString('utf-8'));
-                            fs.writeFileSync(path.join(this.objectsDir, id, '_data.json'), JSON.stringify(this.fileOptions[id]));
+                            fs.writeFileSync(
+                                path.join(this.objectsDir, id, '_data.json'),
+                                JSON.stringify(this.fileOptions[id])
+                            );
                         }
                     } catch (err) {
-                        return void handler.sendError(responseId, new Error(`ERROR writeFile-Meta id=${id}: ${err.message}`));
+                        return void handler.sendError(
+                            responseId,
+                            new Error(`ERROR writeFile-Meta id=${id}: ${err.message}`)
+                        );
                     }
                     handler.sendString(responseId, 'OK');
                 } else {
@@ -491,12 +595,21 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                     try {
                         this._writeFile(id, name, data[1]);
                     } catch (err) {
-                        return void handler.sendError(responseId, new Error(`ERROR writeFile id=${id}: ${err.message}`));
+                        return void handler.sendError(
+                            responseId,
+                            new Error(`ERROR writeFile id=${id}: ${err.message}`)
+                        );
                     }
                     handler.sendString(responseId, 'OK');
                 }
+            } else if (namespace === this.metaNamespace) {
+                this.setMeta(id, data[1].toString('utf-8'));
+                handler.sendString(responseId, 'OK');
             } else {
-                handler.sendError(responseId, new Error(`SET-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`));
+                handler.sendError(
+                    responseId,
+                    new Error(`SET-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`)
+                );
             }
         });
 
@@ -507,7 +620,10 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
 
             if (oldDetails.namespace === this.namespaceFile) {
                 if (oldDetails.id !== newDetails.id) {
-                    return void handler.sendError(responseId, new Error('ERROR renameObject: id needs to stay the same'));
+                    return void handler.sendError(
+                        responseId,
+                        new Error('ERROR renameObject: id needs to stay the same')
+                    );
                 }
 
                 // Handle request for Meta data for files
@@ -523,13 +639,16 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                     handler.sendString(responseId, 'OK');
                 }
             } else {
-                handler.sendError(responseId, new Error(`RENAME-UNSUPPORTED for namespace ${oldDetails.namespace}: Data=${JSON.stringify(data)}`));
+                handler.sendError(
+                    responseId,
+                    new Error(`RENAME-UNSUPPORTED for namespace ${oldDetails.namespace}: Data=${JSON.stringify(data)}`)
+                );
             }
         });
 
         // Handle Redis "DEL" request for state and session namespace
         handler.on('del', (data, responseId) => {
-            const {id, namespace, name, isMeta} = this._normalizeId(data[0]);
+            const { id, namespace, name, isMeta } = this._normalizeId(data[0]);
 
             if (namespace === this.namespaceObj) {
                 try {
@@ -553,7 +672,10 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                     handler.sendString(responseId, 'OK');
                 }
             } else {
-                handler.sendError(responseId, new Error(`DEL-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`));
+                handler.sendError(
+                    responseId,
+                    new Error(`DEL-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`)
+                );
             }
         });
 
@@ -563,7 +685,7 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
             }
 
             // Note: we only simulate single key existence check
-            const {id, namespace, name} = this._normalizeId(data[0]);
+            const { id, namespace, name } = this._normalizeId(data[0]);
 
             if (namespace === this.namespaceObj) {
                 let exists;
@@ -581,6 +703,9 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                     return void handler.sendError(responseId, e);
                 }
                 handler.sendInteger(responseId, exists ? 1 : 0);
+            } else if (namespace === this.namespaceSet) {
+                // we are not using sets in simulator, so just say it exists
+                return void handler.sendInteger(responseId, 1);
             } else {
                 handler.sendError(responseId, new Error(`EXISTS-UNSUPPORTED for namespace ${namespace}`));
             }
@@ -604,33 +729,73 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
             return this._handleScanOrKeys(handler, data[0], responseId);
         });
 
+        // commands for redis SETS, just dummies
+        handler.on('sadd', (data, responseId) => {
+            return void handler.sendInteger(responseId, 1);
+        });
+
+        handler.on('srem', (data, responseId) => {
+            return void handler.sendInteger(responseId, 1);
+        });
+
+        handler.on('sscan', (data, responseId) => {
+            // for file DB it does the same as scan but data looks different
+            if (!data || data.length < 4) {
+                return void handler.sendArray(responseId, ['0', []]);
+            }
+
+            return this._handleScanOrKeys(handler, data[3], responseId, true);
+        });
+
         // Handle Redis "PSUBSCRIBE" request for state, log and session namespace
         handler.on('psubscribe', (data, responseId) => {
-            const {id, namespace} = this._normalizeId(data[0]);
+            const { id, namespace } = this._normalizeId(data[0]);
 
             if (namespace === this.namespaceObj) {
                 this._subscribeConfigForClient(handler, id);
                 handler.sendArray(responseId, ['psubscribe', data[0], 1]);
+            } else if (namespace === this.metaNamespace) {
+                this._subscribeMeta(handler, id);
+                handler.sendArray(responseId, ['psubscribe', data[0], 1]);
             } else {
-                handler.sendError(responseId, new Error('PSUBSCRIBE-UNSUPPORTED for namespace ' + namespace + ': Data=' + JSON.stringify(data)));
+                handler.sendError(
+                    responseId,
+                    new Error('PSUBSCRIBE-UNSUPPORTED for namespace ' + namespace + ': Data=' + JSON.stringify(data))
+                );
             }
         });
 
         // Handle Redis "UNSUBSCRIBE" request for state, log and session namespace
         handler.on('punsubscribe', (data, responseId) => {
-            const {id, namespace} = this._normalizeId(data[0]);
+            const { id, namespace } = this._normalizeId(data[0]);
 
             if (namespace === this.namespaceObj) {
                 this._unsubscribeConfigForClient(handler, id);
                 handler.sendArray(responseId, ['punsubscribe', data[0], 1]);
             } else {
-                handler.sendError(responseId, new Error('PUNSUBSCRIBE-UNSUPPORTED for namespace ' + namespace + ': Data=' + JSON.stringify(data)));
+                handler.sendError(
+                    responseId,
+                    new Error('PUNSUBSCRIBE-UNSUPPORTED for namespace ' + namespace + ': Data=' + JSON.stringify(data))
+                );
+            }
+        });
+
+        // Handle Redis "SUBSCRIBE" ... currently mainly ignored
+        handler.on('subscribe', (data, responseId) => {
+            if (data[0].startsWith('__keyevent@')) {
+                // we ignore these type of events because we publish expires anyway directly
+                handler.sendArray(responseId, ['subscribe', data[0], 1]);
+            } else {
+                handler.sendError(responseId, new Error(`SUBSCRIBE-UNSUPPORTED for ${data[0]}`));
             }
         });
 
         // Handle Redis "CONFIG" ... currently mainly ignored
         handler.on('config', (data, responseId) => {
-            if (data[0] === 'set' && data[1] === 'lua-time-limit') {
+            if (data[0] === 'set' && data[1] === 'notify-keyspace-events') {
+                // we ignore these type of commands for now, should only be to subscribe to keyspace events
+                handler.sendString(responseId, 'OK');
+            } else if (data[0] === 'set' && data[1] === 'lua-time-limit') {
                 // we ignore these type of commands for now, irrelevant
                 handler.sendString(responseId, 'OK');
             } else {
@@ -684,17 +849,19 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                 delete this.serverConnections[s];
             });
 
-            return /** @type {Promise<void>} */ (new Promise(resolve => {
-                if (!this.server) {
-                    return void resolve();
-                }
-                try {
-                    this.server.close(() => resolve());
-                } catch (e) {
-                    console.log(e.message);
-                    resolve();
-                }
-            }));
+            return /** @type {Promise<void>} */ (
+                new Promise(resolve => {
+                    if (!this.server) {
+                        return void resolve();
+                    }
+                    try {
+                        this.server.close(() => resolve());
+                    } catch (e) {
+                        console.log(e.message);
+                        resolve();
+                    }
+                })
+            );
         }
     }
 
@@ -708,13 +875,14 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
      * @private
      */
     _handleScanOrKeys(handler, pattern, responseId, isScan = false) {
-        const {id, namespace, name, isMeta} = this._normalizeId(pattern);
+        const { id, namespace, name, isMeta } = this._normalizeId(pattern);
 
         let response = [];
         if (namespace === this.namespaceObj || namespace === this.namespaceObjects) {
-            response =  this._getKeys(id).map(val => this.namespaceObj + val);
+            response = this._getKeys(id).map(val => this.namespaceObj + val);
             // if scan, we send the cursor as first argument
-            if (namespace !== this.namespaceObjects) { // When it was not the full DB namespace send out response
+            if (namespace !== this.namespaceObjects) {
+                // When it was not the full DB namespace send out response
                 return void handler.sendArray(responseId, isScan ? ['0', response] : response);
             }
         }
@@ -725,13 +893,15 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                 try {
                     res = this._readDir(id, name);
                     if (!res || !res.length) {
-                        res = [{
-                            file: '_data.json',
-                            stats: {},
-                            isDir: false,
-                            virtualFile: true,
-                            notExists: true
-                        }];
+                        res = [
+                            {
+                                file: '_data.json',
+                                stats: {},
+                                isDir: false,
+                                virtualFile: true,
+                                notExists: true
+                            }
+                        ];
                     }
                 } catch (err) {
                     if (!err.message.endsWith(utils.ERRORS.ERROR_NOT_FOUND)) {
@@ -758,11 +928,17 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
                     response.push(this.getFileId(entryId, baseName + arr.file, false));
                 });
                 handler.sendArray(responseId, isScan ? ['0', response] : response); // send out file or full db response
-            } else { // such a request should never happen
+            } else {
+                // such a request should never happen
                 handler.sendArray(responseId, isScan ? ['0', []] : []); // send out file or full db response
             }
+        } else if (namespace === this.namespaceSet) {
+            handler.sendArray(responseId, isScan ? ['0', []] : []); // send out empty array, we have no sets
         } else {
-            handler.sendError(responseId, new Error(`${isScan ? 'SCAN' : 'KEYS'}-UNSUPPORTED for namespace ${namespace}: Pattern=${pattern}`));
+            handler.sendError(
+                responseId,
+                new Error(`${isScan ? 'SCAN' : 'KEYS'}-UNSUPPORTED for namespace ${namespace}: Pattern=${pattern}`)
+            );
         }
     }
 
@@ -806,7 +982,12 @@ class ObjectsInMemoryServer extends ObjectsInMemoryJsonlDB {
             try {
                 this.server = net.createServer();
                 this.server.on('error', err =>
-                    this.log.info(`${this.namespace} ${settings.secure ? 'Secure ' : ''} Error inMem-objects listening on port ${settings.port || 9001}: ${err}`));
+                    this.log.info(
+                        `${this.namespace} ${settings.secure ? 'Secure ' : ''} Error inMem-objects listening on port ${
+                            settings.port || 9001
+                        }: ${err}`
+                    )
+                );
                 this.server.on('connection', socket => this._initSocket(socket));
 
                 this.server.listen(

package/package.json

@@ -1,17 +1,16 @@
 {
   "name": "@iobroker/db-objects-jsonl",
-  "version": "4.0.0-alpha.8-20210909-001a711c",
+  "version": "4.0.3",
   "engines": {
     "node": ">=12.0.0"
   },
   "dependencies": {
-    "@alcalzone/jsonl-db": "^2.1.0",
-    "@iobroker/db-base": "4.0.0-alpha.8-20210909-001a711c",
-    "@iobroker/db-objects-file": "4.0.0-alpha.8-20210909-001a711c",
-    "@iobroker/db-objects-redis": "4.0.0-alpha.8-20210909-001a711c",
+    "@alcalzone/jsonl-db": "~2.4.1",
+    "@iobroker/db-base": "4.0.3",
+    "@iobroker/db-objects-file": "4.0.3",
+    "@iobroker/db-objects-redis": "4.0.3",
     "deep-clone": "^3.0.3",
-    "fs-extra": "^10.0.0",
-    "node.extend": "^2.0.2"
+    "fs-extra": "^10.0.0"
   },
   "keywords": [
     "ioBroker",
@@ -38,5 +37,5 @@
     "lib/",
     "index.js"
   ],
-  "gitHead": "4c022b8b6845fdd4792fc0003dbfc740fef4a942"
+  "gitHead": "79a76a082db91399d0e432ef02bf2981a6739107"
 }