@iobroker/db-objects-jsonl 4.0.21 → 4.0.24

package/lib/objects/objectsInMemJsonlDB.js

@@ -20,6 +20,71 @@ const fs = require('fs');
 const os = require('os');
 const { tools } = require('@iobroker/js-controller-common');
 
+/**
+ * Normalizes options for the JsonlDB
+ * @param {Record<string, any> | undefined} conf The jsonlOptions options from iobroker.json
+ * @returns {import("@alcalzone/jsonl-db").JsonlDBOptions<any>}
+ */
+function normalizeJsonlOptions(conf = {}) {
+    /** @type {import("@alcalzone/jsonl-db").JsonlDBOptions<any>} */
+    const ret = {
+        autoCompress: {
+            sizeFactor: 2,
+            sizeFactorMinimumSize: 25000,
+            // Compress at least daily to avoid a huge file when DBs have few objects
+            // but big objects are updated regularly (e.g. the repositories)
+            intervalMs: 1000 * 60 * 60 * 23
+        },
+        ignoreReadErrors: true,
+        throttleFS: {
+            intervalMs: 60000,
+            maxBufferedCommands: 1000
+        },
+        lockfile: {
+            // 5 retries starting at 250ms add up to just above 2s,
+            // so the DB has 3 more seconds to load all data if it wants to stay within the 5s connectionTimeout
+            retries: 5,
+            retryMinTimeoutMs: 250,
+            // This makes sure the DB stays locked for maximum 2s even if the process crashes
+            staleMs: 2000
+        }
+    };
+
+    // Be really careful what we allow here. Incorrect settings may cause problems in production.
+    if (tools.isObject(conf.autoCompress)) {
+        const ac = conf.autoCompress;
+        // Letting the DB grow more than 100x is risky
+        if (typeof ac.sizeFactor === 'number' && ac.sizeFactor >= 2 && ac.sizeFactor <= 100) {
+            ret.autoCompress.sizeFactor = ac.sizeFactor;
+        }
+        // Also we should definitely compress once the DB has reached 100k lines or it might grow too big
+        if (
+            typeof ac.sizeFactorMinimumSize === 'number' &&
+            ac.sizeFactorMinimumSize >= 0 &&
+            ac.sizeFactorMinimumSize <= 100000
+        ) {
+            ret.autoCompress.sizeFactorMinimumSize = ac.sizeFactorMinimumSize;
+        }
+    }
+    if (tools.isObject(conf.throttleFS)) {
+        const thr = conf.throttleFS;
+        // Don't write more often than every second and write at least once every hour
+        if (typeof thr.intervalMs === 'number' && thr.intervalMs >= 1000 && thr.intervalMs <= 3600000) {
+            ret.throttleFS.intervalMs = thr.intervalMs;
+        }
+        // Don't keep too much in memory - 100k changes are more than enough
+        if (
+            typeof thr.maxBufferedCommands === 'number' &&
+            thr.maxBufferedCommands >= 0 &&
+            thr.maxBufferedCommands <= 100000
+        ) {
+            ret.throttleFS.maxBufferedCommands = thr.maxBufferedCommands;
+        }
+    }
+
+    return ret;
+}
+
 /**
  * This class inherits InMemoryFileDB class and adds all relevant logic for objects
  * including the available methods for use by js-controller directly
@@ -32,26 +97,7 @@ class ObjectsInMemoryJsonlDB extends ObjectsInMemoryFileDB {
             backupDirName: 'backup-objects'
         };
 
-        /** @type {import("@alcalzone/jsonl-db").JsonlDBOptions<any>} */
-        const jsonlOptions = settings.connection.jsonlOptions || {
-            autoCompress: {
-                sizeFactor: 2,
-                sizeFactorMinimumSize: 25000
-            },
-            ignoreReadErrors: true,
-            throttleFS: {
-                intervalMs: 60000,
-                maxBufferedCommands: 1000
-            },
-            lockfile: {
-                // 5 retries starting at 250ms add up to just above 2s,
-                // so the DB has 3 more seconds to load all data if it wants to stay within the 5s connectionTimeout
-                retries: 5,
-                retryMinTimeoutMs: 250,
-                // This makes sure the DB stays locked for maximum 2s even if the process crashes
-                staleMs: 2000
-            }
-        };
+        const jsonlOptions = normalizeJsonlOptions(settings.connection.jsonlOptions);
         settings.jsonlDB = {
             fileName: 'objects.jsonl'
         };

package/package.json

@@ -1,14 +1,14 @@
 {
   "name": "@iobroker/db-objects-jsonl",
-  "version": "4.0.21",
+  "version": "4.0.24",
   "engines": {
     "node": ">=12.0.0"
   },
   "dependencies": {
     "@alcalzone/jsonl-db": "~2.5.1",
-    "@iobroker/db-base": "4.0.21",
-    "@iobroker/db-objects-file": "4.0.21",
-    "@iobroker/db-objects-redis": "4.0.21",
+    "@iobroker/db-base": "4.0.24",
+    "@iobroker/db-objects-file": "4.0.24",
+    "@iobroker/db-objects-redis": "4.0.24",
     "deep-clone": "^3.0.3",
     "fs-extra": "^10.0.0"
   },
@@ -37,5 +37,5 @@
     "lib/",
     "index.js"
   ],
-  "gitHead": "4749d256d312623ae7621e7ae47d44580b3aa8c1"
+  "gitHead": "5645debf3394177abeef58917f872a7d8c43e320"
 }