@autofleet/matmon 2.2.2-beta.2 → 2.3.1-beta-1

package/lib/cache.d.ts

@@ -18,7 +18,8 @@ interface GetMultipleWithCacheOptions<V = unknown> {
     setInCache: (key: string, value: V) => Promise<void>;
     getter?: (query: any) => Promise<V>;
     multiGetter?: (queries: any[]) => Promise<V[]>;
+    setMultiInCache?: (keyValues: Record<string, object>) => Promise<void>;
     idField?: string;
 }
-export declare const getMultipleWithCache: <V = unknown>({ getFromCache, multiGetterFromCache, setInCache, getter, multiGetter, idField, }: GetMultipleWithCacheOptions<V>) => (queries: any[]) => Promise<V[]>;
+export declare const getMultipleWithCache: <V = unknown>({ getFromCache, multiGetterFromCache, setInCache, setMultiInCache, getter, multiGetter, idField, }: GetMultipleWithCacheOptions<V>) => (queries: any[]) => Promise<V[]>;
 export {};

package/lib/cache.js

@@ -30,6 +30,7 @@ exports.getMultipleWithCache = exports.getWithCacheSupport = exports.getNewLRU =
 const lru_cache_1 = __importDefault(require("lru-cache"));
 const locking = __importStar(require("./locking"));
 const dotenv_1 = __importDefault(require("dotenv"));
+const logger_1 = __importDefault(require("./logger"));
 dotenv_1.default.config();
 const DEFAULT_CACHE_SIZE = 300;
 const MAX_SIZE = 100000;
@@ -52,10 +53,20 @@ const getNewLRU = (lifeTimeInSec, size) => new lru_cache_1.default(getOptions({
     size,
 }));
 exports.getNewLRU = getNewLRU;
+const IN_LOCAL_TEST = process.env.IS_IN_MATMON_TESTING === 'true';
+const IS_IN_SERVICE_TEST = process.env.NODE_ENV === 'test' && !IN_LOCAL_TEST;
+const tryToSetInCache = async (callback) => {
+    try {
+        await callback();
+    }
+    catch (e) {
+        logger_1.default.error('Failed to set in cache', e);
+    }
+};
 const getWithCacheSupport = async ({ cacheKey, cacheGet, cacheSet, fetching, skipCache, }) => {
-    if (skipCache || process.env.NODE_ENV === 'test') {
+    if (skipCache || IS_IN_SERVICE_TEST) {
         const res = await fetching();
-        await cacheSet(res);
+        await tryToSetInCache(() => cacheSet(res));
         return res;
     }
     let valueToReturn = null;
@@ -64,7 +75,7 @@ const getWithCacheSupport = async ({ cacheKey, cacheGet, cacheSet, fetching, ski
             valueToReturn = await cacheGet();
             if (!valueToReturn) {
                 valueToReturn = await fetching();
-                await cacheSet(valueToReturn);
+                await tryToSetInCache(() => cacheSet(valueToReturn));
             }
             else {
                 // logger.info('get value from cache');
@@ -75,7 +86,7 @@ const getWithCacheSupport = async ({ cacheKey, cacheGet, cacheSet, fetching, ski
     }
     catch (e) {
         valueToReturn = await fetching();
-        await cacheSet(valueToReturn);
+        await tryToSetInCache(() => cacheSet(valueToReturn));
         deleteMutexByCacheKey(cacheKey);
     }
     return valueToReturn;
@@ -87,7 +98,7 @@ const getIdField = (query, idField) => {
     }
     return query[idField];
 };
-const getMultipleWithCache = ({ getFromCache, multiGetterFromCache, setInCache, getter, multiGetter, idField = 'id', }) => async (queries) => {
+const getMultipleWithCache = ({ getFromCache, multiGetterFromCache, setInCache, setMultiInCache, getter, multiGetter, idField = 'id', }) => async (queries) => {
     const queriesMap = new Map(queries.filter(Boolean).map(query => [getIdField(query, idField), query]));
     const resultMap = new Map();
     const valuesToPullFromCache = [...queriesMap.values()];
@@ -98,13 +109,24 @@ const getMultipleWithCache = ({ getFromCache, multiGetterFromCache, setInCache,
         queriesMap.delete(getIdField(value, idField));
         resultMap.set(getIdField(value, idField), value);
     });
-    let valuesFromGetter = [];
     if (queriesMap.size > 0) {
         const valuesFromGetter = await (multiGetter?.([...queriesMap.values()]) || // Use multiGetter if it's provided
             Promise.all([...queriesMap.values()].map(id => getter(id))) // Otherwise, iterate over the queries with getter
         );
+        if (setMultiInCache) {
+            const setCacheObject = valuesFromGetter.reduce((acc, value) => {
+                acc[getIdField(value, idField)] = value;
+                return acc;
+            }, {});
+            await tryToSetInCache(() => setMultiInCache(setCacheObject));
+        }
         valuesFromGetter.forEach((value) => {
-            setInCache(value[idField], value);
+            if (!value) {
+                return;
+            }
+            if (!setMultiInCache) {
+                tryToSetInCache(() => setInCache(value[idField], value));
+            }
             resultMap.set(getIdField(value, idField), value);
         });
     }

package/lib/cache.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/cache.test.js

@@ -0,0 +1,172 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const cache_1 = require("./cache");
+const async_mutex_1 = require("async-mutex");
+jest.mock('./locking', () => ({
+    getMutex: jest.fn(() => new async_mutex_1.Mutex()),
+    wrapWithMutex: jest.fn((mutex, fn) => fn()),
+}));
+describe('Cache', () => {
+    describe('getNewLRU', () => {
+        it('should create a new LRU cache with default size', () => {
+            const cache = (0, cache_1.getNewLRU)(60);
+            expect(cache.max).toBe(300);
+        });
+        it('should create a new LRU cache with specified size', () => {
+            const cache = (0, cache_1.getNewLRU)(60, 500);
+            expect(cache.max).toBe(500);
+        });
+        it('should not exceed the maximum size', () => {
+            const cache = (0, cache_1.getNewLRU)(60, 200000);
+            expect(cache.max).toBe(100000);
+        });
+    });
+    describe('getWithCacheSupport', () => {
+        const cacheKey = 'testKey';
+        const createMocks = () => {
+            const cacheGet = jest.fn();
+            const cacheSet = jest.fn();
+            const fetching = jest.fn();
+            const getFromCache = jest.fn();
+            const setInCache = jest.fn();
+            const multiGetterFromCache = jest.fn();
+            const setMultiInCache = jest.fn();
+            return {
+                cacheGet,
+                cacheSet,
+                fetching,
+                getFromCache,
+                setInCache,
+                multiGetterFromCache,
+                setMultiInCache
+            };
+        };
+        beforeEach(() => {
+            jest.clearAllMocks();
+        });
+        it('should fetch and set value if skipCache is true', async () => {
+            const { cacheGet, cacheSet, fetching } = createMocks();
+            fetching.mockResolvedValue('fetchedValue');
+            await (0, cache_1.getWithCacheSupport)({ cacheKey, cacheGet, cacheSet, fetching, skipCache: true });
+            expect(fetching).toHaveBeenCalled();
+            expect(cacheSet).toHaveBeenCalledWith('fetchedValue');
+        });
+        it('should fetch and set value if cache is empty', async () => {
+            const { cacheGet, cacheSet, fetching } = createMocks();
+            cacheGet.mockResolvedValue(null);
+            fetching.mockResolvedValue('fetchedValue');
+            await (0, cache_1.getWithCacheSupport)({ cacheKey, cacheGet, cacheSet, fetching });
+            expect(fetching).toHaveBeenCalled();
+            expect(cacheSet).toHaveBeenCalledWith('fetchedValue');
+        });
+        it('should return cached value if available', async () => {
+            const { cacheGet, cacheSet, fetching } = createMocks();
+            cacheGet.mockResolvedValue('cachedValue');
+            const result = await (0, cache_1.getWithCacheSupport)({ cacheKey, cacheGet, cacheSet, fetching });
+            expect(result).toBe('cachedValue');
+            expect(fetching).not.toHaveBeenCalled();
+        });
+        it('should retry fetching if an error occurs', async () => {
+            const { cacheGet, cacheSet, fetching } = createMocks();
+            cacheGet.mockRejectedValue(new Error('Cache error'));
+            fetching.mockResolvedValue('fetchedValue');
+            const result = await (0, cache_1.getWithCacheSupport)({ cacheKey, cacheGet, cacheSet, fetching });
+            expect(result).toBe('fetchedValue');
+            expect(fetching).toHaveBeenCalled();
+            expect(cacheSet).toHaveBeenCalledWith('fetchedValue');
+        });
+        it('should not throw an error if cacheSet fails', async () => {
+            const { cacheGet, cacheSet, fetching } = createMocks();
+            cacheGet.mockResolvedValue(null);
+            cacheSet.mockRejectedValue(new Error('Cache set error'));
+            fetching.mockResolvedValue('fetchedValue');
+            await expect((0, cache_1.getWithCacheSupport)({ cacheKey, cacheGet, cacheSet, fetching })).resolves.not.toThrow();
+        });
+    });
+    describe('getMultipleWithCache', () => {
+        const buildMocks = () => {
+            const getFromCache = jest.fn();
+            const multiGetterFromCache = jest.fn();
+            const setInCache = jest.fn();
+            const setMultiInCache = jest.fn();
+            const getter = jest.fn();
+            const multiGetter = jest.fn();
+            return {
+                getFromCache,
+                multiGetterFromCache,
+                setInCache,
+                setMultiInCache,
+                getter,
+                multiGetter,
+            };
+        };
+        beforeEach(() => {
+            jest.clearAllMocks();
+        });
+        it('should return values from cache if available', async () => {
+            const { getFromCache, setInCache, getter } = buildMocks();
+            const queries = [{ id: 1 }, { id: 2 }];
+            getFromCache.mockResolvedValueOnce({ id: 1, value: 'cachedValue1' }).mockResolvedValueOnce({ id: 2, value: 'cachedValue2' });
+            const getMultiple = (0, cache_1.getMultipleWithCache)({ getFromCache, setInCache, getter });
+            const result = await getMultiple(queries);
+            expect(result.map(({ value }) => value)).toEqual(['cachedValue1', 'cachedValue2']);
+        });
+        it('should fetch and cache values if not in cache', async () => {
+            const { getFromCache, setInCache, getter } = buildMocks();
+            const queries = [{ id: 1 }, { id: 2 }];
+            getFromCache.mockResolvedValueOnce(null).mockResolvedValueOnce(null);
+            getter.mockResolvedValueOnce({ id: 1, value: 'fetchedValue1' }).mockResolvedValueOnce({ id: 2, value: 'fetchedValue2' });
+            const getMultiple = (0, cache_1.getMultipleWithCache)({ getFromCache, setInCache, getter });
+            const result = await getMultiple(queries);
+            expect(result.map(({ value }) => value)).toEqual(['fetchedValue1', 'fetchedValue2']);
+            expect(setInCache).toHaveBeenCalledWith(1, { id: 1, value: 'fetchedValue1' });
+            expect(setInCache).toHaveBeenCalledWith(2, { id: 2, value: 'fetchedValue2' });
+        });
+        it('should use multiGetterFromCache if provided', async () => {
+            const { multiGetterFromCache, setInCache, getter } = buildMocks();
+            const queries = [{ id: 1 }, { id: 2 }];
+            multiGetterFromCache.mockResolvedValue([{ id: 1, value: 'cachedValue1' }, { id: 2, value: 'cachedValue2' }]);
+            const getMultiple = (0, cache_1.getMultipleWithCache)({ multiGetterFromCache, setInCache, getter });
+            const result = await getMultiple(queries);
+            expect(result.map(({ value }) => value)).toEqual(['cachedValue1', 'cachedValue2']);
+        });
+        it('should use multiGetter if provided', async () => {
+            const { getFromCache, setInCache, multiGetter } = buildMocks();
+            const queries = [{ id: 1 }, { id: 2 }];
+            getFromCache.mockResolvedValueOnce(null).mockResolvedValueOnce(null);
+            multiGetter.mockResolvedValue([{ id: 1, value: 'fetchedValue1' }, { id: 2, value: 'fetchedValue2' }]);
+            const getMultiple = (0, cache_1.getMultipleWithCache)({ getFromCache, setInCache, multiGetter });
+            const result = await getMultiple(queries);
+            expect(result.map(({ value }) => value)).toEqual(['fetchedValue1', 'fetchedValue2']);
+            expect(setInCache).toHaveBeenCalledWith(1, { id: 1, value: 'fetchedValue1' });
+            expect(setInCache).toHaveBeenCalledWith(2, { id: 2, value: 'fetchedValue2' });
+        });
+        it('should use setMultiInCache if provided', async () => {
+            const { getFromCache, setInCache, multiGetter, setMultiInCache } = buildMocks();
+            const queries = [{ id: 1 }, { id: 2 }];
+            getFromCache.mockResolvedValueOnce(null).mockResolvedValueOnce(null);
+            multiGetter.mockResolvedValue([{ id: 1, value: 'fetchedValue1' }, { id: 2, value: 'fetchedValue2' }]);
+            const getMultiple = (0, cache_1.getMultipleWithCache)({ getFromCache, setInCache, multiGetter, setMultiInCache });
+            await getMultiple(queries);
+            expect(setMultiInCache).toHaveBeenCalledWith({ 1: { id: 1, value: 'fetchedValue1' }, 2: { id: 2, value: 'fetchedValue2' } });
+        });
+        it('Should not throw and error if setMultiInCache throws an error', async () => {
+            const { getFromCache, setInCache, multiGetter, setMultiInCache } = buildMocks();
+            const queries = [{ id: 1 }, { id: 2 }];
+            getFromCache.mockResolvedValueOnce(null).mockResolvedValueOnce(null);
+            multiGetter.mockResolvedValue([{ id: 1, value: 'fetchedValue1' }, { id: 2, value: 'fetchedValue2' }]);
+            setMultiInCache.mockRejectedValue(new Error('Error setting cache'));
+            const getMultiple = (0, cache_1.getMultipleWithCache)({ getFromCache, setInCache, multiGetter, setMultiInCache });
+            await expect(getMultiple(queries)).resolves.not.toThrow();
+        });
+        it('should not throw an error if setInCache fails', async () => {
+            const { getFromCache, setInCache, getter } = buildMocks();
+            const queries = [{ id: 1 }, { id: 2 }];
+            getFromCache.mockResolvedValueOnce(null).mockResolvedValueOnce(null);
+            getter.mockResolvedValueOnce({ id: 1, value: 'fetchedValue1' }).mockResolvedValueOnce({ id: 2, value: 'fetchedValue2' });
+            setInCache.mockRejectedValue(new Error('Error setting cache'));
+            const getMultiple = (0, cache_1.getMultipleWithCache)({ getFromCache, setInCache, getter });
+            await expect(getMultiple(queries)).resolves.not.toThrow();
+        });
+    });
+});

package/lib/redis/index.d.ts

@@ -3,17 +3,18 @@ declare class RedisCache {
     private locker;
     private lockTimeout;
     private lockDuration;
-    private locks;
+    locks: any;
     private baseTTL;
     private lockRetries;
-    private useLock;
+    keyPrefix: string;
+    useLock: boolean;
     constructor(options: any);
     get(key: any): Promise<any>;
-    getMultiple(keys: any, addPrefixToKeys?: boolean): Promise<any>;
+    getMultiple(keys: any): Promise<any>;
     set(key: any, value: any): Promise<void>;
+    setMultiple(keyValues: Record<string, any>): Promise<any>;
     remove(key: any): Promise<void>;
     removeMultiple(keys: any): Promise<void>;
-    flushAll(): Promise<void>;
     getClient(): any;
     private lock;
 }

package/lib/redis/index.js

@@ -5,11 +5,18 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const redis_1 = __importDefault(require("redis"));
 const redis_lock_1 = __importDefault(require("redis-lock"));
-const bluebird_1 = __importDefault(require("bluebird"));
 const util_1 = require("util");
 const errors_1 = require("./errors");
-bluebird_1.default.promisifyAll(redis_1.default.RedisClient.prototype);
-bluebird_1.default.promisifyAll(redis_1.default.Multi.prototype);
+const logger_1 = __importDefault(require("../logger"));
+const promisifyAll = (obj) => {
+    Object.keys(obj).forEach((key) => {
+        if (typeof obj[key] === 'function') {
+            obj[`${key}Async`] = (0, util_1.promisify)(obj[key]);
+        }
+    });
+};
+promisifyAll(redis_1.default.RedisClient.prototype);
+promisifyAll(redis_1.default.Multi.prototype);
 const { env } = process;
 const HOST = env.REDIS_HOST_NAME || '127.0.0.1';
 const PORT = env.REDIS_HOST_PORT || 6379;
@@ -33,6 +40,7 @@ class RedisCache {
         this.baseTTL = options.ttl ?? DEFAULT_BASE_TTL;
         this.locks = {};
         this.useLock = !!options.useLock;
+        this.keyPrefix = KEY_PREFIX;
     }
     async get(key) {
         const keyWithPrefix = KEY_PREFIX + key;
@@ -58,8 +66,8 @@ class RedisCache {
         }
         return JSON.parse(value);
     }
-    async getMultiple(keys, addPrefixToKeys = true) {
-        const keysWithPrefix = addPrefixToKeys ? keys.map(key => KEY_PREFIX + key) : keys;
+    async getMultiple(keys) {
+        const keysWithPrefix = keys.map(key => KEY_PREFIX + key);
         let values;
         try {
             if (keysWithPrefix.length === 0) {
@@ -87,6 +95,27 @@ class RedisCache {
             throw new errors_1.RedisCacheError('Failed to set a key-value pair', err);
         }
     }
+    async setMultiple(keyValues) {
+        if (typeof keyValues !== 'object') {
+            const error = new errors_1.RedisCacheError('keyValues must be an object', new Error('keyValues must be an object'));
+            logger_1.default.error('keyValues must be an object', { error });
+            throw error;
+        }
+        const keyValuesWithPrefix = Object.entries(keyValues).map(([key, value]) => [KEY_PREFIX + key, JSON.stringify(value)]);
+        const ttl = parseInt(String(this.baseTTL * (Math.random() + 1)), 10);
+        try {
+            const multi = this.client.multi();
+            const setPromise = multi.msetAsync(...keyValuesWithPrefix.flat());
+            keyValuesWithPrefix.map(([key]) => {
+                return multi.expireAsync(key, ttl);
+            });
+            await multi.exec();
+            return setPromise;
+        }
+        catch (err) {
+            throw new errors_1.RedisCacheError('Failed to set multiple key-value pairs', err);
+        }
+    }
     async remove(key) {
         const keyWithPrefix = KEY_PREFIX + key;
         try {
@@ -109,14 +138,6 @@ class RedisCache {
             throw new errors_1.RedisCacheError(`Failed to delete multiple keys ${keysWithPrefix.join('|')}`, err);
         }
     }
-    async flushAll() {
-        try {
-            await this.client.flushAll();
-        }
-        catch (err) {
-            throw new errors_1.RedisCacheError(`Failed to flush all keys`, err);
-        }
-    }
     getClient() {
         return this.client;
     }

package/lib/redis/index.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/redis/index.test.js

@@ -0,0 +1,65 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const redis_1 = __importDefault(require("redis"));
+const index_1 = __importDefault(require("./index"));
+const util_1 = require("util");
+const promisifyAll = (obj) => {
+    Object.keys(obj).forEach((key) => {
+        if (typeof obj[key] === 'function') {
+            obj[`${key}Async`] = (0, util_1.promisify)(obj[key]);
+        }
+    });
+};
+promisifyAll(redis_1.default.RedisClient.prototype);
+describe('RedisCache', () => {
+    let redisCache;
+    const testClient = redis_1.default.createClient();
+    beforeEach(() => {
+        redisCache = new index_1.default({});
+    });
+    afterEach(() => {
+        jest.clearAllMocks();
+    });
+    describe('setMultiple', () => {
+        it('should set multiple key-value pairs in redis', async () => {
+            const keyValues = { testKey1: 'testValue1', testKey2: 'testValue2' };
+            const keyValuesArray = Object.entries(keyValues);
+            await redisCache.setMultiple(keyValues);
+            const expectedKeyValues = keyValuesArray.map(([key, value]) => [`${redisCache.keyPrefix}${key}`, JSON.stringify(value)]);
+            await Promise.all(expectedKeyValues.map(async ([key, value]) => {
+                const valueInRedis = await testClient.getAsync(key);
+                expect(valueInRedis).toBe(value);
+            }));
+            // Check that the keys expire after a random amount of time.
+            const currentTTL = await testClient.ttlAsync(expectedKeyValues[0][0]);
+            expect(currentTTL).toBeGreaterThanOrEqual(0);
+        });
+    });
+    describe('set', () => {
+        it('should set a key-value pair in redis', async () => {
+            const key = 'testKey';
+            const value = 'testValue';
+            await redisCache.set(key, value);
+            const keyWithPrefix = `${redisCache.keyPrefix}${key}`;
+            const valueInRedis = await testClient.getAsync(keyWithPrefix);
+            expect(valueInRedis).toBe(JSON.stringify(value));
+            // Check that the key expires after a random amount of time.
+            const currentTTL = await testClient.ttlAsync(keyWithPrefix);
+            expect(currentTTL).toBeGreaterThanOrEqual(0);
+        });
+        it('should release the lock after setting a key-value pair', async () => {
+            const key = 'testKeyWithLock';
+            const value = 'testValueWithLock';
+            redisCache.useLock = true;
+            await redisCache.set(key, value);
+            const keyWithPrefix = `${redisCache.keyPrefix}${key}`;
+            const valueInRedis = await testClient.getAsync(keyWithPrefix);
+            expect(valueInRedis).toBe(JSON.stringify(value));
+            // Check that the lock is released
+            expect(redisCache.locks[keyWithPrefix]).toBeUndefined();
+        });
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autofleet/matmon",
-  "version": "2.2.2-beta.2",
+  "version": "2.3.1-beta-1",
   "description": "manage cache",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
@@ -12,10 +12,6 @@
     "test-auto": "jest --watch --runInBand",
     "linter": "./node_modules/.bin/eslint ."
   },
-  "jest": {
-    "setupTestFrameworkScriptFile": "jest-extended",
-    "testURL": "http://localhost:8085/"
-  },
   "repository": {
     "type": "git",
     "url": "git+ssh://git@gitlab.com/AutoFleet/matmon.git"
@@ -27,10 +23,9 @@
   },
   "homepage": "https://github.com/Autofleet/matmon",
   "dependencies": {
-    "@autofleet/logger": "4.0.6",
+    "@autofleet/logger": "^4",
     "@types/node": "^14.14.20",
     "async-mutex": "^0.2.6",
-    "bluebird": "^3.7.2",
     "dotenv": "^8.2.0",
     "lru-cache": "^6.0.0",
     "redis": "^3.0.2",
@@ -40,8 +35,12 @@
   },
   "devDependencies": {
     "@autofleet/logger": "^4.0.3",
+    "@types/jest": "^29.5.14",
     "@types/lru-cache": "^5.1.1",
     "@types/node": "^14.14.20",
+    "@types/redis": "^4.0.11",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.2.5",
     "typescript": "^5.5.2"
   },
   "peerDependencies": {