@autofleet/matmon 2.3.0 → 2.3.1-beta-1

package/lib/cache.d.ts

@@ -0,0 +1,25 @@
+import LRU from 'lru-cache';
+interface Options {
+    lifeTimeInSec: number;
+    size?: number;
+}
+export declare const getNewLRU: <K = unknown, V = unknown>(lifeTimeInSec: Options["lifeTimeInSec"], size?: Options["size"]) => LRU<K, V>;
+interface GetWithCacheOptions<V = unknown> {
+    cacheKey: string;
+    cacheGet: () => Promise<V>;
+    cacheSet: (value: V) => Promise<void>;
+    fetching: () => Promise<V>;
+    skipCache?: boolean;
+}
+export declare const getWithCacheSupport: <V = unknown>({ cacheKey, cacheGet, cacheSet, fetching, skipCache, }: GetWithCacheOptions<V>) => Promise<any>;
+interface GetMultipleWithCacheOptions<V = unknown> {
+    getFromCache?: (query: any) => Promise<V>;
+    multiGetterFromCache?: (queries: any[]) => Promise<V[]>;
+    setInCache: (key: string, value: V) => Promise<void>;
+    getter?: (query: any) => Promise<V>;
+    multiGetter?: (queries: any[]) => Promise<V[]>;
+    setMultiInCache?: (keyValues: Record<string, object>) => Promise<void>;
+    idField?: string;
+}
+export declare const getMultipleWithCache: <V = unknown>({ getFromCache, multiGetterFromCache, setInCache, setMultiInCache, getter, multiGetter, idField, }: GetMultipleWithCacheOptions<V>) => (queries: any[]) => Promise<V[]>;
+export {};

package/lib/cache.js

@@ -0,0 +1,140 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getMultipleWithCache = exports.getWithCacheSupport = exports.getNewLRU = void 0;
+const lru_cache_1 = __importDefault(require("lru-cache"));
+const locking = __importStar(require("./locking"));
+const dotenv_1 = __importDefault(require("dotenv"));
+const logger_1 = __importDefault(require("./logger"));
+dotenv_1.default.config();
+const DEFAULT_CACHE_SIZE = 300;
+const MAX_SIZE = 100000;
+const MUTEX_MAP = {};
+const getOptions = ({ lifeTimeInSec, size = DEFAULT_CACHE_SIZE, }) => ({
+    max: Math.min(size, MAX_SIZE),
+    maxAge: process.env.NODE_ENV !== 'test' ? 1000 * lifeTimeInSec : 0,
+});
+const getMutexByCacheKey = (key) => {
+    MUTEX_MAP[key] || (MUTEX_MAP[key] = locking.getMutex());
+    return MUTEX_MAP[key];
+};
+const deleteMutexByCacheKey = (key) => {
+    if (MUTEX_MAP[key]) {
+        delete MUTEX_MAP[key];
+    }
+};
+const getNewLRU = (lifeTimeInSec, size) => new lru_cache_1.default(getOptions({
+    lifeTimeInSec,
+    size,
+}));
+exports.getNewLRU = getNewLRU;
+const IN_LOCAL_TEST = process.env.IS_IN_MATMON_TESTING === 'true';
+const IS_IN_SERVICE_TEST = process.env.NODE_ENV === 'test' && !IN_LOCAL_TEST;
+const tryToSetInCache = async (callback) => {
+    try {
+        await callback();
+    }
+    catch (e) {
+        logger_1.default.error('Failed to set in cache', e);
+    }
+};
+const getWithCacheSupport = async ({ cacheKey, cacheGet, cacheSet, fetching, skipCache, }) => {
+    if (skipCache || IS_IN_SERVICE_TEST) {
+        const res = await fetching();
+        await tryToSetInCache(() => cacheSet(res));
+        return res;
+    }
+    let valueToReturn = null;
+    try {
+        await locking.wrapWithMutex(getMutexByCacheKey(cacheKey), async () => {
+            valueToReturn = await cacheGet();
+            if (!valueToReturn) {
+                valueToReturn = await fetching();
+                await tryToSetInCache(() => cacheSet(valueToReturn));
+            }
+            else {
+                // logger.info('get value from cache');
+            }
+            deleteMutexByCacheKey(cacheKey);
+        });
+        // retry without locking if failed
+    }
+    catch (e) {
+        valueToReturn = await fetching();
+        await tryToSetInCache(() => cacheSet(valueToReturn));
+        deleteMutexByCacheKey(cacheKey);
+    }
+    return valueToReturn;
+};
+exports.getWithCacheSupport = getWithCacheSupport;
+const getIdField = (query, idField) => {
+    if (typeof query === 'string') {
+        return query;
+    }
+    return query[idField];
+};
+const getMultipleWithCache = ({ getFromCache, multiGetterFromCache, setInCache, setMultiInCache, getter, multiGetter, idField = 'id', }) => async (queries) => {
+    const queriesMap = new Map(queries.filter(Boolean).map(query => [getIdField(query, idField), query]));
+    const resultMap = new Map();
+    const valuesToPullFromCache = [...queriesMap.values()];
+    const valuesFromCache = await (multiGetterFromCache?.(valuesToPullFromCache) || // Use multiGetterFromCache if it's provided
+        Promise.all(valuesToPullFromCache.map(query => getFromCache(query))) // Otherwise, iterate over the queries with getFromCache
+    );
+    valuesFromCache.filter(Boolean).forEach((value) => {
+        queriesMap.delete(getIdField(value, idField));
+        resultMap.set(getIdField(value, idField), value);
+    });
+    if (queriesMap.size > 0) {
+        const valuesFromGetter = await (multiGetter?.([...queriesMap.values()]) || // Use multiGetter if it's provided
+            Promise.all([...queriesMap.values()].map(id => getter(id))) // Otherwise, iterate over the queries with getter
+        );
+        if (setMultiInCache) {
+            const setCacheObject = valuesFromGetter.reduce((acc, value) => {
+                acc[getIdField(value, idField)] = value;
+                return acc;
+            }, {});
+            await tryToSetInCache(() => setMultiInCache(setCacheObject));
+        }
+        valuesFromGetter.forEach((value) => {
+            if (!value) {
+                return;
+            }
+            if (!setMultiInCache) {
+                tryToSetInCache(() => setInCache(value[idField], value));
+            }
+            resultMap.set(getIdField(value, idField), value);
+        });
+    }
+    return queries.map((query) => {
+        if (!query) {
+            return null;
+        }
+        return resultMap.get(getIdField(query, idField));
+    });
+};
+exports.getMultipleWithCache = getMultipleWithCache;

package/lib/cache.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/cache.test.js

@@ -0,0 +1,172 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const cache_1 = require("./cache");
+const async_mutex_1 = require("async-mutex");
+jest.mock('./locking', () => ({
+    getMutex: jest.fn(() => new async_mutex_1.Mutex()),
+    wrapWithMutex: jest.fn((mutex, fn) => fn()),
+}));
+describe('Cache', () => {
+    describe('getNewLRU', () => {
+        it('should create a new LRU cache with default size', () => {
+            const cache = (0, cache_1.getNewLRU)(60);
+            expect(cache.max).toBe(300);
+        });
+        it('should create a new LRU cache with specified size', () => {
+            const cache = (0, cache_1.getNewLRU)(60, 500);
+            expect(cache.max).toBe(500);
+        });
+        it('should not exceed the maximum size', () => {
+            const cache = (0, cache_1.getNewLRU)(60, 200000);
+            expect(cache.max).toBe(100000);
+        });
+    });
+    describe('getWithCacheSupport', () => {
+        const cacheKey = 'testKey';
+        const createMocks = () => {
+            const cacheGet = jest.fn();
+            const cacheSet = jest.fn();
+            const fetching = jest.fn();
+            const getFromCache = jest.fn();
+            const setInCache = jest.fn();
+            const multiGetterFromCache = jest.fn();
+            const setMultiInCache = jest.fn();
+            return {
+                cacheGet,
+                cacheSet,
+                fetching,
+                getFromCache,
+                setInCache,
+                multiGetterFromCache,
+                setMultiInCache
+            };
+        };
+        beforeEach(() => {
+            jest.clearAllMocks();
+        });
+        it('should fetch and set value if skipCache is true', async () => {
+            const { cacheGet, cacheSet, fetching } = createMocks();
+            fetching.mockResolvedValue('fetchedValue');
+            await (0, cache_1.getWithCacheSupport)({ cacheKey, cacheGet, cacheSet, fetching, skipCache: true });
+            expect(fetching).toHaveBeenCalled();
+            expect(cacheSet).toHaveBeenCalledWith('fetchedValue');
+        });
+        it('should fetch and set value if cache is empty', async () => {
+            const { cacheGet, cacheSet, fetching } = createMocks();
+            cacheGet.mockResolvedValue(null);
+            fetching.mockResolvedValue('fetchedValue');
+            await (0, cache_1.getWithCacheSupport)({ cacheKey, cacheGet, cacheSet, fetching });
+            expect(fetching).toHaveBeenCalled();
+            expect(cacheSet).toHaveBeenCalledWith('fetchedValue');
+        });
+        it('should return cached value if available', async () => {
+            const { cacheGet, cacheSet, fetching } = createMocks();
+            cacheGet.mockResolvedValue('cachedValue');
+            const result = await (0, cache_1.getWithCacheSupport)({ cacheKey, cacheGet, cacheSet, fetching });
+            expect(result).toBe('cachedValue');
+            expect(fetching).not.toHaveBeenCalled();
+        });
+        it('should retry fetching if an error occurs', async () => {
+            const { cacheGet, cacheSet, fetching } = createMocks();
+            cacheGet.mockRejectedValue(new Error('Cache error'));
+            fetching.mockResolvedValue('fetchedValue');
+            const result = await (0, cache_1.getWithCacheSupport)({ cacheKey, cacheGet, cacheSet, fetching });
+            expect(result).toBe('fetchedValue');
+            expect(fetching).toHaveBeenCalled();
+            expect(cacheSet).toHaveBeenCalledWith('fetchedValue');
+        });
+        it('should not throw an error if cacheSet fails', async () => {
+            const { cacheGet, cacheSet, fetching } = createMocks();
+            cacheGet.mockResolvedValue(null);
+            cacheSet.mockRejectedValue(new Error('Cache set error'));
+            fetching.mockResolvedValue('fetchedValue');
+            await expect((0, cache_1.getWithCacheSupport)({ cacheKey, cacheGet, cacheSet, fetching })).resolves.not.toThrow();
+        });
+    });
+    describe('getMultipleWithCache', () => {
+        const buildMocks = () => {
+            const getFromCache = jest.fn();
+            const multiGetterFromCache = jest.fn();
+            const setInCache = jest.fn();
+            const setMultiInCache = jest.fn();
+            const getter = jest.fn();
+            const multiGetter = jest.fn();
+            return {
+                getFromCache,
+                multiGetterFromCache,
+                setInCache,
+                setMultiInCache,
+                getter,
+                multiGetter,
+            };
+        };
+        beforeEach(() => {
+            jest.clearAllMocks();
+        });
+        it('should return values from cache if available', async () => {
+            const { getFromCache, setInCache, getter } = buildMocks();
+            const queries = [{ id: 1 }, { id: 2 }];
+            getFromCache.mockResolvedValueOnce({ id: 1, value: 'cachedValue1' }).mockResolvedValueOnce({ id: 2, value: 'cachedValue2' });
+            const getMultiple = (0, cache_1.getMultipleWithCache)({ getFromCache, setInCache, getter });
+            const result = await getMultiple(queries);
+            expect(result.map(({ value }) => value)).toEqual(['cachedValue1', 'cachedValue2']);
+        });
+        it('should fetch and cache values if not in cache', async () => {
+            const { getFromCache, setInCache, getter } = buildMocks();
+            const queries = [{ id: 1 }, { id: 2 }];
+            getFromCache.mockResolvedValueOnce(null).mockResolvedValueOnce(null);
+            getter.mockResolvedValueOnce({ id: 1, value: 'fetchedValue1' }).mockResolvedValueOnce({ id: 2, value: 'fetchedValue2' });
+            const getMultiple = (0, cache_1.getMultipleWithCache)({ getFromCache, setInCache, getter });
+            const result = await getMultiple(queries);
+            expect(result.map(({ value }) => value)).toEqual(['fetchedValue1', 'fetchedValue2']);
+            expect(setInCache).toHaveBeenCalledWith(1, { id: 1, value: 'fetchedValue1' });
+            expect(setInCache).toHaveBeenCalledWith(2, { id: 2, value: 'fetchedValue2' });
+        });
+        it('should use multiGetterFromCache if provided', async () => {
+            const { multiGetterFromCache, setInCache, getter } = buildMocks();
+            const queries = [{ id: 1 }, { id: 2 }];
+            multiGetterFromCache.mockResolvedValue([{ id: 1, value: 'cachedValue1' }, { id: 2, value: 'cachedValue2' }]);
+            const getMultiple = (0, cache_1.getMultipleWithCache)({ multiGetterFromCache, setInCache, getter });
+            const result = await getMultiple(queries);
+            expect(result.map(({ value }) => value)).toEqual(['cachedValue1', 'cachedValue2']);
+        });
+        it('should use multiGetter if provided', async () => {
+            const { getFromCache, setInCache, multiGetter } = buildMocks();
+            const queries = [{ id: 1 }, { id: 2 }];
+            getFromCache.mockResolvedValueOnce(null).mockResolvedValueOnce(null);
+            multiGetter.mockResolvedValue([{ id: 1, value: 'fetchedValue1' }, { id: 2, value: 'fetchedValue2' }]);
+            const getMultiple = (0, cache_1.getMultipleWithCache)({ getFromCache, setInCache, multiGetter });
+            const result = await getMultiple(queries);
+            expect(result.map(({ value }) => value)).toEqual(['fetchedValue1', 'fetchedValue2']);
+            expect(setInCache).toHaveBeenCalledWith(1, { id: 1, value: 'fetchedValue1' });
+            expect(setInCache).toHaveBeenCalledWith(2, { id: 2, value: 'fetchedValue2' });
+        });
+        it('should use setMultiInCache if provided', async () => {
+            const { getFromCache, setInCache, multiGetter, setMultiInCache } = buildMocks();
+            const queries = [{ id: 1 }, { id: 2 }];
+            getFromCache.mockResolvedValueOnce(null).mockResolvedValueOnce(null);
+            multiGetter.mockResolvedValue([{ id: 1, value: 'fetchedValue1' }, { id: 2, value: 'fetchedValue2' }]);
+            const getMultiple = (0, cache_1.getMultipleWithCache)({ getFromCache, setInCache, multiGetter, setMultiInCache });
+            await getMultiple(queries);
+            expect(setMultiInCache).toHaveBeenCalledWith({ 1: { id: 1, value: 'fetchedValue1' }, 2: { id: 2, value: 'fetchedValue2' } });
+        });
+        it('Should not throw and error if setMultiInCache throws an error', async () => {
+            const { getFromCache, setInCache, multiGetter, setMultiInCache } = buildMocks();
+            const queries = [{ id: 1 }, { id: 2 }];
+            getFromCache.mockResolvedValueOnce(null).mockResolvedValueOnce(null);
+            multiGetter.mockResolvedValue([{ id: 1, value: 'fetchedValue1' }, { id: 2, value: 'fetchedValue2' }]);
+            setMultiInCache.mockRejectedValue(new Error('Error setting cache'));
+            const getMultiple = (0, cache_1.getMultipleWithCache)({ getFromCache, setInCache, multiGetter, setMultiInCache });
+            await expect(getMultiple(queries)).resolves.not.toThrow();
+        });
+        it('should not throw an error if setInCache fails', async () => {
+            const { getFromCache, setInCache, getter } = buildMocks();
+            const queries = [{ id: 1 }, { id: 2 }];
+            getFromCache.mockResolvedValueOnce(null).mockResolvedValueOnce(null);
+            getter.mockResolvedValueOnce({ id: 1, value: 'fetchedValue1' }).mockResolvedValueOnce({ id: 2, value: 'fetchedValue2' });
+            setInCache.mockRejectedValue(new Error('Error setting cache'));
+            const getMultiple = (0, cache_1.getMultipleWithCache)({ getFromCache, setInCache, getter });
+            await expect(getMultiple(queries)).resolves.not.toThrow();
+        });
+    });
+});

package/lib/index.d.ts

@@ -0,0 +1,4 @@
+import { getNewLRU, getWithCacheSupport, getMultipleWithCache } from './cache';
+import RedisCache from './redis';
+import { ORMCache, ORMTypes } from './orm-cache';
+export { getNewLRU, getWithCacheSupport, getMultipleWithCache, RedisCache, ORMCache, ORMTypes };

package/lib/index.js

@@ -0,0 +1,15 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ORMTypes = exports.ORMCache = exports.RedisCache = exports.getMultipleWithCache = exports.getWithCacheSupport = exports.getNewLRU = void 0;
+const cache_1 = require("./cache");
+Object.defineProperty(exports, "getNewLRU", { enumerable: true, get: function () { return cache_1.getNewLRU; } });
+Object.defineProperty(exports, "getWithCacheSupport", { enumerable: true, get: function () { return cache_1.getWithCacheSupport; } });
+Object.defineProperty(exports, "getMultipleWithCache", { enumerable: true, get: function () { return cache_1.getMultipleWithCache; } });
+const redis_1 = __importDefault(require("./redis"));
+exports.RedisCache = redis_1.default;
+const orm_cache_1 = require("./orm-cache");
+Object.defineProperty(exports, "ORMCache", { enumerable: true, get: function () { return orm_cache_1.ORMCache; } });
+Object.defineProperty(exports, "ORMTypes", { enumerable: true, get: function () { return orm_cache_1.ORMTypes; } });

package/lib/locking.d.ts

@@ -0,0 +1,3 @@
+import { MutexInterface } from 'async-mutex';
+export declare const wrapWithMutex: (mutex: MutexInterface, funcToRun: () => PromiseLike<void>) => Promise<void>;
+export declare const getMutex: () => MutexInterface;

package/lib/locking.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getMutex = exports.wrapWithMutex = void 0;
+const async_mutex_1 = require("async-mutex");
+const CACHE_LOCK_TIMEOUT_MILIS = 3000;
+const LOCK_TIMEOUT_MESSAGE = 'mutex - locking timeout';
+const wrapWithMutex = async (mutex, funcToRun) => {
+    const release = await mutex.acquire();
+    try {
+        await funcToRun();
+    }
+    finally {
+        release();
+    }
+};
+exports.wrapWithMutex = wrapWithMutex;
+const getMutex = () => (0, async_mutex_1.withTimeout)(new async_mutex_1.Mutex(), CACHE_LOCK_TIMEOUT_MILIS, new Error(LOCK_TIMEOUT_MESSAGE));
+exports.getMutex = getMutex;

package/lib/logger.d.ts

@@ -0,0 +1,2 @@
+declare const logger: import("@autofleet/logger").LoggerInstanceManager;
+export default logger;

package/lib/logger.js

@@ -0,0 +1,8 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const logger_1 = __importDefault(require("@autofleet/logger"));
+const logger = (0, logger_1.default)();
+exports.default = logger;

package/lib/orm-cache/adapter.d.ts

@@ -0,0 +1,17 @@
+import type RedisCache from '../redis';
+export interface ModelOptions {
+    name: string;
+    associations?: AssociationOptions[];
+}
+export interface AssociationOptions {
+    name: string;
+    alias: string;
+    accessKey?: string;
+    innerAssociation?: AssociationOptions;
+}
+export interface Adapter {
+    ormInstance: any;
+    getModel: (modelName: string) => any;
+    injectGetWithCacheFunction: (cache: RedisCache, modelOptions: ModelOptions) => void;
+    addInvalidationHooks: (cache: RedisCache, modelOptions: ModelOptions) => void;
+}

package/lib/orm-cache/adapter.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/lib/orm-cache/errors.d.ts

@@ -0,0 +1,2 @@
+export declare class UnsupportedOrmTypeError extends Error {
+}

package/lib/orm-cache/errors.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UnsupportedOrmTypeError = void 0;
+class UnsupportedOrmTypeError extends Error {
+}
+exports.UnsupportedOrmTypeError = UnsupportedOrmTypeError;

package/lib/orm-cache/index.d.ts

@@ -0,0 +1,12 @@
+import { ModelOptions } from './adapter';
+export declare enum ORMTypes {
+    SEQUELIZE = "sequelize"
+}
+interface ORMCacheOptions {
+    type: ORMTypes;
+    models: ModelOptions[];
+    ormInstance: any;
+    debug: boolean;
+}
+export declare const ORMCache: (options: ORMCacheOptions) => void;
+export {};

package/lib/orm-cache/index.js

@@ -0,0 +1,39 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ORMCache = exports.ORMTypes = void 0;
+const sequelize_adapter_1 = __importDefault(require("./sequelize-adapter"));
+const errors_1 = require("./errors");
+const redis_1 = __importDefault(require("../redis"));
+const logger_1 = __importDefault(require("../logger"));
+var ORMTypes;
+(function (ORMTypes) {
+    ORMTypes["SEQUELIZE"] = "sequelize";
+})(ORMTypes || (exports.ORMTypes = ORMTypes = {}));
+const ORMInstanceFactory = (options) => {
+    switch (options.type) {
+        case ORMTypes.SEQUELIZE: {
+            return new sequelize_adapter_1.default(options.ormInstance, options.debug);
+        }
+        default: {
+            throw new errors_1.UnsupportedOrmTypeError(`ORM type ${options.type} is unsupported at the moment`);
+        }
+    }
+};
+const ORMCache = (options) => {
+    const { models } = options;
+    logger_1.default.info('Starting ORM Cache', { options });
+    const adapter = ORMInstanceFactory(options);
+    const cache = new redis_1.default({
+        host: process.env.REDIS_HOST,
+        port: process.env.REDIS_PORT,
+    });
+    // eslint-disable-next-line array-callback-return
+    models.map((modelOptions) => {
+        adapter.addInvalidationHooks(cache, modelOptions);
+        adapter.injectGetWithCacheFunction(cache, modelOptions);
+    });
+};
+exports.ORMCache = ORMCache;

package/lib/orm-cache/sequelize-adapter.d.ts

@@ -0,0 +1,13 @@
+import { Sequelize } from 'sequelize';
+import { Adapter, AssociationOptions, ModelOptions } from './adapter';
+import RedisCache from '../redis';
+export default class SequelizeAdapter implements Adapter {
+    ormInstance: Sequelize;
+    debugMode: boolean;
+    constructor(sequelize: Sequelize, debug: any);
+    getModel(modelName: string): any;
+    getModelDependencies(modelName: string): AssociationOptions[];
+    debug(message: any, payload: any): void;
+    injectGetWithCacheFunction(cache: RedisCache, modelOptions: ModelOptions): void;
+    addInvalidationHooks(cache: RedisCache, modelOptions: ModelOptions): void;
+}

package/lib/orm-cache/sequelize-adapter.js

@@ -0,0 +1,156 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const logger_1 = __importDefault(require("../logger"));
+const util_1 = require("util");
+const { AF_SERVICE_NAME } = process.env;
+const ORM_CACHE_PREFIX = 'ormCache';
+const INVALIDATION_HOOKS = ['afterSave', 'afterUpdate', 'afterDestroy'];
+const BULK_HOOKS = ['beforeBulkUpdate', 'beforeBulkDestroy'];
+const generateInstanceKey = (name, id) => `${AF_SERVICE_NAME}:${ORM_CACHE_PREFIX}:${name}_${id}`;
+const generateDependencyKey = (modelName, associationName, associationId) => `${AF_SERVICE_NAME}:${ORM_CACHE_PREFIX}:${modelName}_${associationName}_${associationId}_DEPENDENCIES`;
+const getInstanceDependencyKeys = (modelOptions, instance) => {
+    const keys = modelOptions.associations
+        .filter((associationOptions) => instance[associationOptions.alias])
+        .map((associationOptions) => {
+        const accessKey = associationOptions.accessKey;
+        const depKeys = [];
+        if (Array.isArray(instance[associationOptions.alias])) {
+            instance[associationOptions.alias].map(associationInstance => {
+                depKeys.push(generateDependencyKey(modelOptions.name, associationOptions.name, associationInstance[accessKey]));
+                if (associationOptions.innerAssociation &&
+                    associationInstance[associationOptions.innerAssociation.alias]) {
+                    depKeys.push(generateDependencyKey(modelOptions.name, associationOptions.innerAssociation.name, associationInstance[associationOptions.innerAssociation.alias][associationOptions.innerAssociation.accessKey]));
+                }
+            });
+            return depKeys;
+        }
+        return [
+            generateDependencyKey(modelOptions.name, associationOptions.name, instance[associationOptions.alias][accessKey]),
+        ];
+    });
+    return keys.reduce((flattenArray, array) => flattenArray.concat(array), []);
+};
+const handleTransactionHook = (instance, options, func) => {
+    const { transaction } = options;
+    if (transaction) {
+        transaction.afterCommit(() => func(instance));
+    }
+    else {
+        func(instance);
+    }
+};
+class SequelizeAdapter {
+    constructor(sequelize, debug) {
+        this.ormInstance = sequelize;
+        this.debugMode = debug;
+    }
+    getModel(modelName) {
+        return this.ormInstance.models[modelName];
+    }
+    getModelDependencies(modelName) {
+        const { associations } = this.ormInstance.models[modelName];
+        return [
+            ...Object.keys(associations).map(association => {
+                const sequelizeAssociation = associations[association];
+                const associationOptions = {
+                    alias: sequelizeAssociation.as,
+                    name: sequelizeAssociation.target.name,
+                    accessKey: sequelizeAssociation.target.primaryKeyAttribute,
+                };
+                if (sequelizeAssociation.through) {
+                    const relationModel = sequelizeAssociation.through.model;
+                    associationOptions.innerAssociation = {
+                        alias: relationModel.name,
+                        name: relationModel.name,
+                        accessKey: relationModel.primaryKeyAttribute,
+                    };
+                }
+                return associationOptions;
+            }),
+        ];
+    }
+    debug(message, payload) {
+        if (this.debugMode) {
+            logger_1.default.info(`[ORM_CACHE Debug] ${message}`, payload);
+        }
+    }
+    injectGetWithCacheFunction(cache, modelOptions) {
+        const addDependencies = async (instance) => {
+            const dependencyKeys = getInstanceDependencyKeys(modelOptions, instance);
+            const instanceKey = generateInstanceKey(modelOptions.name, instance.id);
+            this.debug('Adding dependencies', { instanceKey, dependencyKeys });
+            const addDependenciesMulti = cache.getClient().multi();
+            const addDependenciesMultiAsync = (0, util_1.promisify)(addDependenciesMulti.exec).bind(addDependenciesMulti);
+            dependencyKeys.reduce((multi, key) => multi.sadd(key, instanceKey), addDependenciesMulti);
+            return addDependenciesMultiAsync();
+        };
+        const model = this.getModel(modelOptions.name);
+        model.findByPkCached = async (id, scopes, options) => {
+            const cacheKey = generateInstanceKey(modelOptions.name, id);
+            let value = JSON.parse(await cache.getClient().getAsync(cacheKey));
+            if (!value) {
+                this.debug('Value not found in cache, looking in db', { id, cacheKey });
+                value = await model.scope(scopes).findByPk(id, options);
+                this.debug('Value from DB', { value: value || 'not found', cacheKey });
+                await Promise.all([
+                    cache.getClient().setAsync(cacheKey, JSON.stringify(value)),
+                    value && addDependencies(value),
+                ]);
+            }
+            else {
+                value = this.getModel(modelOptions.name).build(value, { isNewRecord: false, include: options.include });
+                this.debug('Found cached value', { value, id, cacheKey });
+            }
+            return value;
+        };
+    }
+    addInvalidationHooks(cache, modelOptions) {
+        const invalidateModelInstance = async (instance) => {
+            const dependencyKeys = getInstanceDependencyKeys(modelOptions, instance);
+            const instanceKey = generateInstanceKey(modelOptions.name, instance.id);
+            this.debug('Removing dependencies', { instance, instanceKey, dependencyKeys });
+            const removeMulti = cache.getClient().multi();
+            const removeMultiAsync = (0, util_1.promisify)(removeMulti.exec).bind(removeMulti);
+            dependencyKeys.map(key => removeMulti.srem(key, instanceKey));
+            removeMulti.del(instanceKey);
+            return removeMultiAsync();
+        };
+        const invalidateModelInstanceByAssociation = async (association, associationId) => {
+            const dependentInstancesKeys = await cache.getClient().smembersAsync(generateDependencyKey(modelOptions.name, association, associationId));
+            this.debug('Invalidating dependent instances', { dependentInstancesKeys });
+            const removeMulti = cache.getClient().multi();
+            const removeMultiAsync = (0, util_1.promisify)(removeMulti.exec).bind(removeMulti);
+            const dependenciesToRemove = await Promise.all(dependentInstancesKeys.map(async (instanceKey) => {
+                const instance = JSON.parse(await cache.getClient().getAsync(instanceKey));
+                if (!instance) {
+                    return [];
+                }
+                const dependencyKeys = getInstanceDependencyKeys(modelOptions, instance);
+                dependencyKeys.reduce((multi, key) => multi.srem(key, instanceKey), removeMulti);
+                removeMulti.del(instanceKey);
+                return dependencyKeys;
+            }));
+            this.debug('Removing dependencies', { dependentInstancesKeys, dependenciesToRemove });
+            return removeMultiAsync();
+        };
+        const model = this.getModel(modelOptions.name);
+        INVALIDATION_HOOKS.map(hook => model.addHook(hook, (instance, options) => handleTransactionHook(instance, options, instance => invalidateModelInstance(instance))));
+        BULK_HOOKS.map(hook => model.addHook(hook, options => options.individualHook = true));
+        modelOptions.associations = this.getModelDependencies(modelOptions.name);
+        this.debug(`Adding Invalidations Hooks to ${modelOptions.name}'s associations`, { associations: modelOptions.associations });
+        modelOptions.associations.map((associationOptions) => {
+            const associationModel = this.getModel(associationOptions.name);
+            INVALIDATION_HOOKS.map(hook => associationModel.addHook(hook, (instance, options) => handleTransactionHook(instance, options, associationInstance => invalidateModelInstanceByAssociation(associationOptions.name, associationInstance[associationOptions.accessKey]))));
+            BULK_HOOKS.map(hook => associationModel.addHook(hook, options => options.individualHook = true));
+            if (associationOptions.innerAssociation) {
+                const innerAssociationModel = this.getModel(associationOptions.innerAssociation.name);
+                INVALIDATION_HOOKS.map(hook => innerAssociationModel.addHook(hook, (instance, options) => handleTransactionHook(instance, options, innerAssociationInstance => invalidateModelInstanceByAssociation(associationOptions.innerAssociation.name, innerAssociationInstance[associationOptions.innerAssociation.accessKey]))));
+                BULK_HOOKS.map(hook => innerAssociationModel.addHook(hook, options => options.individualHook = true));
+            }
+        });
+    }
+}
+exports.default = SequelizeAdapter;

package/lib/redis/errors.d.ts

@@ -0,0 +1,13 @@
+import { RedisError } from 'redis';
+declare class RedisCacheError extends RedisError {
+    innerError: any;
+    message: string;
+    constructor(msg: string, err: any);
+    toString(): string;
+}
+declare class RedisLockError extends RedisCacheError {
+    key: string;
+    constructor(msg: string, err: any, key: string);
+    toString(): string;
+}
+export { RedisCacheError, RedisLockError, };

package/lib/redis/errors.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisLockError = exports.RedisCacheError = void 0;
+const redis_1 = require("redis");
+class RedisCacheError extends redis_1.RedisError {
+    constructor(msg, err) {
+        super();
+        this.innerError = err;
+        this.message = msg;
+    }
+    toString() {
+        return `${this.message}: ${this.innerError}`;
+    }
+}
+exports.RedisCacheError = RedisCacheError;
+class RedisLockError extends RedisCacheError {
+    constructor(msg, err, key) {
+        super(msg, err);
+        this.key = key;
+    }
+    toString() {
+        return `${this.message} (${this.key}): ${this.innerError}`;
+    }
+}
+exports.RedisLockError = RedisLockError;

package/lib/redis/index.d.ts

@@ -0,0 +1,21 @@
+declare class RedisCache {
+    private client;
+    private locker;
+    private lockTimeout;
+    private lockDuration;
+    locks: any;
+    private baseTTL;
+    private lockRetries;
+    keyPrefix: string;
+    useLock: boolean;
+    constructor(options: any);
+    get(key: any): Promise<any>;
+    getMultiple(keys: any): Promise<any>;
+    set(key: any, value: any): Promise<void>;
+    setMultiple(keyValues: Record<string, any>): Promise<any>;
+    remove(key: any): Promise<void>;
+    removeMultiple(keys: any): Promise<void>;
+    getClient(): any;
+    private lock;
+}
+export default RedisCache;

package/lib/redis/index.js

@@ -0,0 +1,151 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const redis_1 = __importDefault(require("redis"));
+const redis_lock_1 = __importDefault(require("redis-lock"));
+const util_1 = require("util");
+const errors_1 = require("./errors");
+const logger_1 = __importDefault(require("../logger"));
+const promisifyAll = (obj) => {
+    Object.keys(obj).forEach((key) => {
+        if (typeof obj[key] === 'function') {
+            obj[`${key}Async`] = (0, util_1.promisify)(obj[key]);
+        }
+    });
+};
+promisifyAll(redis_1.default.RedisClient.prototype);
+promisifyAll(redis_1.default.Multi.prototype);
+const { env } = process;
+const HOST = env.REDIS_HOST_NAME || '127.0.0.1';
+const PORT = env.REDIS_HOST_PORT || 6379;
+const SERVICE_NAME = env.AF_SERVICE_NAME;
+const DEFAULT_LOCK_TIMEOUT = 5000;
+const DEFAULT_LOCK_DURATION = 1000;
+const DEFAULT_BASE_TTL = 3600;
+const KEY_PREFIX = SERVICE_NAME + ':';
+if (!SERVICE_NAME) {
+    throw new Error('SERVICE_NAME cannot be null, please check your env.AF_SERVICE_NAME');
+}
+class RedisCache {
+    constructor(options) {
+        this.client = redis_1.default.createClient({
+            host: options.host || HOST,
+            port: options.port || PORT,
+        });
+        this.locker = (0, util_1.promisify)((0, redis_lock_1.default)(this.client, options.lockRetries ?? 10));
+        this.lockTimeout = options.lockTimeout ?? DEFAULT_LOCK_TIMEOUT;
+        this.lockDuration = options.lockDuration ?? DEFAULT_LOCK_DURATION;
+        this.baseTTL = options.ttl ?? DEFAULT_BASE_TTL;
+        this.locks = {};
+        this.useLock = !!options.useLock;
+        this.keyPrefix = KEY_PREFIX;
+    }
+    async get(key) {
+        const keyWithPrefix = KEY_PREFIX + key;
+        let value;
+        try {
+            // Try to get the value from redis.
+            value = await this.client.getAsync(keyWithPrefix);
+        }
+        catch (err) {
+            throw new errors_1.RedisCacheError('Failed to get a value', err);
+        }
+        if (this.useLock) {
+            let lock;
+            try {
+                // Try to lock the key.
+                lock = await this.lock(keyWithPrefix);
+            }
+            catch (err) {
+                throw new errors_1.RedisLockError('Failed to lock key', err, keyWithPrefix);
+            }
+            // If the lock did not fail, add it to a locks dictionary.
+            this.locks[keyWithPrefix] = lock;
+        }
+        return JSON.parse(value);
+    }
+    async getMultiple(keys) {
+        const keysWithPrefix = keys.map(key => KEY_PREFIX + key);
+        let values;
+        try {
+            if (keysWithPrefix.length === 0) {
+                return [];
+            }
+            // Try to get the value from redis.
+            values = await this.client.mgetAsync(keysWithPrefix);
+        }
+        catch (err) {
+            throw new errors_1.RedisCacheError('Failed to get a value', err);
+        }
+        return values.map(value => JSON.parse(value));
+    }
+    async set(key, value) {
+        const keyWithPrefix = KEY_PREFIX + key;
+        const ttl = parseInt(String(this.baseTTL * (Math.random() + 1)), 10);
+        try {
+            await this.client.setAsync(keyWithPrefix, JSON.stringify(value), 'EX', ttl);
+            if (this.locks[keyWithPrefix]) {
+                await this.locks[keyWithPrefix]();
+                delete this.locks[keyWithPrefix];
+            }
+        }
+        catch (err) {
+            throw new errors_1.RedisCacheError('Failed to set a key-value pair', err);
+        }
+    }
+    async setMultiple(keyValues) {
+        if (typeof keyValues !== 'object') {
+            const error = new errors_1.RedisCacheError('keyValues must be an object', new Error('keyValues must be an object'));
+            logger_1.default.error('keyValues must be an object', { error });
+            throw error;
+        }
+        const keyValuesWithPrefix = Object.entries(keyValues).map(([key, value]) => [KEY_PREFIX + key, JSON.stringify(value)]);
+        const ttl = parseInt(String(this.baseTTL * (Math.random() + 1)), 10);
+        try {
+            const multi = this.client.multi();
+            const setPromise = multi.msetAsync(...keyValuesWithPrefix.flat());
+            keyValuesWithPrefix.map(([key]) => {
+                return multi.expireAsync(key, ttl);
+            });
+            await multi.exec();
+            return setPromise;
+        }
+        catch (err) {
+            throw new errors_1.RedisCacheError('Failed to set multiple key-value pairs', err);
+        }
+    }
+    async remove(key) {
+        const keyWithPrefix = KEY_PREFIX + key;
+        try {
+            if (this.locks[keyWithPrefix]) {
+                await this.locks[keyWithPrefix]();
+                delete this.locks[keyWithPrefix];
+            }
+            await this.client.delAsync(keyWithPrefix);
+        }
+        catch (err) {
+            throw new errors_1.RedisCacheError(`Failed to delete key ${keyWithPrefix}`, err);
+        }
+    }
+    async removeMultiple(keys) {
+        const keysWithPrefix = keys.map(key => KEY_PREFIX + key);
+        try {
+            await this.client.delAsync(keysWithPrefix);
+        }
+        catch (err) {
+            throw new errors_1.RedisCacheError(`Failed to delete multiple keys ${keysWithPrefix.join('|')}`, err);
+        }
+    }
+    getClient() {
+        return this.client;
+    }
+    lock(key) {
+        return Promise.race([
+            this.locker(key, this.lockDuration),
+            new Promise((resolve, reject) => setTimeout(() => reject(), this.lockTimeout)),
+        ]);
+    }
+}
+exports.default = RedisCache;

package/lib/redis/index.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/redis/index.test.js

@@ -0,0 +1,65 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const redis_1 = __importDefault(require("redis"));
+const index_1 = __importDefault(require("./index"));
+const util_1 = require("util");
+const promisifyAll = (obj) => {
+    Object.keys(obj).forEach((key) => {
+        if (typeof obj[key] === 'function') {
+            obj[`${key}Async`] = (0, util_1.promisify)(obj[key]);
+        }
+    });
+};
+promisifyAll(redis_1.default.RedisClient.prototype);
+describe('RedisCache', () => {
+    let redisCache;
+    const testClient = redis_1.default.createClient();
+    beforeEach(() => {
+        redisCache = new index_1.default({});
+    });
+    afterEach(() => {
+        jest.clearAllMocks();
+    });
+    describe('setMultiple', () => {
+        it('should set multiple key-value pairs in redis', async () => {
+            const keyValues = { testKey1: 'testValue1', testKey2: 'testValue2' };
+            const keyValuesArray = Object.entries(keyValues);
+            await redisCache.setMultiple(keyValues);
+            const expectedKeyValues = keyValuesArray.map(([key, value]) => [`${redisCache.keyPrefix}${key}`, JSON.stringify(value)]);
+            await Promise.all(expectedKeyValues.map(async ([key, value]) => {
+                const valueInRedis = await testClient.getAsync(key);
+                expect(valueInRedis).toBe(value);
+            }));
+            // Check that the keys expire after a random amount of time.
+            const currentTTL = await testClient.ttlAsync(expectedKeyValues[0][0]);
+            expect(currentTTL).toBeGreaterThanOrEqual(0);
+        });
+    });
+    describe('set', () => {
+        it('should set a key-value pair in redis', async () => {
+            const key = 'testKey';
+            const value = 'testValue';
+            await redisCache.set(key, value);
+            const keyWithPrefix = `${redisCache.keyPrefix}${key}`;
+            const valueInRedis = await testClient.getAsync(keyWithPrefix);
+            expect(valueInRedis).toBe(JSON.stringify(value));
+            // Check that the key expires after a random amount of time.
+            const currentTTL = await testClient.ttlAsync(keyWithPrefix);
+            expect(currentTTL).toBeGreaterThanOrEqual(0);
+        });
+        it('should release the lock after setting a key-value pair', async () => {
+            const key = 'testKeyWithLock';
+            const value = 'testValueWithLock';
+            redisCache.useLock = true;
+            await redisCache.set(key, value);
+            const keyWithPrefix = `${redisCache.keyPrefix}${key}`;
+            const valueInRedis = await testClient.getAsync(keyWithPrefix);
+            expect(valueInRedis).toBe(JSON.stringify(value));
+            // Check that the lock is released
+            expect(redisCache.locks[keyWithPrefix]).toBeUndefined();
+        });
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autofleet/matmon",
-  "version": "2.3.0",
+  "version": "2.3.1-beta-1",
   "description": "manage cache",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",