@autofleet/matmon 2.0.6 → 2.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cache.js +18 -20
- package/lib/locking.js +4 -13
- package/lib/orm-cache/sequelize-adapter.js +15 -24
- package/lib/redis/index.js +54 -72
- package/package.json +1 -1
package/lib/cache.js
CHANGED
|
@@ -18,15 +18,6 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
18
18
|
__setModuleDefault(result, mod);
|
|
19
19
|
return result;
|
|
20
20
|
};
|
|
21
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
22
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
23
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
24
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
25
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
26
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
27
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
28
|
-
});
|
|
29
|
-
};
|
|
30
21
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
31
22
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
32
23
|
};
|
|
@@ -49,33 +40,40 @@ const getMutexByCacheKey = (key) => {
|
|
|
49
40
|
}
|
|
50
41
|
return MUTEX_MAP[key];
|
|
51
42
|
};
|
|
43
|
+
const deleteMutexByCacheKey = (key) => {
|
|
44
|
+
if (MUTEX_MAP[key]) {
|
|
45
|
+
delete MUTEX_MAP[key];
|
|
46
|
+
}
|
|
47
|
+
};
|
|
52
48
|
exports.getNewLRU = (lifeTimeInSec, size) => new lru_cache_1.default(getOptions({
|
|
53
49
|
lifeTimeInSec,
|
|
54
50
|
size,
|
|
55
51
|
}));
|
|
56
|
-
exports.getWithCacheSupport = ({ cacheKey, cacheGet, cacheSet, fetching, skipCache, }) =>
|
|
52
|
+
exports.getWithCacheSupport = async ({ cacheKey, cacheGet, cacheSet, fetching, skipCache, }) => {
|
|
57
53
|
if (skipCache || process.env.NODE_ENV === 'test') {
|
|
58
|
-
const res =
|
|
59
|
-
|
|
54
|
+
const res = await fetching();
|
|
55
|
+
await cacheSet(res);
|
|
60
56
|
return res;
|
|
61
57
|
}
|
|
62
58
|
let valueToReturn = null;
|
|
63
59
|
try {
|
|
64
|
-
|
|
65
|
-
valueToReturn =
|
|
60
|
+
await locking.wrapWithMutex(getMutexByCacheKey(cacheKey), async () => {
|
|
61
|
+
valueToReturn = await cacheGet();
|
|
66
62
|
if (!valueToReturn) {
|
|
67
|
-
valueToReturn =
|
|
68
|
-
|
|
63
|
+
valueToReturn = await fetching();
|
|
64
|
+
await cacheSet(valueToReturn);
|
|
69
65
|
}
|
|
70
66
|
else {
|
|
71
67
|
// logger.info('get value from cache');
|
|
72
68
|
}
|
|
73
|
-
|
|
69
|
+
deleteMutexByCacheKey(cacheKey);
|
|
70
|
+
});
|
|
74
71
|
// retry without locking if failed
|
|
75
72
|
}
|
|
76
73
|
catch (e) {
|
|
77
|
-
valueToReturn =
|
|
78
|
-
|
|
74
|
+
valueToReturn = await fetching();
|
|
75
|
+
await cacheSet(valueToReturn);
|
|
76
|
+
deleteMutexByCacheKey(cacheKey);
|
|
79
77
|
}
|
|
80
78
|
return valueToReturn;
|
|
81
|
-
}
|
|
79
|
+
};
|
package/lib/locking.js
CHANGED
|
@@ -1,26 +1,17 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
3
|
exports.getMutex = exports.wrapWithMutex = void 0;
|
|
13
4
|
const async_mutex_1 = require("async-mutex");
|
|
14
5
|
const CACHE_LOCK_TIMEOUT_MILIS = 3000;
|
|
15
6
|
const LOCK_TIMEOUT_MESSAGE = 'mutex - locking timeout';
|
|
16
7
|
// eslint-disable-next-line import/prefer-default-export
|
|
17
|
-
exports.wrapWithMutex = (mutex, funcToRun) =>
|
|
18
|
-
const release =
|
|
8
|
+
exports.wrapWithMutex = async (mutex, funcToRun) => {
|
|
9
|
+
const release = await mutex.acquire();
|
|
19
10
|
try {
|
|
20
|
-
|
|
11
|
+
await funcToRun();
|
|
21
12
|
}
|
|
22
13
|
finally {
|
|
23
14
|
release();
|
|
24
15
|
}
|
|
25
|
-
}
|
|
16
|
+
};
|
|
26
17
|
exports.getMutex = () => async_mutex_1.withTimeout(new async_mutex_1.Mutex(), CACHE_LOCK_TIMEOUT_MILIS, new Error(LOCK_TIMEOUT_MESSAGE));
|
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
4
|
};
|
|
@@ -87,7 +78,7 @@ class SequelizeAdapter {
|
|
|
87
78
|
}
|
|
88
79
|
}
|
|
89
80
|
injectGetWithCacheFunction(cache, modelOptions) {
|
|
90
|
-
const addDependencies = (instance) =>
|
|
81
|
+
const addDependencies = async (instance) => {
|
|
91
82
|
const dependencyKeys = getInstanceDependencyKeys(modelOptions, instance);
|
|
92
83
|
const instanceKey = generateInstanceKey(modelOptions.name, instance.id);
|
|
93
84
|
this.debug('Adding dependencies', { instanceKey, dependencyKeys });
|
|
@@ -95,16 +86,16 @@ class SequelizeAdapter {
|
|
|
95
86
|
const addDependenciesMultiAsync = util_1.promisify(addDependenciesMulti.exec).bind(addDependenciesMulti);
|
|
96
87
|
dependencyKeys.reduce((multi, key) => multi.sadd(key, instanceKey), addDependenciesMulti);
|
|
97
88
|
return addDependenciesMultiAsync();
|
|
98
|
-
}
|
|
89
|
+
};
|
|
99
90
|
const model = this.getModel(modelOptions.name);
|
|
100
|
-
model.findByPkCached = (id, scopes, options) =>
|
|
91
|
+
model.findByPkCached = async (id, scopes, options) => {
|
|
101
92
|
const cacheKey = generateInstanceKey(modelOptions.name, id);
|
|
102
|
-
let value = JSON.parse(
|
|
93
|
+
let value = JSON.parse(await cache.getClient().getAsync(cacheKey));
|
|
103
94
|
if (!value) {
|
|
104
95
|
this.debug('Value not found in cache, looking in db', { id, cacheKey });
|
|
105
|
-
value =
|
|
96
|
+
value = await model.scope(scopes).findByPk(id, options);
|
|
106
97
|
this.debug('Value from DB', { value: value || 'not found', cacheKey });
|
|
107
|
-
|
|
98
|
+
await Promise.all([
|
|
108
99
|
cache.getClient().setAsync(cacheKey, JSON.stringify(value)),
|
|
109
100
|
value && addDependencies(value),
|
|
110
101
|
]);
|
|
@@ -114,10 +105,10 @@ class SequelizeAdapter {
|
|
|
114
105
|
this.debug('Found cached value', { value, id, cacheKey });
|
|
115
106
|
}
|
|
116
107
|
return value;
|
|
117
|
-
}
|
|
108
|
+
};
|
|
118
109
|
}
|
|
119
110
|
addInvalidationHooks(cache, modelOptions) {
|
|
120
|
-
const invalidateModelInstance = (instance) =>
|
|
111
|
+
const invalidateModelInstance = async (instance) => {
|
|
121
112
|
const dependencyKeys = getInstanceDependencyKeys(modelOptions, instance);
|
|
122
113
|
const instanceKey = generateInstanceKey(modelOptions.name, instance.id);
|
|
123
114
|
this.debug('Removing dependencies', { instance, instanceKey, dependencyKeys });
|
|
@@ -126,14 +117,14 @@ class SequelizeAdapter {
|
|
|
126
117
|
dependencyKeys.map(key => removeMulti.srem(key, instanceKey));
|
|
127
118
|
removeMulti.del(instanceKey);
|
|
128
119
|
return removeMultiAsync();
|
|
129
|
-
}
|
|
130
|
-
const invalidateModelInstanceByAssociation = (association, associationId) =>
|
|
131
|
-
const dependentInstancesKeys =
|
|
120
|
+
};
|
|
121
|
+
const invalidateModelInstanceByAssociation = async (association, associationId) => {
|
|
122
|
+
const dependentInstancesKeys = await cache.getClient().smembersAsync(generateDependencyKey(modelOptions.name, association, associationId));
|
|
132
123
|
this.debug('Invalidating dependent instances', { dependentInstancesKeys });
|
|
133
124
|
const removeMulti = cache.getClient().multi();
|
|
134
125
|
const removeMultiAsync = util_1.promisify(removeMulti.exec).bind(removeMulti);
|
|
135
|
-
const dependenciesToRemove =
|
|
136
|
-
const instance = JSON.parse(
|
|
126
|
+
const dependenciesToRemove = await Promise.all(dependentInstancesKeys.map(async (instanceKey) => {
|
|
127
|
+
const instance = JSON.parse(await cache.getClient().getAsync(instanceKey));
|
|
137
128
|
if (!instance) {
|
|
138
129
|
return [];
|
|
139
130
|
}
|
|
@@ -141,10 +132,10 @@ class SequelizeAdapter {
|
|
|
141
132
|
dependencyKeys.reduce((multi, key) => multi.srem(key, instanceKey), removeMulti);
|
|
142
133
|
removeMulti.del(instanceKey);
|
|
143
134
|
return dependencyKeys;
|
|
144
|
-
}))
|
|
135
|
+
}));
|
|
145
136
|
this.debug('Removing dependencies', { dependentInstancesKeys, dependenciesToRemove });
|
|
146
137
|
return removeMultiAsync();
|
|
147
|
-
}
|
|
138
|
+
};
|
|
148
139
|
const model = this.getModel(modelOptions.name);
|
|
149
140
|
INVALIDATION_HOOKS.map(hook => model.addHook(hook, (instance, options) => handleTransactionHook(instance, options, instance => invalidateModelInstance(instance))));
|
|
150
141
|
BULK_HOOKS.map(hook => model.addHook(hook, options => options.individualHook = true));
|
package/lib/redis/index.js
CHANGED
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
4
|
};
|
|
@@ -32,85 +23,76 @@ if (!SERVICE_NAME) {
|
|
|
32
23
|
}
|
|
33
24
|
class RedisCache {
|
|
34
25
|
constructor(options) {
|
|
35
|
-
var _a, _b, _c, _d;
|
|
36
26
|
this.client = redis_1.default.createClient({
|
|
37
27
|
host: options.host || HOST,
|
|
38
28
|
port: options.port || PORT,
|
|
39
29
|
});
|
|
40
|
-
this.locker = util_1.promisify(redis_lock_1.default(this.client,
|
|
41
|
-
this.lockTimeout =
|
|
42
|
-
this.lockDuration =
|
|
43
|
-
this.baseTTL =
|
|
30
|
+
this.locker = util_1.promisify(redis_lock_1.default(this.client, options.lockRetries ?? 10));
|
|
31
|
+
this.lockTimeout = options.lockTimeout ?? DEFAULT_LOCK_TIMEOUT;
|
|
32
|
+
this.lockDuration = options.lockDuration ?? DEFAULT_LOCK_DURATION;
|
|
33
|
+
this.baseTTL = options.ttl ?? DEFAULT_BASE_TTL;
|
|
44
34
|
this.locks = {};
|
|
45
35
|
this.useLock = !!options.useLock;
|
|
46
36
|
}
|
|
47
|
-
get(key) {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
37
|
+
async get(key) {
|
|
38
|
+
const keyWithPrefix = KEY_PREFIX + key;
|
|
39
|
+
let value;
|
|
40
|
+
try {
|
|
41
|
+
// Try to get the value from redis.
|
|
42
|
+
value = await this.client.getAsync(keyWithPrefix);
|
|
43
|
+
}
|
|
44
|
+
catch (err) {
|
|
45
|
+
throw new errors_1.RedisCacheError('Failed to get a value', err);
|
|
46
|
+
}
|
|
47
|
+
if (this.useLock) {
|
|
48
|
+
let lock;
|
|
51
49
|
try {
|
|
52
|
-
// Try to
|
|
53
|
-
|
|
50
|
+
// Try to lock the key.
|
|
51
|
+
lock = await this.lock(keyWithPrefix);
|
|
54
52
|
}
|
|
55
53
|
catch (err) {
|
|
56
|
-
throw new errors_1.
|
|
57
|
-
}
|
|
58
|
-
if (this.useLock) {
|
|
59
|
-
let lock;
|
|
60
|
-
try {
|
|
61
|
-
// Try to lock the key.
|
|
62
|
-
lock = yield this.lock(keyWithPrefix);
|
|
63
|
-
}
|
|
64
|
-
catch (err) {
|
|
65
|
-
throw new errors_1.RedisLockError('Failed to lock key', err, keyWithPrefix);
|
|
66
|
-
}
|
|
67
|
-
// If the lock did not fail, add it to a locks dictionary.
|
|
68
|
-
this.locks[keyWithPrefix] = lock;
|
|
54
|
+
throw new errors_1.RedisLockError('Failed to lock key', err, keyWithPrefix);
|
|
69
55
|
}
|
|
70
|
-
|
|
71
|
-
|
|
56
|
+
// If the lock did not fail, add it to a locks dictionary.
|
|
57
|
+
this.locks[keyWithPrefix] = lock;
|
|
58
|
+
}
|
|
59
|
+
return JSON.parse(value);
|
|
72
60
|
}
|
|
73
|
-
set(key, value) {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
delete this.locks[keyWithPrefix];
|
|
82
|
-
}
|
|
61
|
+
async set(key, value) {
|
|
62
|
+
const keyWithPrefix = KEY_PREFIX + key;
|
|
63
|
+
const ttl = parseInt(String(this.baseTTL * (Math.random() + 1)), 10);
|
|
64
|
+
try {
|
|
65
|
+
await this.client.setAsync(keyWithPrefix, JSON.stringify(value), 'EX', ttl);
|
|
66
|
+
if (this.locks[keyWithPrefix]) {
|
|
67
|
+
await this.locks[keyWithPrefix]();
|
|
68
|
+
delete this.locks[keyWithPrefix];
|
|
83
69
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
}
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
throw new errors_1.RedisCacheError('Failed to set a key-value pair', err);
|
|
73
|
+
}
|
|
88
74
|
}
|
|
89
|
-
remove(key) {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
delete this.locks[keyWithPrefix];
|
|
96
|
-
}
|
|
97
|
-
yield this.client.delAsync(keyWithPrefix);
|
|
75
|
+
async remove(key) {
|
|
76
|
+
const keyWithPrefix = KEY_PREFIX + key;
|
|
77
|
+
try {
|
|
78
|
+
if (this.locks[keyWithPrefix]) {
|
|
79
|
+
await this.locks[keyWithPrefix]();
|
|
80
|
+
delete this.locks[keyWithPrefix];
|
|
98
81
|
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
82
|
+
await this.client.delAsync(keyWithPrefix);
|
|
83
|
+
}
|
|
84
|
+
catch (err) {
|
|
85
|
+
throw new errors_1.RedisCacheError(`Failed to delete key ${keyWithPrefix}`, err);
|
|
86
|
+
}
|
|
103
87
|
}
|
|
104
|
-
removeMultiple(keys) {
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
}
|
|
113
|
-
});
|
|
88
|
+
async removeMultiple(keys) {
|
|
89
|
+
const keysWithPrefix = keys.map(key => KEY_PREFIX + key);
|
|
90
|
+
try {
|
|
91
|
+
await this.client.delAsync(keysWithPrefix);
|
|
92
|
+
}
|
|
93
|
+
catch (err) {
|
|
94
|
+
throw new errors_1.RedisCacheError(`Failed to delete multiple keys ${keysWithPrefix.join('|')}`, err);
|
|
95
|
+
}
|
|
114
96
|
}
|
|
115
97
|
getClient() {
|
|
116
98
|
return this.client;
|