@sdflc/backend-helpers 1.0.50 → 1.0.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -22,6 +22,7 @@ const constants_1 = require("./constants");
22
22
  const DEFAULT_CHUNCK_SIZE = 2500;
23
23
  class BaseGateway {
24
24
  constructor(props) {
25
+ var _a, _b;
25
26
  this.dateTime = new utils_1.DateTimeHelpers();
26
27
  this.dateTimeNow = new Date();
27
28
  this.config = props;
@@ -30,6 +31,13 @@ class BaseGateway {
30
31
  this.config.activeStatuses = Array.isArray(this.config.activeStatuses)
31
32
  ? this.config.activeStatuses
32
33
  : [utils_1.STATUSES.ACTIVE];
34
+ this.config.keyPrefix = ((_a = this.config.context) === null || _a === void 0 ? void 0 : _a.keyPrefix) || this.config.keyPrefix || this.config.table;
35
+ if (this.config.keyPrefix && ((_b = this.config.context) === null || _b === void 0 ? void 0 : _b.usedKeyPrefixes)) {
36
+ if (this.config.context.usedKeyPrefixes[this.config.keyPrefix]) {
37
+ throw new Error(`The "${this.config.keyPrefix}" is already used as the "keyPrefix" in the application. Please use another one that is not as: ${Object.keys(this.config.context.usedKeyPrefixes).join(', ')}`);
38
+ }
39
+ this.config.context.usedKeyPrefixes[this.config.keyPrefix] = true;
40
+ }
33
41
  this.loader = typeof this.config.loader === 'function' ? this.config.loader : new DataCache_1.DataCache(this.config);
34
42
  }
35
43
  setConfigPart(props) {
@@ -1,9 +1,9 @@
1
1
  import { CacheInterface, LoaderInterface } from './interfaces';
2
2
  declare class DataCache implements LoaderInterface {
3
3
  private buildKey;
4
- private redisDataLoader;
5
4
  private loader;
6
5
  private config;
6
+ private redisClient;
7
7
  constructor(props: CacheInterface);
8
8
  batchLoaderDefault(props: CacheInterface): (ids: any[]) => Promise<any[]>;
9
9
  init(batchFunction: any, keyPrefix?: string, context?: any): void;
@@ -14,9 +14,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
14
14
  Object.defineProperty(exports, "__esModule", { value: true });
15
15
  exports.DataCache = void 0;
16
16
  const dataloader_1 = __importDefault(require("dataloader"));
17
- const redis_dataloader_1 = __importDefault(require("redis-dataloader"));
17
+ const RedisDataLoader_1 = require("./RedisDataLoader");
18
18
  const utils_1 = require("@sdflc/utils");
19
19
  const _1 = require(".");
20
+ const lodash_1 = require("lodash");
20
21
  class DataCache {
21
22
  constructor(props) {
22
23
  const { loaderFn, keyPrefix, context, buildKey, redisClient, table } = props || {};
@@ -24,9 +25,9 @@ class DataCache {
24
25
  props.idField = _1.FIELDS.ID;
25
26
  }
26
27
  this.config = props;
28
+ this.redisClient = redisClient;
27
29
  this.buildKey = typeof buildKey === 'function' ? buildKey : utils_1.buildKey;
28
30
  this.batchLoaderDefault = this.batchLoaderDefault.bind(this);
29
- this.redisDataLoader = redisClient ? (0, redis_dataloader_1.default)({ redis: redisClient }) : null;
30
31
  const fn = typeof loaderFn === 'function' ? loaderFn : this.batchLoaderDefault(props);
31
32
  const prefixToUse = (context === null || context === void 0 ? void 0 : context.keyPrefix) || keyPrefix || table;
32
33
  this.init(fn, prefixToUse, context);
@@ -45,11 +46,12 @@ class DataCache {
45
46
  return (ids) => __awaiter(this, void 0, void 0, function* () {
46
47
  const resultItems = [];
47
48
  const readCfg = Object.assign(Object.assign({}, props), { buildKey: this.buildKey });
49
+ const idsToUse = (0, lodash_1.castArray)(ids);
48
50
  if (!readAll) {
49
51
  readCfg.mapBy = idField;
50
52
  readCfg.whereIn = {
51
53
  field: idField,
52
- values: ids.filter((id) => id),
54
+ values: idsToUse.filter((id) => id),
53
55
  };
54
56
  }
55
57
  const dbItems = yield db.read(readCfg);
@@ -57,21 +59,21 @@ class DataCache {
57
59
  resultItems.push(...dbItems);
58
60
  }
59
61
  else {
60
- resultItems.push(...ids.map((id) => dbItems[this.buildKey(id)] || null));
62
+ resultItems.push(...idsToUse.map((id) => dbItems[this.buildKey(id)] || null));
61
63
  }
62
64
  return resultItems;
63
65
  });
64
66
  }
65
67
  init(batchFunction, keyPrefix, context) {
66
- if (this.redisDataLoader) {
67
- this.loader = new this.redisDataLoader(
68
+ if (this.redisClient) {
69
+ this.loader = new RedisDataLoader_1.RedisDataLoader(this.redisClient,
68
70
  // set a prefix for the keys stored in redis. This way you can avoid key
69
71
  // collisions for different data-sets in your redis instance.
70
72
  keyPrefix,
71
- // create a regular dataloader. This should always be set with caching disabled.
72
- new dataloader_1.default((ids) => batchFunction(ids, context), {
73
- cache: false,
74
- }), {
73
+ // create a regular loader. This should always be set with caching disabled.
74
+ {
75
+ load: (ids) => batchFunction(ids, context),
76
+ }, {
75
77
  cache: true,
76
78
  expire: this.config.cacheTtl,
77
79
  });
@@ -0,0 +1,41 @@
1
+ declare class RedisDataLoader {
2
+ private opt?;
3
+ private dataLoaderOpt?;
4
+ private keySpace;
5
+ private loader?;
6
+ private redisClient;
7
+ private static messages;
8
+ constructor(redisClient: any, keySpace?: string, userLoader?: any, opt?: any);
9
+ /**
10
+ * Adds a prefix to the passed key
11
+ * @param key Key value
12
+ * @returns Trasnformed key value that includes key space
13
+ */
14
+ makeKey(key: string): string;
15
+ /**
16
+ * Deserializes response from Readis
17
+ * @param str String to deserialize
18
+ * @returns
19
+ */
20
+ parse(str: any): Promise<any>;
21
+ /**
22
+ * Serializes passed value to a string
23
+ * @param value A value to be serialized
24
+ * @returns serialized value in the form of string
25
+ */
26
+ toString(value: any): Promise<string>;
27
+ rSetAndGet(key: string, rawValue: any): Promise<any>;
28
+ rMSetAndGet(keys: string[], rawValues: any[]): Promise<any[]>;
29
+ rGet(key: string): Promise<any>;
30
+ rMGet(keys: string[]): Promise<any[]>;
31
+ rSet(key: string, rawValue: any): Promise<any>;
32
+ rMSet(keys: string[], values: any[]): Promise<any>;
33
+ rDel(key: string): Promise<any>;
34
+ load(key: string): Promise<any>;
35
+ loadMany(keys: string[]): Promise<any>;
36
+ prime(key: string, value: any): Promise<any>;
37
+ clear(key: string): Promise<any>;
38
+ clearAllLocal(): Promise<any>;
39
+ clearLocal(key: string): Promise<any>;
40
+ }
41
+ export { RedisDataLoader };
@@ -0,0 +1,248 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ var __importDefault = (this && this.__importDefault) || function (mod) {
12
+ return (mod && mod.__esModule) ? mod : { "default": mod };
13
+ };
14
+ Object.defineProperty(exports, "__esModule", { value: true });
15
+ exports.RedisDataLoader = void 0;
16
+ const lodash_1 = require("lodash");
17
+ const dataloader_1 = __importDefault(require("dataloader"));
18
+ const json_stable_stringify_1 = __importDefault(require("json-stable-stringify"));
19
+ const customOptions = ['expire', 'serialize', 'deserialize', 'cacheKeyFn'];
20
+ const mapPromise = (promise, fn) => Promise.all(promise.map(fn));
21
+ class RedisDataLoader {
22
+ constructor(redisClient, keySpace, userLoader, opt) {
23
+ this.redisClient = redisClient;
24
+ this.opt = (0, lodash_1.pick)(opt, customOptions) || {};
25
+ this.dataLoaderOpt = Object.assign(Object.assign({}, (0, lodash_1.omit)(opt, customOptions)), { cacheKeyFn: this.opt.cacheKeyFn });
26
+ this.opt.cacheKeyFn =
27
+ typeof this.opt.cacheKeyFn === 'function'
28
+ ? this.opt.cacheKeyFn
29
+ : (key) => ((0, lodash_1.isObject)(key) ? (0, json_stable_stringify_1.default)(key) : key);
30
+ this.keySpace = keySpace ? keySpace + ':' : '';
31
+ this.loader = new dataloader_1.default((keys) => __awaiter(this, void 0, void 0, function* () {
32
+ const results = yield this.rMGet(keys);
33
+ const keysToRead = [];
34
+ const keysIdxsToRead = [];
35
+ keys.forEach((key, idx) => {
36
+ if (results[idx] == null) {
37
+ keysToRead.push(key);
38
+ keysIdxsToRead.push(idx);
39
+ }
40
+ });
41
+ if (keysToRead.length > 0) {
42
+ const loadedData = yield userLoader.load(keysToRead);
43
+ const data = yield this.rMSetAndGet(keysToRead, loadedData);
44
+ keysIdxsToRead.forEach((keyIdx, idx) => {
45
+ results[keyIdx] = data[idx];
46
+ });
47
+ }
48
+ return results;
49
+ }), this.dataLoaderOpt);
50
+ }
51
+ /**
52
+ * Adds a prefix to the passed key
53
+ * @param key Key value
54
+ * @returns Trasnformed key value that includes key space
55
+ */
56
+ makeKey(key) {
57
+ return `${this.keySpace}${this.opt.cacheKeyFn(key)}`;
58
+ }
59
+ /**
60
+ * Deserializes response from Readis
61
+ * @param str String to deserialize
62
+ * @returns
63
+ */
64
+ parse(str) {
65
+ return __awaiter(this, void 0, void 0, function* () {
66
+ try {
67
+ if (str === '' || str === null) {
68
+ return str;
69
+ }
70
+ else if (typeof this.opt.deserialize === 'function') {
71
+ return yield this.opt.deserialize(str);
72
+ }
73
+ else {
74
+ return JSON.parse(str);
75
+ }
76
+ }
77
+ catch (ex) {
78
+ throw ex;
79
+ }
80
+ });
81
+ }
82
+ /**
83
+ * Serializes passed value to a string
84
+ * @param value A value to be serialized
85
+ * @returns serialized value in the form of string
86
+ */
87
+ toString(value) {
88
+ return __awaiter(this, void 0, void 0, function* () {
89
+ if (value === null) {
90
+ return '';
91
+ }
92
+ else if (typeof this.opt.serialize === 'function') {
93
+ return yield this.opt.serialize(value);
94
+ }
95
+ else if ((0, lodash_1.isObject)(value)) {
96
+ return JSON.stringify(value);
97
+ }
98
+ else {
99
+ throw new Error(RedisDataLoader.messages.valueObjectOrNull);
100
+ }
101
+ });
102
+ }
103
+ rSetAndGet(key, rawValue) {
104
+ return __awaiter(this, void 0, void 0, function* () {
105
+ const value = yield this.toString(rawValue);
106
+ const fullKey = this.makeKey(key);
107
+ const multi = this.redisClient.multi();
108
+ multi.set(fullKey, value);
109
+ if (this.opt.expire) {
110
+ multi.expire(fullKey, this.opt.expire);
111
+ }
112
+ multi.get(fullKey);
113
+ const replies = yield multi.exec();
114
+ const lastReply = (0, lodash_1.last)(replies);
115
+ const result = yield this.parse(lastReply);
116
+ return result;
117
+ });
118
+ }
119
+ rMSetAndGet(keys, rawValues) {
120
+ return __awaiter(this, void 0, void 0, function* () {
121
+ const multi = this.redisClient.multi();
122
+ const fullKeys = yield mapPromise(keys, (key) => this.makeKey(key));
123
+ const values = yield mapPromise(rawValues, (rawValue) => this.toString(rawValue));
124
+ multi.mSet(fullKeys.reduce((acc, fullKey, idx) => {
125
+ acc.push(fullKey);
126
+ acc.push(values[idx]);
127
+ return acc;
128
+ }, []));
129
+ if (this.opt.expire) {
130
+ for (const fullKey of fullKeys) {
131
+ multi.expire(fullKey, this.opt.expire);
132
+ }
133
+ }
134
+ multi.mGet(fullKeys);
135
+ const replies = yield multi.exec();
136
+ const lastReplies = (0, lodash_1.last)(replies);
137
+ const result = yield mapPromise(lastReplies, (lastReply) => this.parse(lastReply));
138
+ return result;
139
+ });
140
+ }
141
+ rGet(key) {
142
+ return __awaiter(this, void 0, void 0, function* () {
143
+ const reply = yield this.redisClient.get(this.makeKey(key));
144
+ const result = yield this.parse(reply);
145
+ return result;
146
+ });
147
+ }
148
+ rMGet(keys) {
149
+ return __awaiter(this, void 0, void 0, function* () {
150
+ const replies = yield this.redisClient.mGet((0, lodash_1.map)(keys, (key) => this.makeKey(key)));
151
+ const results = yield mapPromise(replies, (reply) => this.parse(reply));
152
+ return results;
153
+ });
154
+ }
155
+ rSet(key, rawValue) {
156
+ return __awaiter(this, void 0, void 0, function* () {
157
+ const value = yield this.toString(rawValue);
158
+ const reply = yield this.redisClient.set(this.makeKey(key), value);
159
+ const result = yield this.parse(reply);
160
+ // result is expected to be OK
161
+ return result;
162
+ });
163
+ }
164
+ rMSet(keys, values) {
165
+ return __awaiter(this, void 0, void 0, function* () {
166
+ if (keys.length != values.length) {
167
+ throw new Error(RedisDataLoader.messages.keysValuesLengths);
168
+ }
169
+ const multi = this.redisClient.multi();
170
+ multi.mSet(keys.reduce((acc, key, idx) => {
171
+ acc.push(this.makeKey(key));
172
+ acc.push(values[idx]);
173
+ return acc;
174
+ }, []));
175
+ if (this.opt.expire) {
176
+ for (const key of keys) {
177
+ multi.expire(key, this.opt.expire);
178
+ }
179
+ }
180
+ // The reply is expected to be "OK"
181
+ const reply = yield multi.exec();
182
+ return reply;
183
+ });
184
+ }
185
+ rDel(key) {
186
+ return __awaiter(this, void 0, void 0, function* () {
187
+ return yield this.redisClient.del(this.makeKey(key));
188
+ });
189
+ }
190
+ load(key) {
191
+ return __awaiter(this, void 0, void 0, function* () {
192
+ if (!key) {
193
+ throw new TypeError(RedisDataLoader.messages.keyRequired);
194
+ }
195
+ return yield this.loader.load(key);
196
+ });
197
+ }
198
+ loadMany(keys) {
199
+ return __awaiter(this, void 0, void 0, function* () {
200
+ if (!Array.isArray(keys)) {
201
+ throw new TypeError(RedisDataLoader.messages.keysArrayRequired);
202
+ }
203
+ return yield this.loader.loadMany(keys);
204
+ });
205
+ }
206
+ prime(key, value) {
207
+ return __awaiter(this, void 0, void 0, function* () {
208
+ if (!key) {
209
+ throw new TypeError(RedisDataLoader.messages.keyRequired);
210
+ }
211
+ else if (value === undefined) {
212
+ throw new TypeError(RedisDataLoader.messages.valueRequired);
213
+ }
214
+ else {
215
+ const result = yield this.rSetAndGet(key, value);
216
+ return yield this.loader.clear(key).prime(key, result === '' ? null : result);
217
+ }
218
+ });
219
+ }
220
+ clear(key) {
221
+ return __awaiter(this, void 0, void 0, function* () {
222
+ if (!key) {
223
+ throw new TypeError(RedisDataLoader.messages.keyRequired);
224
+ }
225
+ const resp = yield this.rDel(key);
226
+ yield this.loader.clear(key);
227
+ return resp;
228
+ });
229
+ }
230
+ clearAllLocal() {
231
+ return __awaiter(this, void 0, void 0, function* () {
232
+ return yield this.loader.clearAll();
233
+ });
234
+ }
235
+ clearLocal(key) {
236
+ return __awaiter(this, void 0, void 0, function* () {
237
+ return yield this.loader.clear(key);
238
+ });
239
+ }
240
+ }
241
+ exports.RedisDataLoader = RedisDataLoader;
242
+ RedisDataLoader.messages = {
243
+ keyRequired: 'The "key" parameter is required',
244
+ keysArrayRequired: 'The "keys" parameter is required and should be an array',
245
+ valueRequired: 'The "value" parameter is required',
246
+ valueObjectOrNull: 'The "value" parameter must be an object or null',
247
+ keysValuesLengths: 'The "keys" and "values" are expected to be the same',
248
+ };
@@ -28,6 +28,7 @@ const createApiContext = (args) => __awaiter(void 0, void 0, void 0, function* (
28
28
  requestId,
29
29
  cores: {},
30
30
  gateways: {},
31
+ usedKeyPrefixes: {},
31
32
  req,
32
33
  };
33
34
  context.cores = Object.keys(cores || {}).reduce((acc, key) => {
@@ -31,4 +31,5 @@ export interface BaseGatewayPropsInterface extends CacheInterface {
31
31
  activeStatuses?: number[];
32
32
  logger: Logger;
33
33
  isProduction: boolean;
34
+ usedKeyPrefixes?: any;
34
35
  }
@@ -7,6 +7,11 @@ interface CreateRedisClientInterface {
7
7
  firstTimeout?: number;
8
8
  pingInterval?: number;
9
9
  connectionTimeout?: number;
10
+ onConnecting?(): void;
11
+ onConnected?(): void;
12
+ onDisconnected?(): void;
13
+ onReconnecting?(): void;
14
+ onError?(err: any): void;
10
15
  }
11
16
  declare const createRedisClient: (args: CreateRedisClientInterface) => any;
12
17
  export { createRedisClient };
@@ -1,25 +1,58 @@
1
1
  "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
2
  Object.defineProperty(exports, "__esModule", { value: true });
6
3
  exports.createRedisClient = void 0;
7
- const redis_1 = __importDefault(require("redis"));
4
+ const redis_1 = require("redis");
8
5
  // const DEFAULT_RETRY_ATTEMPTS = 10;
9
6
  // const DEFAULT_FIRST_TIMEOUT = 3000;
10
7
  const createRedisClient = (args) => {
11
8
  const { redisHost, redisPort, redisPassword, redisSsl,
12
9
  //retryAttempts,
13
10
  //firstTimeout,
14
- pingInterval, connectionTimeout, } = args || {};
11
+ pingInterval, connectionTimeout, onConnecting, onConnected, onDisconnected, onReconnecting, onError, } = args || {};
15
12
  if (!redisHost) {
16
13
  return null;
17
14
  }
18
- const newRedisClient = redis_1.default.createClient({
19
- socket: Object.assign(Object.assign({ host: redisHost, port: Number(redisPort), family: 0, connectTimeout: connectionTimeout != undefined ? connectionTimeout : 5000, noDelay: true, keepAlive: pingInterval != undefined ? pingInterval : 5000 }, (redisSsl && { tls: true })), { rejectUnauthorized: false, reconnectStrategy: (retries) => Math.min(retries * 50, 500) }),
15
+ const cfg = {
16
+ socket: {
17
+ host: redisHost,
18
+ port: Number(redisPort),
19
+ family: 0,
20
+ connectTimeout: connectionTimeout != undefined ? connectionTimeout : 5000,
21
+ noDelay: true,
22
+ keepAlive: pingInterval != undefined ? pingInterval : 5000,
23
+ tls: redisSsl,
24
+ rejectUnauthorized: false,
25
+ reconnectStrategy: (retries) => Math.min(retries * 50, 500),
26
+ },
20
27
  password: redisPassword || undefined,
28
+ };
29
+ const redisClient = (0, redis_1.createClient)(cfg);
30
+ redisClient.on('connect', () => {
31
+ if (typeof onConnecting === 'function') {
32
+ onConnecting();
33
+ }
34
+ });
35
+ redisClient.on('ready', () => {
36
+ if (typeof onConnected === 'function') {
37
+ onConnected();
38
+ }
39
+ });
40
+ redisClient.on('end', () => {
41
+ if (typeof onDisconnected === 'function') {
42
+ onDisconnected();
43
+ }
44
+ });
45
+ redisClient.on('reconnecting', () => {
46
+ if (typeof onReconnecting === 'function') {
47
+ onReconnecting();
48
+ }
49
+ });
50
+ redisClient.on('error', (err) => {
51
+ if (typeof onError === 'function') {
52
+ onError(err);
53
+ }
21
54
  });
22
- // const newRedisClientOld = redis.createClient({
55
+ // const redisClientOld = redis.createClient({
23
56
  // host: redisHost,
24
57
  // port: Number(redisPort),
25
58
  // password: redisPassword || undefined,
@@ -56,9 +89,9 @@ const createRedisClient = (args) => {
56
89
  // });
57
90
  // if (pingInterval && !isNaN(pingInterval) && pingInterval > 1000) {
58
91
  // setInterval(async () => {
59
- // newRedisClient.ping();
92
+ // redisClient.ping();
60
93
  // }, pingInterval);
61
94
  // }
62
- return newRedisClient;
95
+ return redisClient;
63
96
  };
64
97
  exports.createRedisClient = createRedisClient;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sdflc/backend-helpers",
3
- "version": "1.0.50",
3
+ "version": "1.0.51",
4
4
  "description": "Set of helper functions and classes to simplify building backend apps within @sdflc ecosystem",
5
5
  "main": "build/index.js",
6
6
  "types": "build/index.d.ts",
@@ -32,20 +32,21 @@
32
32
  "dayjs": "^1.11.0",
33
33
  "dotenv-safe": "^8.2.0",
34
34
  "jest": "^27.5.1",
35
+ "json-stable-stringify": "^1.0.1",
35
36
  "jsonwebtoken": "^8.5.1",
36
37
  "knex": "^1.0.5",
37
38
  "lodash": "^4.17.21",
38
39
  "pg": "^8.7.3",
39
40
  "redis": "^4.0.6",
40
- "redis-dataloader": "^1.0.2",
41
41
  "rimraf": "^3.0.2",
42
- "uuid": "^8.3.2",
43
- "typescript": "^4.6.3"
42
+ "typescript": "^4.6.3",
43
+ "uuid": "^8.3.2"
44
44
  },
45
45
  "devDependencies": {
46
46
  "@types/bcrypt": "^5.0.0",
47
47
  "@types/bluebird": "^3.5.36",
48
48
  "@types/dotenv-safe": "^8.1.2",
49
+ "@types/json-stable-stringify": "^1.0.34",
49
50
  "@types/jsonwebtoken": "^8.5.8",
50
51
  "@types/lodash": "^4.14.181",
51
52
  "@types/node": "^14.18.12",