corebasic 1.0.211 → 1.0.213

package/dist/libs/features.js

@@ -0,0 +1,366 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import * as Dip from './dip.js';
+import * as Kafka from './kafka.js';
+import * as Utils from './utils.js';
+import * as Session from './session.js';
+import * as Messaging from './messaging.js';
+import axios from 'axios';
+import jwt from 'jsonwebtoken';
+let features = {};
+let apis = {};
+const getFeatureMethod = api => api.split(' ')[0].toLowerCase();
+const getFeatureUrl = api => api.split(' ')[1];
+const getFeature = name => features[name];
+const DEPLOY_TOKEN_SECRET = process.env.DEPLOY_TOKEN_SECRET || "MY_SECRET_DEPLOY_TOKEN";
+const SERVICE_ACCESS_TOKEN = jwt.sign({ app: process.env.APP_DEPLOYMENT_NAME }, DEPLOY_TOKEN_SECRET, { expiresIn: '365d' });
+const NDCURVE_DEVELOPER_LICENSE_ACCESS_TOKEN = await (async (_) => { if (process.env.NDCURVE_DEVELOPER_SERVICE)
+    try {
+        return (await Utils.fileToJson('file://', '/.ndcurve/developer.license.json')).accessToken?.trim();
+    }
+    catch { } })();
+async function loadLocalFeatures() {
+    let data = {};
+    if (process.env.LOAD_LOCAL_FEATURES) {
+        try {
+            let url = process.env.LOAD_LOCAL_FEATURES_URL ? `${process.env.LOAD_LOCAL_FEATURES_URL}/features` : 'https://slyp.app/slyp-dev/api/v1/features';
+            let result = (await axios.get(url, { data: {}, headers: { jwt: SERVICE_ACCESS_TOKEN, service: true, NDCURVE_DEVELOPER_LICENSE_ACCESS_TOKEN }, timeout: 3000 })).data;
+            await Utils.stringToFile('/slyp.local.features.json', JSON.stringify(result.data, null, '\t'));
+        }
+        catch {
+            console.log('Error Fetching Remote Features');
+        }
+        try {
+            data = JSON.parse(await Utils.fileToString('/slyp.local.features.json'));
+        }
+        catch {
+            console.log('Error Loading Remote Features');
+        }
+    }
+    return data;
+}
+let SLYP_FEATURES_LIST = await loadLocalFeatures();
+let appId = Utils.uid();
+let SERVICE_ADDRESS = process.env.APP_ENDPOINT || 'http://127.0.0.1:3000';
+export const get = feature => {
+    const throwError = () => { throw new Error(`Feature ${feature} not found in internal or external list during Features.subscribe call`); };
+    let { api, service = SERVICE_ADDRESS, topic } = getFeature(feature) ?? SLYP_FEATURES_LIST[feature] ?? throwError();
+    return { api, service, topic };
+};
+export const send = async (meta, feature, data, params) => {
+    const throwError = () => { throw new Error(`Feature ${feature} not found in internal or external list during inter feature call`); };
+    let { api, service = SERVICE_ADDRESS } = getFeature(feature) ?? SLYP_FEATURES_LIST[feature] ?? throwError();
+    let method = getFeatureMethod(api);
+    let url = getFeatureUrl(api);
+    if (params || url.includes(':')) {
+        for (let key in params)
+            url = url.replace(`:${key}`, params[key]);
+        if (url.includes(':'))
+            throw new Error(`Error: Internal feature call send on feature: ${feature} when api/params/substitution`);
+    }
+    let payload = { ...meta, data, feature, txn: Utils.uid() };
+    service = service.replace('http://slyp.app', 'https://slyp.app');
+    // return  (await axios[method](`${service}${url}`, { data: payload, headers: {jwt: headers.jwt}, timeout: 1000 })).data
+    // Used to work. But request to 127.0.0.1 fails to send the specified headers along with the request.
+    // return  (await axios[method](`${service}${url}`, payload, {headers: {jwt: SERVICE_ACCESS_TOKEN, service: true}, timeout: 1000 })).data // Worked Earlier, but issue spotted
+    if (getFeature(feature)) { // Local call
+        let response;
+        let req = { meta, body: payload, params, method };
+        req = JSON.parse(JSON.stringify(req));
+        let res = { json: payload => response = payload, end: _ => true };
+        await apiHandler(req, res);
+        return response;
+    }
+    // Workaround: https://stackoverflow.com/questions/72343387/axios-not-sending-headers-request-failing-getting-401-error
+    return (await axios({
+        method,
+        url: `${service}${url}`,
+        headers: { jwt: SERVICE_ACCESS_TOKEN, service: true, NDCURVE_DEVELOPER_LICENSE_ACCESS_TOKEN },
+        data: payload,
+        timeout: 30000 // 30 secs
+    })).data;
+};
+let appids = {};
+async function announce() {
+    let exp_features = {};
+    for (let [key, { api, subscribe }] of Object.entries(features))
+        exp_features[key] = { api, service: SERVICE_ADDRESS, subscribe };
+    await Messaging.subscribe(`${process.env.REDIS_CHANNEL_PREFIX}_SLYP_FEATURES_LIST`, async (message, channel) => {
+        let msg = JSON.parse(message);
+        if (msg.uid !== appId && !appids[msg.uid]) {
+            appids[msg.uid] = true;
+            SLYP_FEATURES_LIST = { ...SLYP_FEATURES_LIST, ...msg };
+            delete SLYP_FEATURES_LIST.uid;
+            await Messaging.produce(`${process.env.REDIS_CHANNEL_PREFIX}_SLYP_FEATURES_LIST`, JSON.stringify({ ...exp_features, uid: appId }));
+            await subscribe();
+        }
+    });
+    await Messaging.produce(`${process.env.REDIS_CHANNEL_PREFIX}_SLYP_FEATURES_LIST`, JSON.stringify({ ...exp_features, uid: appId }));
+}
+function getFeaturelessFeature(req) {
+    if (Session.ALLOWED_URLS.includes(req.path)) {
+        for (let key in features) {
+            if (features[key].featureless && features[key].api === req.method.toUpperCase() + ' ' + req.path) {
+                return getFeature(key);
+            }
+        }
+    }
+}
+const apiHandler = async (req, res) => {
+    let method = req.method.toLowerCase();
+    let featureless;
+    try {
+        let feature = getFeature(req.body?.feature);
+        featureless = feature ? undefined : (feature = getFeaturelessFeature(req));
+        if (!req.body?.feature && !featureless) {
+            console.warn(`Feature: feature not sent by client`);
+            throw { status: 404, message: "Resource not found. Feature not sent by client." };
+        }
+        if (!feature && !featureless) {
+            console.warn(`Feature: ${req.body.feature} not available`);
+            throw { status: 404, message: `Resource not found. Feature ${req.body.feature} not available.` };
+        }
+        let topic = feature.topic;
+        let params = getFeatureUrl(feature.api).split("/").filter(item => item.startsWith(":")).map(item => item.replace(":", ""));
+        for (let param of params)
+            if (!req.params[param])
+                throw { status: 404, message: "Resource not found. One or more url parameter not specified." };
+        let meta = { ...req.body, data: undefined };
+        req.meta = meta;
+        if (process.env.USE_DEFAULT_COMPANY) {
+            req.meta.company = 'DEFAULT_COMPANY';
+            req.meta.outlet = 'DEFAULT_OUTLET';
+        }
+        if (method === "get") {
+            try {
+                req.body = { ...req.body, topic };
+                await feature.handler(req, res);
+            }
+            catch (err) {
+                if (process.env.DEBUG_MODE)
+                    console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
+                throw { status: 500, message: "Failed to GET feature", ...err };
+            }
+        }
+        else if (method !== "get" && feature.bypass) {
+            try {
+                await feature.handler(topic, prepareMessage(req), req, res);
+            }
+            catch (err) {
+                if (process.env.DEBUG_MODE)
+                    console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
+                throw { status: 500, message: "Failed to POST feature", ...err };
+            }
+        }
+        else
+            await commandAction(req, res, topic);
+    }
+    catch (err) {
+        if (process.env.DEBUG_MODE)
+            console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
+        try { // Sometimes error occurs when disconnecting stream from front end
+            res.status(err.status ?? 500).json(err);
+        }
+        catch (_) { }
+    }
+};
+let ExpressApp;
+let PROJECT_ROOT_URL;
+export async function registerFeatures(newFeatures) {
+    // Registering handlers
+    await registerHandler(newFeatures);
+    features = { ...newFeatures, ...features };
+    // Registering apis
+    registerApi();
+}
+async function registerHandler(features) {
+    for (let name in features) {
+        let feature = features[name];
+        apis[feature.api] = apis[feature.api] ?? {};
+        apis[feature.api][name] = feature;
+        const featureName = name;
+        name = name.split('.');
+        let handler = name.pop();
+        name = `src/${name.join('/')}`;
+        if (name === 'src/transactions/query')
+            continue;
+        if (featureName.startsWith("flows.") && !featureName.startsWith("flows.query.") && !featureName.startsWith("flows.command.") && !featureName.startsWith("flows.designs."))
+            feature.handler = (await import(__rewriteRelativeImportExtension(`${new URL(`src/flows/query.js`, PROJECT_ROOT_URL).toString().replace('file://', '')}`))).FEATURE_HANDLERS[handler];
+        else
+            feature.handler = (await import(__rewriteRelativeImportExtension(`${new URL(`${name}.js`, PROJECT_ROOT_URL).toString().replace('file://', '')}`)))[handler];
+    }
+}
+function registerApi() {
+    for (let api in apis) {
+        let method = getFeatureMethod(api);
+        let url = api.split(' ')[1];
+        if (api === 'GET /transactions/:id')
+            continue;
+        ExpressApp[method](url, apiHandler);
+    }
+}
+export const start = async (app, url, file) => {
+    const globalMeta = Dip.globalMeta();
+    Dip.excludeCollectionConfig("users.txns");
+    Dip.excludeCollectionConfig("Features.txns");
+    ExpressApp = app;
+    PROJECT_ROOT_URL = url;
+    features = await Utils.fileToJson(url, file);
+    await announce();
+    app.get('/features', async (req, res) => {
+        let exp_features = {};
+        for (let [key, { api }] of Object.entries(features))
+            exp_features[key] = { api, service: `${SERVICE_ADDRESS}` };
+        if (process.env.LOAD_LOCAL_FEATURES) {
+            for (let key in SLYP_FEATURES_LIST)
+                SLYP_FEATURES_LIST[key].headers = { JWT: SERVICE_ACCESS_TOKEN, service: true, NDCURVE_DEVELOPER_LICENSE_ACCESS_TOKEN };
+        }
+        res.json({ data: { ...SLYP_FEATURES_LIST, ...exp_features } });
+    });
+    // Registering features
+    await registerFeatures(features);
+    // Get unique kafka topics list from features
+    let kafkaTopics = [];
+    for (let name in features) {
+        let feature = features[name];
+        if (feature.api.split(' ')[0].toLowerCase() === "get")
+            continue;
+        kafkaTopics = [...new Set(kafkaTopics.concat([feature.topic]))];
+    }
+    // Subscribe to each topic
+    for (let topic of kafkaTopics) {
+        Kafka.receive(`Features.${topic}`, async (topic, message) => {
+            if (Utils.isEmpty(message.meta?.company))
+                message.meta = { ...message.meta, company: "GLOBAL", outlet: "GLOBAL" };
+            if (!message?.feature)
+                return;
+            const timer = ms => new Promise(res => setTimeout(res, ms)); // A promise that resolves after "ms" Milliseconds
+            topic = topic.replace(/^.*Features./, '');
+            while (true) {
+                try {
+                    // const iso_date = new Date(message.date).toISOString()
+                    // // TODO: use $useChunks: [] once support is added in dip insert
+                    // await Dip.insert(globalMeta, `users.txns`, { _id: `${message.user}_${iso_date}_${message.txn}`, user: message.user, feature: message.feature, date: message.date, created: message.date, updated: message.date, status: "Queued" }, {idempotent: true}) // Can always update the status later
+                    // TODO: Avoid duplicate processing
+                    await features[message.feature].handler(topic, message);
+                    await Dip.insert(globalMeta, "Features.txns", { _id: message.txn, status: "Processed" }, { idempotent: true });
+                    break;
+                }
+                catch (err) {
+                    if (process.env.DEBUG_MODE)
+                        console.log('Error: Feature: ', message.feature, err);
+                    console.warn(`Feature: ${message.feature}, error in executing handler during Kafka.receive(topic: ${topic})`);
+                }
+                await timer(10000);
+            }
+        });
+    }
+    await subscribe();
+    // Transaction status api
+    app.get('/transactions/:id', async (req, res) => {
+        try {
+            let items = await Dip.query(globalMeta, "Features.txns", { _id: req.params.id });
+            if (items.length)
+                res.json({ data: { ...items[0], txn: items._id } });
+            else
+                throw false;
+        }
+        catch {
+            res.status(500).json({ message: "Failed to retreive data" });
+        }
+    });
+    // app.get('/users/:id/transactions', async (req, res) => {
+    //     try {
+    //         const idPrefix = `${req.params.id}_`
+    //         // TODO: use $useChunks: [] once support is added in dip insert
+    //         // TODO: remove $orderby: { date: -1 } and remove $orderby: { _id: -1 }
+    //         // res.json({ data: await Dip.query(globalMeta, `users.txns`, {}, { $rangeFrom: idPrefix, $rangeTo: idPrefix, $rangeDirection: -1, $rangeInclusive: true, $orderby: { date: -1 } }) })
+    //         res.json({ data: await Dip.query(globalMeta, `users.txns`, {}, { $rangeFrom: idPrefix, $rangeTo: idPrefix, $rangeDirection: -1, $rangeInclusive: true, $orderby: { _id: -1 } }) })
+    //     } catch {
+    //         res.status(500).json({ message: "Failed to retreive data" })
+    //     }
+    // })
+    try {
+        await Messaging.start(app);
+    }
+    catch {
+        console.log('Messaging failed to start. Maybe missing redis');
+    }
+};
+function prepareMessage(req) {
+    let txn = req.body.txn ?? Utils.uid();
+    let _id = req.body.data?._id ?? txn;
+    let { data, feature, app, user, outlet, company, client, version } = req.body;
+    let meta = req.meta;
+    let params = req.params;
+    return { data, params, meta, feature, app, user, outlet, company, client, version, txn, id: _id, date: new Date().getTime() };
+}
+const commandAction = async (req, res, topic) => {
+    let message = prepareMessage(req);
+    try {
+        await Kafka.send(`Features.${topic}`, message, message.user, { compression: Kafka.CompressionTypes.GZIP });
+    }
+    catch {
+        throw { status: 500, message: "Failed to queue the transaction" };
+    }
+    res.json({ data: { txn: message.txn, success: true, status: "Queued", featureQueued: true } });
+};
+let subscriptions = {
+// "coins.query.hello": "invoices.command.add",     // Consumer : Feature of Topic
+};
+let subscribed_consumers = {};
+async function subscribe() {
+    let Features = { get, send };
+    for (let [key, { subscribe }] of Object.entries(features))
+        if (!Utils.isEmpty(subscribe) && subscriptions[key] !== subscribe && subscribed_consumers[key]) {
+            try {
+                await subscribed_consumers[key].disconnect();
+            }
+            catch (_) { }
+            delete subscribed_consumers[key];
+        }
+    for (let key in SLYP_FEATURES_LIST)
+        if (!Utils.isEmpty(SLYP_FEATURES_LIST[key].subscribe) && subscriptions[key] !== SLYP_FEATURES_LIST[key].subscribe && subscribed_consumers[key]) {
+            try {
+                await subscribed_consumers[key].disconnect();
+            }
+            catch (_) { }
+            delete subscribed_consumers[key];
+        }
+    let exp_features = {};
+    for (let key in SLYP_FEATURES_LIST)
+        if (!Utils.isEmpty(SLYP_FEATURES_LIST[key].subscribe)) {
+            subscriptions[key] = SLYP_FEATURES_LIST[key].subscribe;
+        }
+    for (let [key, { subscribe }] of Object.entries(features))
+        if (!Utils.isEmpty(subscribe)) {
+            subscriptions[key] = subscribe;
+        }
+    for (let consumer in subscriptions) {
+        let publisher = subscriptions[consumer];
+        let { topic } = getFeature(publisher);
+        if (subscribed_consumers[consumer] || !getFeature(publisher))
+            continue;
+        let kafka_consumer = Kafka.receive(`Features.${topic}`, consumer, async (topic, message) => {
+            topic = topic.replace(/^.*Features./, '');
+            const timer = ms => new Promise(res => setTimeout(res, ms)); // A promise that resolves after "ms" Milliseconds
+            while (true) {
+                try {
+                    await Features.send(message.meta, consumer, message.data, message.params);
+                    break;
+                }
+                catch {
+                    console.warn(`Feature: ${consumer}, error in executing handler during Features.subscribe(topic: ${topic})`);
+                }
+                await timer(10000);
+            }
+        });
+        subscribed_consumers[consumer] = kafka_consumer;
+    }
+}

package/dist/libs/kafka.js

@@ -0,0 +1,121 @@
+import { Kafka, logLevel as KafkaLogLevel, CompressionTypes as KafkaCompressionTypes } from 'kafkajs';
+export const logLevel = KafkaLogLevel;
+export const CompressionTypes = KafkaCompressionTypes;
+let kafka, producer, admin;
+let appName;
+let topicPrefix = '';
+export const start = (arg) => {
+    arg = !arg && !process.env.APP_DEPLOYMENT_NAME ? { clientId: (global.app?.name ?? 'app') + '-dev', brokers: ['127.0.0.1:9092'], sasl: false, logLevel: logLevel.ERROR } : arg;
+    appName = arg?.clientId ?? (process.env.APP_DEPLOYMENT_NAME ?? (global.app?.name ?? 'app'));
+    topicPrefix = `${appName}.`;
+    kafka = new Kafka({
+        logLevel: arg?.logLevel ?? logLevel.INFO,
+        clientId: appName,
+        brokers: arg?.brokers ?? ['redpanda-0.redpanda.redpanda.svc.cluster.local:9093'], // ['107.155.108.78:9092']     ['127.0.0.1:29092']
+        sasl: arg?.sasl == false ? undefined : {
+            mechanism: 'SCRAM-SHA-512',
+            username: process.env.KAFKA_USERNAME,
+            password: process.env.KAFKA_PASSWORD
+        },
+    });
+    producer = kafka.producer({ allowAutoTopicCreation: true });
+    admin = kafka.admin();
+};
+export const createTopic = async (topic, partition, replicas) => {
+    topic = topicPrefix + topic;
+    return await admin.createTopics({ topics: [{ topic, numPartitions: partition ?? 1, replicationFactor: replicas ?? 1 }] });
+};
+// ===========
+// Consumer
+// ===========
+let consumers = [];
+const start_consumer = async function (topic, groupId, callback) {
+    callback = typeof groupId === "string" ? callback : groupId;
+    groupId = typeof groupId === "string" ? `${appName}.${groupId}` : `${appName}.${topic}`;
+    const consumer = kafka.consumer({ groupId: groupId });
+    consumers.push(consumer);
+    await consumer.connect();
+    await consumer.subscribe({ topic: topic, fromBeginning: true });
+    await consumer.run({
+        autoCommit: false,
+        eachMessage: async ({ topic, partition, message }) => {
+            // callback(topic, message.value.toString(), partition) // toString() returns array so won't parse if json.
+            let success;
+            try {
+                success = await callback(topic, JSON.parse(message.value), partition);
+            }
+            catch (ex) {
+                success = await callback(topic, message.value, partition);
+            }
+            await consumer.commitOffsets([{ topic, partition, offset: (Number(message.offset) + 1).toString() }]);
+        },
+    });
+    return consumer;
+};
+// ===========
+// Producer
+// ===========
+let producerInvoked = false;
+const start_producer = async function (topic, message, key, options) {
+    let isJson = Object.prototype.toString.call(message) === '[object Object]' || Object.prototype.toString.call(message) === '[object Array]';
+    if (!producerInvoked) {
+        await producer.connect();
+        producerInvoked = true;
+    }
+    let value = isJson ? JSON.stringify(message) : message;
+    await producer.send({
+        topic: topic,
+        ...(options ?? {}),
+        messages: [
+            key ? { key, value } : { value },
+        ],
+    });
+    // .then(console.log)
+    // .catch(e => console.error("[example/producer] " + e.message, e))
+    // await producer.disconnect()
+};
+// start_consumer('quickstart-events')
+// start_producer('quickstart-events', 'Hello KafkaJS user! Little')
+export const receive = async function (topic, groupId, callback) {
+    topic = topicPrefix + topic;
+    return (await start_consumer(topic, groupId, callback));
+};
+export const send = async function (topic, message, key, options) {
+    topic = topicPrefix + topic;
+    await start_producer(topic, message, key, options);
+};
+// Usage
+// -------
+// send('quickstart-events', { name: 'Muneer', age: 30 })
+// receive('quickstart-events', function (topic, message) {
+//     console.log(message)
+// })
+// const errorTypes = ['unhandledRejection', 'uncaughtException']
+// const signalTraps = ['SIGTERM', 'SIGINT', 'SIGUSR2']
+//
+// errorTypes.forEach(type => {
+//     process.on(type, async () => {
+//         try {
+//             console.log(`process.on ${type}`)
+//             if (producer)
+//                 await producer.disconnect()
+//             for (let i = 0; i < consumers.length; i++)
+//                 await consumers[i].disconnect()
+//             process.exit(0)
+//         } catch (_) {
+//             process.exit(1)
+//         }
+//     })
+// })
+// signalTraps.forEach(type => {
+//     process.once(type, async () => {
+//         try {
+//             if (producer)
+//                 await producer.disconnect()
+//             for (let i = 0; i < consumers.length; i++)
+//                 await consumers[i].disconnect()
+//         } finally {
+//             process.kill(process.pid, type)
+//         }
+//     })
+// })

package/dist/libs/messaging.js

@@ -0,0 +1,130 @@
+import axios from 'axios';
+import { createClient } from 'redis';
+import * as Utils from './utils.js';
+const url = process.env.REDIS_URL || "redis://localhost:6380";
+// url: 'redis://alice:foobared@localhost:6380'
+let consumer, publisher;
+let users_pool = {};
+async function disconnect(user, uid) {
+    if (!users_pool[user])
+        return;
+    let [{ res }] = users_pool[user].reqres.filter(item => item.uid === uid).concat([{}]);
+    res.end();
+    if (users_pool[user].listeners[uid]) {
+        if (users_pool[user].listeners.count == 1)
+            await consumer.unsubscribe(user, users_pool[user].listeners[uid]);
+        users_pool[user].listeners.count--;
+    }
+    users_pool[user].reqres = users_pool[user].reqres.filter(item => item.uid !== uid);
+    delete users_pool[user].listeners[uid];
+    if (!users_pool[user].listeners.count)
+        delete users_pool[user];
+    // await consumer.unsubscribe(user);
+}
+async function connect({ user, req, res, uid }) {
+    res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive' });
+    const listener = (message, channel) => {
+        if (users_pool[user]) {
+            try {
+                message = JSON.parse(message);
+            }
+            catch { }
+            for (let { req, res } of users_pool[user].reqres) {
+                res.write(JSON.stringify({ message, channel }) + "\n");
+                if (res.flush) // If compression enabled
+                    res.flush();
+            }
+        }
+    };
+    users_pool[user] = users_pool[user] ?? { reqres: [], listeners: { count: 0 } };
+    users_pool[user].reqres.push({ req, res, uid });
+    users_pool[user].listeners[uid] = listener;
+    if (!users_pool[user].listeners.count)
+        await consumer.subscribe(user, listener);
+    users_pool[user].listeners.count++;
+}
+export async function start(app) {
+    consumer = createClient({ url });
+    await consumer.connect();
+    publisher = createClient({ url });
+    await publisher.connect();
+    consumer.on('error', err => console.log('Redis Client Error', err));
+    app.get('/messages/:user', async (req, res) => {
+        let uid = req.body.client;
+        await connect({ user: req.params.user, req, res, uid });
+        req.on("close", async () => await disconnect(req.params.user, uid)); // request closed unexpectedly
+        req.on("end", async () => await disconnect(req.params.user, uid)); // request ended normally
+    });
+    app.get('/messages/disconnects/:user', async (req, res) => {
+        let uid = req.body.client;
+        if (users_pool[req.params.user])
+            await disconnect(req.params.user, uid);
+        res.json({});
+    });
+}
+export async function publish(channel, message) {
+    if (publisher && !Utils.isEmpty(channel)) {
+        try {
+            await publisher.publish(channel, typeof message === "object" ? JSON.stringify(message) : message);
+        }
+        catch {
+            console.log('Error: Publishing on redis channel', channel);
+        }
+    }
+}
+// Generic Functions
+// -----------------
+export async function newConsumer() {
+    let consumer = createClient({ url });
+    await consumer.connect();
+    return consumer;
+}
+export async function newProducer() {
+    let publisher = createClient({ url });
+    await publisher.connect();
+    return publisher;
+}
+let defaultConsumer = await newConsumer();
+let defaultProducer = await newProducer();
+export async function subscribe(channel, listener) {
+    await defaultConsumer.subscribe(channel, listener);
+}
+export async function produce(channel, message) {
+    await defaultProducer.publish(channel, message);
+}
+let defaultSetGetConsumer = await newConsumer();
+export async function set(key, value) {
+    await defaultSetGetConsumer.set(`${process.env.REDIS_CHANNEL_PREFIX}_${key}`, typeof value === 'object' ? JSON.stringify(value) : value);
+}
+export async function get(key) {
+    let value = await defaultSetGetConsumer.get(`${process.env.REDIS_CHANNEL_PREFIX}_${key}`);
+    try {
+        value = JSON.parse(value);
+    }
+    catch (err) { }
+    return value;
+}
+export async function del(key) {
+    await defaultSetGetConsumer.del(`${process.env.REDIS_CHANNEL_PREFIX}_${key}`);
+}
+export async function hset(key, subkey, value) {
+    await defaultSetGetConsumer.HSET(`${process.env.REDIS_CHANNEL_PREFIX}_${key}`, subkey, typeof value === 'object' ? JSON.stringify(value) : value);
+}
+export async function hget(key, subkey) {
+    let value = await defaultSetGetConsumer.HGET(`${process.env.REDIS_CHANNEL_PREFIX}_${key}`, subkey);
+    try {
+        value = JSON.parse(value);
+    }
+    catch (err) { }
+    return value;
+}
+export async function hdel(key, subkey) {
+    await defaultSetGetConsumer.HDEL(`${process.env.REDIS_CHANNEL_PREFIX}_${key}`, subkey);
+}
+// Hydrate
+export async function hydrate(callback) {
+    setTimeout(async (_) => {
+        let { user, company, items } = await callback();
+        await publish(user, { event: "RPC_hydrate", data: { company, items }, txn: Utils.uid() });
+    }, 100);
+}