corebasic 1.0.231 → 1.0.232

package/dist/libs/dip.js

@@ -15,6 +15,7 @@ export const companyMeta = Elabase.companyMeta;
 export const outletMeta = Elabase.outletMeta;
 export const customMeta = Elabase.customMeta;
 export const schema = Elabase.schema;
+export const DipMeta = Elabase.DipMeta;
 export const isDipMeta = Elabase.isDipMeta;
 export const excludeCollectionConfig = Elabase.excludeCollectionConfig;
 export const includeCollectionConfig = Elabase.includeCollectionConfig;

package/dist/libs/features.js

@@ -16,10 +16,13 @@ import axios from 'axios';
 // @ts-ignore
 import jwt from 'jsonwebtoken';
 let features = {};
-let apis = {};
-const getFeatureMethod = api => api.split(' ')[0].toLowerCase();
-const getFeatureUrl = api => api.split(' ')[1];
-const getFeature = name => features[name];
+let apis = {
+// <api>: {<feature>: FeatureEntry}
+};
+const getFeatureMethod = (api) => api.split(' ')[0].toLowerCase();
+const getFeatureUrl = (api) => { const url = api.split(' ')[1]; if (!url)
+    throw new Error(`Invalid Feature url in api: ${api}`); return url; };
+const getFeature = (name) => features[name];
 const DEPLOY_TOKEN_SECRET = process.env.DEPLOY_TOKEN_SECRET || "MY_SECRET_DEPLOY_TOKEN";
 const SERVICE_ACCESS_TOKEN = jwt.sign({ app: process.env.APP_DEPLOYMENT_NAME }, DEPLOY_TOKEN_SECRET, { expiresIn: '365d' });
 const NDCURVE_DEVELOPER_LICENSE_ACCESS_TOKEN = await (async () => { if (process.env.NDCURVE_DEVELOPER_SERVICE)
@@ -50,7 +53,7 @@ async function loadLocalFeatures() {
 let SLYP_FEATURES_LIST = await loadLocalFeatures();
 let appId = Utils.uid();
 let SERVICE_ADDRESS = process.env.APP_ENDPOINT || 'http://127.0.0.1:3000';
-export const get = feature => {
+export const get = (feature) => {
     const throwError = () => { throw new Error(`Feature ${feature} not found in internal or external list during Features.subscribe call`); };
     const baseFeature = getFeature(feature);
     let { api, service = SERVICE_ADDRESS, topic } = baseFeature ? baseFeature : (SLYP_FEATURES_LIST[feature] ?? throwError());
@@ -68,16 +71,27 @@ export const send = async (meta, feature, data, params) => {
         if (url.includes(':'))
             throw new Error(`Error: Internal feature call send on feature: ${feature} when api/params/substitution`);
     }
-    let payload = { ...meta, data, feature, txn: Utils.uid() };
+    const payload = { ...meta, data, feature, txn: Utils.uid() };
     service = service.replace('http://slyp.app', 'https://slyp.app');
     // return  (await axios[method](`${service}${url}`, { data: payload, headers: {jwt: headers.jwt}, timeout: 1000 })).data
     // Used to work. But request to 127.0.0.1 fails to send the specified headers along with the request.
     // return  (await axios[method](`${service}${url}`, payload, {headers: {jwt: SERVICE_ACCESS_TOKEN, service: true}, timeout: 1000 })).data // Worked Earlier, but issue spotted
-    if (getFeature(feature)) { // Local call
+    if (baseFeature) { // Local call
         let response;
-        let req = { meta, body: payload, params, method };
+        let req = { body: payload, params: params ?? {}, method, path: url, url: '' };
         req = JSON.parse(JSON.stringify(req));
-        let res = { json: payload => response = payload, end: () => true };
+        const callback = (payload) => { response = payload; };
+        const res = {
+            json: callback,
+            send: callback,
+            end: () => { },
+            status: (_status) => { return { send: callback, json: callback }; },
+            sendStatus: (_status) => { },
+            setHeader: (_header, _value) => { },
+            flushHeaders: () => { },
+            headersSent: false,
+            writableEnded: false,
+        };
         await apiHandler(req, res);
         return response;
     }
@@ -109,7 +123,8 @@ async function announce() {
 function getFeaturelessFeature(req) {
     if (Session.ALLOWED_URLS.includes(req.path)) {
         for (let key in features) {
-            if (features[key].featureless && features[key].api === req.method.toUpperCase() + ' ' + req.path) {
+            const feature = features[key];
+            if (feature.featureless && feature.api === req.method.toUpperCase() + ' ' + req.path) {
                 return getFeature(key);
             }
         }
@@ -129,44 +144,77 @@ const apiHandler = async (req, res) => {
             console.warn(`Feature: ${req.body.feature} not available`);
             throw { status: 404, message: `Resource not found. Feature ${req.body.feature} not available.` };
         }
+        feature = feature; // typescript validation with !
         let topic = feature.topic;
-        let params = getFeatureUrl(feature.api).split("/").filter(item => item.startsWith(":")).map(item => item.replace(":", ""));
+        let params = getFeatureUrl(feature.api).split("/").filter((item) => item.startsWith(":")).map((item) => item.replace(":", ""));
         for (let param of params)
             if (!req.params[param])
                 throw { status: 404, message: "Resource not found. One or more url parameter not specified." };
-        let meta = { ...req.body, data: undefined };
-        req.meta = meta;
+        let meta = { ...req.body, data: undefined }; // TODO: Must also include invoiceTxn
         if (process.env.USE_DEFAULT_COMPANY) {
-            req.meta.company = 'DEFAULT_COMPANY';
-            req.meta.outlet = 'DEFAULT_OUTLET';
+            meta.company = 'DEFAULT_COMPANY';
+            meta.outlet = 'DEFAULT_OUTLET';
         }
+        meta.txn = meta.txn ?? Utils.uid();
+        const message = {
+            meta,
+            data: req.body.data,
+            params: req.params,
+            // To be removed 1
+            date: new Date().getTime(),
+            // To be removed 2
+            topic,
+            feature: req.body.feature,
+            user: req.body.user,
+            txn: meta.txn,
+        };
+        const appReq = {
+            meta: message.meta,
+            body: message,
+            params: message.params,
+            method: method,
+            path: req.path,
+            url: req.url,
+            headers: req.headers,
+            query: req.query,
+            on: req.on,
+        };
+        const appRes = res;
         if (method === "get") {
             try {
-                req.body = { ...req.body, topic };
-                await feature.handler(req, res);
+                // await feature.handler({...req, headers: req.headers, body: {...req.body, topic} }, res)
+                // req.body = {...req.body, topic}
+                await feature.handler(appReq, appRes);
             }
             catch (err) {
                 if (process.env.DEBUG_MODE)
-                    console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
+                    console.error('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
                 throw { status: 500, message: "Failed to GET feature", ...err };
             }
         }
         else if (method !== "get" && feature.bypass) {
             try {
-                await feature.handler(topic, prepareMessage(req), req, res);
+                await feature.handler(topic, message, appReq, appRes);
             }
             catch (err) {
                 if (process.env.DEBUG_MODE)
-                    console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
+                    console.error('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
                 throw { status: 500, message: "Failed to POST feature", ...err };
             }
         }
-        else
-            await commandAction(req, res, topic);
+        else {
+            try {
+                await Kafka.send(`Features.${topic}`, message, message.user, { compression: Kafka.CompressionTypes.GZIP });
+            }
+            catch {
+                throw { status: 500, message: "Failed to queue the transaction" };
+            }
+            appRes.json({ data: { txn: message.txn, success: true, status: "Queued", featureQueued: true } });
+        }
     }
     catch (err) {
         if (process.env.DEBUG_MODE)
-            console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
+            console.error('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
         try { // Sometimes error occurs when disconnecting stream from front end
             res.status(err.status ?? 500).json(err);
         }
@@ -203,6 +251,8 @@ function registerApi() {
     for (let api in apis) {
         let method = getFeatureMethod(api);
         let url = api.split(' ')[1];
+        if (!url)
+            throw new Error(`No url path found in api ${api} during Features.registerApi()`);
         if (api === 'GET /transactions/:id')
             continue;
         ExpressApp[method](url, apiHandler);
@@ -219,10 +269,10 @@ export const start = async (app, url, file) => {
     app.get('/features', async (req, res) => {
         let exp_features = {};
         for (let [key, { api }] of Object.entries(features))
-            exp_features[key] = { api, service: `${SERVICE_ADDRESS}` };
+            exp_features[key] = { api, service: `${SERVICE_ADDRESS}`, subscribe: '', topic: '' };
         if (process.env.LOAD_LOCAL_FEATURES) {
             for (let key in SLYP_FEATURES_LIST)
-                SLYP_FEATURES_LIST[key].headers = { JWT: SERVICE_ACCESS_TOKEN, service: true, NDCURVE_DEVELOPER_LICENSE_ACCESS_TOKEN };
+                SLYP_FEATURES_LIST[key].headers = { JWT: SERVICE_ACCESS_TOKEN, service: true, NDCURVE_DEVELOPER_LICENSE_ACCESS_TOKEN }; // TODO: CRITICAL: Security issue
         }
         res.json({ data: { ...SLYP_FEATURES_LIST, ...exp_features } });
     });
@@ -230,20 +280,21 @@ export const start = async (app, url, file) => {
     await registerFeatures(features);
     // Get unique kafka topics list from features
     let kafkaTopics = [];
-    for (let name in features) {
+    for (const name in features) {
         let feature = features[name];
-        if (feature.api.split(' ')[0].toLowerCase() === "get")
+        if (feature.api.split(' ')[0]?.toLowerCase() === "get")
             continue;
         kafkaTopics = [...new Set(kafkaTopics.concat([feature.topic]))];
     }
     // Subscribe to each topic
     for (let topic of kafkaTopics) {
-        Kafka.receive(`Features.${topic}`, async (topic, message) => {
+        const groupId = '';
+        Kafka.receive(`Features.${topic}`, groupId, async (topic, message) => {
             if (Utils.isEmpty(message.meta?.company))
                 message.meta = { ...message.meta, company: "GLOBAL", outlet: "GLOBAL" };
             if (!message?.feature)
                 return;
-            const timer = ms => new Promise(res => setTimeout(res, ms)); // A promise that resolves after "ms" Milliseconds
+            const timer = (ms) => new Promise(res => setTimeout(res, ms)); // A promise that resolves after "ms" Milliseconds
             topic = topic.replace(/^.*Features./, '');
             while (true) {
                 try {
@@ -251,7 +302,12 @@ export const start = async (app, url, file) => {
                     // // TODO: use $useChunks: [] once support is added in dip insert
                     // await Dip.insert(globalMeta, `users.txns`, { _id: `${message.user}_${iso_date}_${message.txn}`, user: message.user, feature: message.feature, date: message.date, created: message.date, updated: message.date, status: "Queued" }, {idempotent: true}) // Can always update the status later
                     // TODO: Avoid duplicate processing
-                    await features[message.feature].handler(topic, message);
+                    const feature = features[message.feature];
+                    if (!feature) {
+                        console.warn(`Feature ${message.feature} missing, in executing handler during Kafka.receive(topic: ${topic})`);
+                        throw new Error(`Feature ${message.feature} missing, in executing handler during Kafka.receive(topic: ${topic})`);
+                    }
+                    await feature.handler(topic, message);
                     await Dip.insert(globalMeta, "Features.txns", { _id: message.txn, status: "Processed" }, { idempotent: true });
                     break;
                 }
@@ -296,31 +352,13 @@ export const start = async (app, url, file) => {
         console.log('Messaging failed to start. Maybe missing redis');
     }
 };
-function prepareMessage(req) {
-    let txn = req.body.txn ?? Utils.uid();
-    let _id = req.body.data?._id ?? txn;
-    let { data, feature, app, user, outlet, company, client, version } = req.body;
-    let meta = req.meta;
-    let params = req.params;
-    return { data, params, meta, feature, app, user, outlet, company, client, version, txn, id: _id, date: new Date().getTime() };
-}
-const commandAction = async (req, res, topic) => {
-    let message = prepareMessage(req);
-    try {
-        await Kafka.send(`Features.${topic}`, message, message.user, { compression: Kafka.CompressionTypes.GZIP });
-    }
-    catch {
-        throw { status: 500, message: "Failed to queue the transaction" };
-    }
-    res.json({ data: { txn: message.txn, success: true, status: "Queued", featureQueued: true } });
-};
 let subscriptions = {
 // "coins.query.hello": "invoices.command.add",     // Consumer : Feature of Topic
 };
 let subscribed_consumers = {};
 async function subscribe() {
     let Features = { get, send };
-    for (let [key, { subscribe }] of Object.entries(features))
+    for (let [key, { subscribe }] of Object.entries(features)) {
         if (!Utils.isEmpty(subscribe) && subscriptions[key] !== subscribe && subscribed_consumers[key]) {
             try {
                 await subscribed_consumers[key].disconnect();
@@ -328,30 +366,37 @@ async function subscribe() {
             catch (_) { }
             delete subscribed_consumers[key];
         }
-    for (let key in SLYP_FEATURES_LIST)
-        if (!Utils.isEmpty(SLYP_FEATURES_LIST[key].subscribe) && subscriptions[key] !== SLYP_FEATURES_LIST[key].subscribe && subscribed_consumers[key]) {
+    }
+    for (let key in SLYP_FEATURES_LIST) {
+        const feature = SLYP_FEATURES_LIST[key];
+        if (!Utils.isEmpty(feature?.subscribe) && subscriptions[key] !== feature?.subscribe && subscribed_consumers[key]) {
             try {
                 await subscribed_consumers[key].disconnect();
             }
             catch (_) { }
             delete subscribed_consumers[key];
         }
-    for (let key in SLYP_FEATURES_LIST)
-        if (!Utils.isEmpty(SLYP_FEATURES_LIST[key].subscribe)) {
-            subscriptions[key] = SLYP_FEATURES_LIST[key].subscribe;
+    }
+    for (let key in SLYP_FEATURES_LIST) {
+        const feature = SLYP_FEATURES_LIST[key];
+        if (!Utils.isEmpty(feature?.subscribe)) {
+            subscriptions[key] = feature.subscribe;
         }
-    for (let [key, { subscribe }] of Object.entries(features))
+    }
+    for (let [key, { subscribe }] of Object.entries(features)) {
         if (!Utils.isEmpty(subscribe)) {
             subscriptions[key] = subscribe;
         }
+    }
     for (let consumer in subscriptions) {
         let publisher = subscriptions[consumer];
-        let { topic } = getFeature(publisher);
-        if (subscribed_consumers[consumer] || !getFeature(publisher))
+        const publisherFeature = getFeature(publisher);
+        if (subscribed_consumers[consumer] || !publisherFeature)
             continue;
-        let kafka_consumer = Kafka.receive(`Features.${topic}`, consumer, async (topic, message) => {
+        let { topic } = publisherFeature;
+        let kafka_consumer_promise = Kafka.receive(`Features.${topic}`, consumer, async (topic, message) => {
             topic = topic.replace(/^.*Features./, '');
-            const timer = ms => new Promise(res => setTimeout(res, ms)); // A promise that resolves after "ms" Milliseconds
+            const timer = (ms) => new Promise(res => setTimeout(res, ms)); // A promise that resolves after "ms" Milliseconds
             while (true) {
                 try {
                     await Features.send(message.meta, consumer, message.data, message.params);
@@ -363,6 +408,11 @@ async function subscribe() {
                 await timer(10000);
             }
         });
-        subscribed_consumers[consumer] = kafka_consumer;
+        kafka_consumer_promise
+            .then(kafka_consumer => {
+            subscribed_consumers[consumer] = kafka_consumer;
+        }).catch(err => {
+            throw new Error("Error: Remote service to service kafka Feature subscription.");
+        });
     }
 }

package/dist/libs/kafka.js

@@ -33,23 +33,21 @@ export const createTopic = async (topic, partition, replicas) => {
 // ===========
 let consumers = [];
 const start_consumer = async function (topic, groupId, callback) {
-    callback = typeof groupId === "string" ? callback : groupId;
-    groupId = typeof groupId === "string" ? `${appName}.${groupId}` : `${appName}.${topic}`;
+    groupId = groupId ? `${appName}.${groupId}` : `${appName}.${topic}`;
     const consumer = kafka.consumer({ groupId: groupId });
     consumers.push(consumer);
     await consumer.connect();
     await consumer.subscribe({ topic: topic, fromBeginning: true });
-    const fn = callback;
     await consumer.run({
         autoCommit: false,
         eachMessage: async ({ topic, partition, message }) => {
-            // fn(topic, message.value.toString(), partition) // toString() returns array so won't parse if json.
+            // callback(topic, message.value.toString(), partition) // toString() returns array so won't parse if json.
             let success;
             try {
-                success = await fn(topic, JSON.parse(message.value), partition);
+                success = await callback(topic, JSON.parse(message.value), partition);
             }
             catch (ex) {
-                success = await fn(topic, message.value, partition);
+                success = await callback(topic, message.value, partition); // NOTE: Beware: Ensure Kafka.receive<T> can handle raw message type that is not json parseable
             }
             await consumer.commitOffsets([{ topic, partition, offset: (Number(message.offset) + 1).toString() }]);
         },
@@ -82,7 +80,7 @@ const start_producer = async function (topic, message, key, options) {
 // start_producer('quickstart-events', 'Hello KafkaJS user! Little')
 export const receive = async function (topic, groupId, callback) {
     topic = topicPrefix + topic;
-    return (await start_consumer(topic, groupId, callback));
+    return await start_consumer(topic, groupId, callback);
 };
 export const send = async function (topic, message, key, options) {
     topic = topicPrefix + topic;

package/libs/dip.ts

@@ -18,6 +18,8 @@ export const companyMeta = Elabase.companyMeta
 export const outletMeta = Elabase.outletMeta
 export const customMeta = Elabase.customMeta
 export const schema = Elabase.schema
+export type DipMeta = Elabase.DipMeta;
+export const DipMeta = Elabase.DipMeta;
 export const isDipMeta = Elabase.isDipMeta
 export const excludeCollectionConfig = Elabase.excludeCollectionConfig
 export const includeCollectionConfig = Elabase.includeCollectionConfig

package/libs/features.ts

@@ -8,11 +8,90 @@ import axios from 'axios'
 // @ts-ignore
 import jwt from 'jsonwebtoken'
 
-export type Message = unknown
-export type Req = unknown
-export type Res = unknown
+
+// ===================================
+
+type HttpMethod = "get" | "post" | "put" | "delete" | "patch" | "head" | "options" | "connect" | "trace"
+
+
+type FeatureMeta = {
+    feature: string,
+    app: string,
+    user: string,
+    outlet: string,
+    company: string,
+    staff: string,
+    client: string,
+    version: string,
+    txn: string,
+    // invoiceTxn?: string // TODO: Must also include invoiceTxn
+}
+type FeatureParams = Record<string, string>
+type FeatureMessage = {
+    meta: FeatureMeta
+    data: unknown
+    params: FeatureParams
+    // To be removed 1
+    date: number // used as message.date
+    // To be removed 2
+    topic: string // used as req.body.topic
+    feature: string // used as req.body.feature or message.feature
+    user: string    // used as req.body.user or message.user
+    txn: string     // used as message.txn
+
+
+    // client  // Only used as req.body.client in auth.ts, session.ts and messaging.ts
+    // version // Only used as req.body.version in auth.ts and session.ts
+    // outlet, company // Only used as req.body.outlet, req.body.company in session.ts
+    // app // Only used as req.body.app in auth.ts and session.ts
+}
+type PayloadMessage = FeatureMeta & {
+    data: unknown
+    params?: FeatureParams
+}
+
+// =========
+
+export type Req = {
+    meta: FeatureMeta
+    body: FeatureMessage
+    params: FeatureParams
+    method: string
+    path: string
+    url?: string
+    headers?: Record<string, string | string[] | undefined>
+    query?: Record<string, unknown>
+    on?: (event: string, callback: () => void) => void
+}
+
+type ResJson = (response: unknown) => void
+type ResSend = (response: unknown) => void;
+type ResStatus = (status: number) => {send: ResSend, json: ResJson};
+export type Res = {
+    send: ResSend
+    json: ResJson
+    end: (finalData?: string) => void
+    status: ResStatus
+    sendStatus: (status: number) => void
+    setHeader: (header: string, value: string) => void
+    flushHeaders: () => void
+    headersSent: boolean
+    writableEnded: boolean
+}
+
+
+// ========= Express Types ==========
+type ExpressRequest = Omit<Req, "meta"  | "body"> & { body: PayloadMessage }
+type ExpressResponse = Res
+type ExpressHandler = (req: ExpressRequest, res: ExpressResponse) => Promise<void>
+type ExpressApplication = { [Method in HttpMethod]: (path: string, handler: ExpressHandler) => unknown }
+// ===================================
+
+
+// export type Req = unknown
+// export type Res = unknown
 export type Query = (req: Req, res: Res) => Promise<void>
-export type Command = (topic: string, message: Message, req?: Req, res?: Res) => Promise<void>
+export type Command = (topic: string, message: FeatureMessage, req?: Req, res?: Res) => Promise<void>
 type Handler = Query | Command
 
 
@@ -28,11 +107,13 @@ type FeatureEntry = { api: string, topic: string, bypass: boolean, featureless:
 
 
 let features: Record<string, FeatureEntry> = {}
-let apis = {}
+let apis: Record<string, Record<string, FeatureEntry>> = {
+    // <api>: {<feature>: FeatureEntry}
+}
 
-const getFeatureMethod = api => api.split(' ')[0].toLowerCase()
-const getFeatureUrl = api => api.split(' ')[1]
-const getFeature = name => features[name]
+const getFeatureMethod = (api: string): HttpMethod => api.split(' ')[0]!.toLowerCase() as HttpMethod
+const getFeatureUrl = (api: string): string => { const url = api.split(' ')[1]; if (!url) throw new Error(`Invalid Feature url in api: ${api}`); return url}
+const getFeature = (name: string): FeatureEntry | undefined => features[name]
 
 const DEPLOY_TOKEN_SECRET = process.env.DEPLOY_TOKEN_SECRET || "MY_SECRET_DEPLOY_TOKEN"
 const SERVICE_ACCESS_TOKEN = jwt.sign({app: process.env.APP_DEPLOYMENT_NAME}, DEPLOY_TOKEN_SECRET, { expiresIn: '365d' });
@@ -63,39 +144,50 @@ let appId = Utils.uid()
 let SERVICE_ADDRESS = process.env.APP_ENDPOINT || 'http://127.0.0.1:3000'
 
 
-export const get = feature => {
+export const get = (feature: string) => {
     const throwError = () => {throw new Error(`Feature ${feature} not found in internal or external list during Features.subscribe call`)}
-    const baseFeature: FeatureEntry = getFeature(feature)
+    const baseFeature: FeatureEntry | undefined = getFeature(feature)
     let {api, service = SERVICE_ADDRESS, topic} = baseFeature ? baseFeature as FeatureEntry & {service: string} : (SLYP_FEATURES_LIST[feature] ?? throwError())
     return {api, service, topic}
 }
 
-export const send = async (meta, feature, data, params) => {
+export const send = async (meta: Partial<FeatureMeta>, feature: string, data?: unknown, params?: FeatureParams) => {
     const throwError = () => {throw new Error(`Feature ${feature} not found in internal or external list during inter feature call`)}
-    const baseFeature: FeatureEntry = getFeature(feature)
+    const baseFeature: FeatureEntry | undefined = getFeature(feature)
     let {api, service = SERVICE_ADDRESS} = baseFeature ? baseFeature as FeatureEntry & {service: string} : (SLYP_FEATURES_LIST[feature] ?? throwError())
     let method = getFeatureMethod(api)
     let url = getFeatureUrl(api)
 
     if (params || url.includes(':')) {
         for (let key in params)
-            url = url.replace(`:${key}`, params[key])
+            url = url.replace(`:${key}`, params[key]!)
         if (url.includes(':'))
             throw new Error(`Error: Internal feature call send on feature: ${feature} when api/params/substitution`)
     }
 
-    let payload = { ...meta, data, feature, txn: Utils.uid() }
+    const payload: PayloadMessage = { ...meta, data, feature, txn: Utils.uid() } as PayloadMessage
     service = service.replace('http://slyp.app', 'https://slyp.app')
 
     // return  (await axios[method](`${service}${url}`, { data: payload, headers: {jwt: headers.jwt}, timeout: 1000 })).data
 
     // Used to work. But request to 127.0.0.1 fails to send the specified headers along with the request.
     // return  (await axios[method](`${service}${url}`, payload, {headers: {jwt: SERVICE_ACCESS_TOKEN, service: true}, timeout: 1000 })).data // Worked Earlier, but issue spotted
-    if (getFeature(feature)) { // Local call
+    if (baseFeature) { // Local call
         let response;
-        let req = {meta,body: payload, params,method}
+        let req: ExpressRequest = { body: payload, params: params ?? {}, method, path: url, url: ''}
         req = JSON.parse(JSON.stringify(req))
-        let res = {json: payload => response = payload, end: () => true  }
+        const callback = (payload: unknown): void => { response = payload }
+        const res: ExpressResponse = {
+            json: callback,
+            send: callback,
+            end: () => {},
+            status: (_status: number) => { return {send: callback, json: callback} },
+            sendStatus: (_status: number) => {},
+            setHeader: (_header: string, _value: string) => {},
+            flushHeaders: () => {},
+            headersSent: false,
+            writableEnded: false,
+        }
         await apiHandler(req, res);
         return response
     }
@@ -109,7 +201,7 @@ export const send = async (meta, feature, data, params) => {
       })).data;
 }
 
-let appids = {}
+let appids: Record<string, boolean> = {}
 async function announce() {
     let exp_features: Record<string, RemoteFeatureEntry> = {}
     for (let [key, {api,subscribe}] of Object.entries(features))
@@ -128,17 +220,18 @@ async function announce() {
 }
 
 
-function getFeaturelessFeature(req) {
+function getFeaturelessFeature(req: ExpressRequest) {
     if (Session.ALLOWED_URLS.includes(req.path)) {
-        for(let key in features){
-            if (features[key].featureless && features[key].api === req.method.toUpperCase() + ' ' + req.path) {
+        for(let key in features) {
+            const feature = features[key]!
+            if (feature.featureless && feature.api === req.method.toUpperCase() + ' ' + req.path) {
                 return getFeature(key)
             }
         }
     }
 }
 
-const apiHandler = async (req, res) => {
+const apiHandler = async (req: ExpressRequest, res: ExpressResponse) => {
     let method = req.method.toLowerCase()
     let featureless
     try {
@@ -155,50 +248,89 @@ const apiHandler = async (req, res) => {
             throw { status: 404, message: `Resource not found. Feature ${req.body.feature} not available.` }
         }
 
+        feature = feature! // typescript validation with !
+
         let topic = feature.topic
-        let params = getFeatureUrl(feature.api).split("/").filter(item => item.startsWith(":")).map(item => item.replace(":", ""))
+        let params = getFeatureUrl(feature.api).split("/").filter((item: string) => item.startsWith(":")).map((item: string) => item.replace(":", ""))
         for (let param of params)
             if (!req.params[param])
                 throw { status: 404, message: "Resource not found. One or more url parameter not specified." }
 
-        let meta = {...req.body, data: undefined}
-        req.meta = meta
+        let meta = {...req.body, data: undefined} as FeatureMeta // TODO: Must also include invoiceTxn
+
         if (process.env.USE_DEFAULT_COMPANY) {
-            req.meta.company = 'DEFAULT_COMPANY'
-            req.meta.outlet = 'DEFAULT_OUTLET'
+            meta.company = 'DEFAULT_COMPANY'
+            meta.outlet = 'DEFAULT_OUTLET'
         }
 
+        meta.txn = meta.txn ?? Utils.uid()
+
+        const message: FeatureMessage = {
+            meta,
+            data: req.body.data,
+            params: req.params,
+
+            // To be removed 1
+            date: new Date().getTime(),
+
+            // To be removed 2
+            topic,
+            feature: req.body.feature,
+            user: req.body.user,
+            txn: meta.txn,
+        }
+
+        const appReq: Req = {
+            meta: message.meta,
+            body: message,
+            params: message.params,
+            method: method,
+            path: req.path,
+            url: req.url,
+            headers: req.headers,
+            query: req.query,
+            on: req.on,
+        }
+        const appRes: Res = res as Res
+
         if (method === "get") {
             try {
-                req.body = {...req.body, topic}
-                await (feature.handler as Query)(req, res)
-            } catch (err) {
+                // await feature.handler({...req, headers: req.headers, body: {...req.body, topic} }, res)
+                // req.body = {...req.body, topic}
+                await (feature.handler as Query)(appReq, appRes)
+            } catch (err: any) {
                 if (process.env.DEBUG_MODE)
-                    console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err)
+                    console.error('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err)
                 throw {status: 500, message: "Failed to GET feature", ...err}
             }
         } else if (method !== "get" && feature.bypass) {
             try {
-                await (feature.handler as Command)(topic, prepareMessage(req), req, res)
-            } catch (err) {
+                await (feature.handler as Command)(topic, message, appReq, appRes)
+            } catch (err: any) {
                 if (process.env.DEBUG_MODE)
-                    console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err)
+                    console.error('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err)
                 throw {status: 500, message: "Failed to POST feature", ...err}
             }
-        } else
-            await commandAction(req, res, topic)
-    } catch (err) {
+        } else {
+            try {
+                await Kafka.send(`Features.${topic}`, message, message.user, { compression: Kafka.CompressionTypes.GZIP })
+            } catch {
+                throw {status: 500, message: "Failed to queue the transaction"}
+            }
+            appRes.json({ data: { txn: message.txn, success: true, status: "Queued", featureQueued: true } })
+        }
+    } catch (err: any) {
         if (process.env.DEBUG_MODE)
-            console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err)
+            console.error('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err)
         try { // Sometimes error occurs when disconnecting stream from front end
             res.status(err.status ?? 500).json(err)
         } catch (_) { }
     }
 }
 
-let ExpressApp
-let PROJECT_ROOT_URL
-export async function registerFeatures(newFeatures) {
+let ExpressApp: ExpressApplication
+let PROJECT_ROOT_URL: URL
+export async function registerFeatures(newFeatures: Record<string, FeatureEntry>) {
     // Registering handlers
     await registerHandler(newFeatures)
 
@@ -207,15 +339,15 @@ export async function registerFeatures(newFeatures) {
     // Registering apis
     registerApi()
 }
-async function registerHandler(features) {
+async function registerHandler(features: Record<string, FeatureEntry>) {
     for (let name in features) {
-        let feature = features[name]
+        let feature = features[name]!
         apis[feature.api] = apis[feature.api] ?? {}
-        apis[feature.api][name] = feature
+        apis[feature.api]![name] = feature
         const featureName = name
 
         const nameParts = name.split('.')
-        let handler = nameParts.pop()
+        let handler = nameParts.pop()!
         const featurePath = `src/${nameParts.join('/')}`
 
         if (featurePath === 'src/transactions/query')
@@ -229,8 +361,10 @@ async function registerHandler(features) {
 }
 function registerApi() {
     for (let api in apis) {
-        let method = getFeatureMethod(api)
+        let method: HttpMethod = getFeatureMethod(api)
         let url = api.split(' ')[1]
+        if (!url)
+            throw new Error(`No url path found in api ${api} during Features.registerApi()`)
         if (api === 'GET /transactions/:id')
             continue
 
@@ -238,22 +372,22 @@ function registerApi() {
     }
 }
 
-export const start = async (app, url, file) => {
+export const start = async (app: ExpressApplication, url: URL, file: string) => {
     const globalMeta = Dip.globalMeta()
     Dip.excludeCollectionConfig("users.txns")
     Dip.excludeCollectionConfig("Features.txns")
     ExpressApp = app
     PROJECT_ROOT_URL = url
-    features = await Utils.fileToJson(url, file)
+    features = await Utils.fileToJson(url, file) as Record<string, FeatureEntry>
     await announce()
 
-    app.get('/features', async (req, res) => {
-        let exp_features = {}
+    app.get('/features', async (req: ExpressRequest, res: ExpressResponse) => {
+        let exp_features: Record<string, RemoteFeatureEntry> = {}
         for (let [key, {api}] of Object.entries(features))
-            exp_features[key] = {api, service: `${SERVICE_ADDRESS}`}
+            exp_features[key] = {api, service: `${SERVICE_ADDRESS}`, subscribe: '', topic: ''}
         if (process.env.LOAD_LOCAL_FEATURES) {
             for (let key in SLYP_FEATURES_LIST)
-                SLYP_FEATURES_LIST[key].headers = {JWT: SERVICE_ACCESS_TOKEN, service: true, NDCURVE_DEVELOPER_LICENSE_ACCESS_TOKEN}
+                SLYP_FEATURES_LIST[key]!.headers = {JWT: SERVICE_ACCESS_TOKEN, service: true, NDCURVE_DEVELOPER_LICENSE_ACCESS_TOKEN}  // TODO: CRITICAL: Security issue
         }
         res.json({ data: { ...SLYP_FEATURES_LIST, ...exp_features } })
     })
@@ -262,23 +396,24 @@ export const start = async (app, url, file) => {
     await registerFeatures(features)
 
     // Get unique kafka topics list from features
-    let kafkaTopics = []
-    for (let name in features) {
-        let feature = features[name]
-        if (feature.api.split(' ')[0].toLowerCase() === "get" )
+    let kafkaTopics: string[] = []
+    for (const name in features) {
+        let feature = features[name]!
+        if (feature.api.split(' ')[0]?.toLowerCase() === "get" )
             continue
         kafkaTopics = [... new Set(kafkaTopics.concat([feature.topic]))]
     }
 
     // Subscribe to each topic
     for (let topic of kafkaTopics) {
-        Kafka.receive(`Features.${topic}`, async (topic, message) => {
+        const groupId = ''
+        Kafka.receive<FeatureMessage>(`Features.${topic}`, groupId, async (topic: string, message: FeatureMessage) => {
             if (Utils.isEmpty(message.meta?.company))
                 message.meta = { ...message.meta, company: "GLOBAL", outlet: "GLOBAL" }
             if (!message?.feature)
                 return
 
-            const timer = ms => new Promise(res => setTimeout(res, ms)) // A promise that resolves after "ms" Milliseconds
+            const timer = (ms: number) => new Promise(res => setTimeout(res, ms)) // A promise that resolves after "ms" Milliseconds
             topic = topic.replace(/^.*Features./,'')
             while (true) {
                 try {
@@ -286,7 +421,12 @@ export const start = async (app, url, file) => {
                     // // TODO: use $useChunks: [] once support is added in dip insert
                     // await Dip.insert(globalMeta, `users.txns`, { _id: `${message.user}_${iso_date}_${message.txn}`, user: message.user, feature: message.feature, date: message.date, created: message.date, updated: message.date, status: "Queued" }, {idempotent: true}) // Can always update the status later
                     // TODO: Avoid duplicate processing
-                    await (features[message.feature].handler as Command)(topic, message)
+                    const feature = features[message.feature]
+                    if (!feature) {
+                        console.warn(`Feature ${message.feature} missing, in executing handler during Kafka.receive(topic: ${topic})`)
+                        throw new Error(`Feature ${message.feature} missing, in executing handler during Kafka.receive(topic: ${topic})`)
+                    }
+                    await (feature.handler as Command)(topic, message)
                     await Dip.insert(globalMeta, "Features.txns", {_id: message.txn, status: "Processed"}, {idempotent: true})
                     break
                 } catch(err) {
@@ -302,9 +442,9 @@ export const start = async (app, url, file) => {
     await subscribe()
 
     // Transaction status api
-    app.get('/transactions/:id', async (req, res) => { // Front end calls this as a regular feature call. so `req.meta` exists
+    app.get('/transactions/:id', async (req: ExpressRequest, res: ExpressResponse) => { // Front end calls this as a regular feature call. so `req.meta` exists
         try {
-            let items = await Dip.query(globalMeta, "Features.txns", {_id: req.params.id})
+            let items = await Dip.query(globalMeta, "Features.txns", {_id: req.params!.id})
             if (items.length)
                 res.json({ data: { ...items[0], txn: items._id } })
             else
@@ -325,70 +465,61 @@ export const start = async (app, url, file) => {
     //     }
     // })
 
-    try { await Messaging.start(app) } catch { console.log('Messaging failed to start. Maybe missing redis') }
+    try { await Messaging.start((app as unknown) as Messaging.ExpressApp ) } catch { console.log('Messaging failed to start. Maybe missing redis') }
 }
 
-function prepareMessage(req) {
-    let txn = req.body.txn ?? Utils.uid()
-    let _id = req.body.data?._id ?? txn
-    let {data, feature, app, user, outlet, company, client, version} = req.body
-    let meta = req.meta
-    let params = req.params
-    return { data, params, meta, feature, app, user, outlet, company, client, version, txn, id: _id, date: new Date().getTime() }
-}
-const commandAction = async (req, res, topic) => {
-    let message = prepareMessage(req)
-    try {
-        await Kafka.send(`Features.${topic}`, message, message.user, { compression: Kafka.CompressionTypes.GZIP })
-    } catch {
-        throw {status: 500, message: "Failed to queue the transaction"}
-    }
-    res.json({ data: { txn: message.txn, success: true, status: "Queued", featureQueued: true } })
-}
 
 
 
-let subscriptions = {
+let subscriptions: Record<string, string> = {
     // "coins.query.hello": "invoices.command.add",     // Consumer : Feature of Topic
 }
-let subscribed_consumers = {
-}
+let subscribed_consumers: Record<string, Kafka.KafkaConsumer> = {}
 async function subscribe() {
     let Features = {get,send}
 
-    for (let [key, {subscribe}] of Object.entries(features))
+    for (let [key, {subscribe}] of Object.entries(features)) {
         if (!Utils.isEmpty(subscribe) && subscriptions[key] !== subscribe && subscribed_consumers[key]) {
             try { await subscribed_consumers[key].disconnect() } catch (_) {}
             delete subscribed_consumers[key]
         }
-    for (let key in SLYP_FEATURES_LIST)
-        if (!Utils.isEmpty(SLYP_FEATURES_LIST[key].subscribe) && subscriptions[key] !==  SLYP_FEATURES_LIST[key].subscribe && subscribed_consumers[key]) {
+    }
+    for (let key in SLYP_FEATURES_LIST) {
+        const feature = SLYP_FEATURES_LIST[key]
+        if (!Utils.isEmpty(feature?.subscribe) && subscriptions[key] !==  feature?.subscribe && subscribed_consumers[key]) {
             try { await subscribed_consumers[key].disconnect() } catch (_) {}
             delete subscribed_consumers[key]
         }
+    }
 
 
-    for (let key in SLYP_FEATURES_LIST)
-        if (!Utils.isEmpty(SLYP_FEATURES_LIST[key].subscribe)) {
-            subscriptions[key] = SLYP_FEATURES_LIST[key].subscribe
+    for (let key in SLYP_FEATURES_LIST) {
+        const feature = SLYP_FEATURES_LIST[key]
+        if (!Utils.isEmpty(feature?.subscribe)) {
+            subscriptions[key] = feature!.subscribe
         }
-    for (let [key, {subscribe}] of Object.entries(features))
+    }
+    for (let [key, {subscribe}] of Object.entries(features)) {
         if (!Utils.isEmpty(subscribe)) {
             subscriptions[key] = subscribe
         }
+    }
 
 
     for (let consumer in subscriptions) {
-        let publisher = subscriptions[consumer]
-        let {topic} = getFeature(publisher)
+        let publisher = subscriptions[consumer]!
+
+        const publisherFeature = getFeature(publisher)
 
-        if (subscribed_consumers[consumer] || !getFeature(publisher))
+        if (subscribed_consumers[consumer] || !publisherFeature)
             continue
 
-        let kafka_consumer = Kafka.receive(`Features.${topic}`, consumer, async (topic, message) => {
+        let {topic} = publisherFeature
+
+        let kafka_consumer_promise = Kafka.receive<FeatureMessage>(`Features.${topic}`, consumer, async (topic: string, message: FeatureMessage) => {
             topic = topic.replace(/^.*Features./,'')
 
-            const timer = ms => new Promise(res => setTimeout(res, ms)) // A promise that resolves after "ms" Milliseconds
+            const timer = (ms: number): Promise<void> => new Promise(res => setTimeout(res, ms)) // A promise that resolves after "ms" Milliseconds
 
             while (true) {
                 try {
@@ -400,6 +531,12 @@ async function subscribe() {
                 await timer(10000);
             }
         })
-        subscribed_consumers[consumer] = kafka_consumer
+
+        kafka_consumer_promise
+        .then(kafka_consumer => {
+            subscribed_consumers[consumer] = kafka_consumer
+        }).catch(err => {
+            throw new Error("Error: Remote service to service kafka Feature subscription.")
+        })
     }
 }

package/libs/kafka.ts

@@ -5,10 +5,11 @@ export const logLevel = KafkaLogLevel
 export const CompressionTypes = KafkaCompressionTypes
 
 
-type KafkaCallback = (topic: string, message: unknown, partition: number) => unknown | Promise<unknown>
+export type KafkaCallback<T = unknown> = (topic: string, message: T, partition: number) => unknown | Promise<unknown>
 
-type KafkaConsumer = {
+export type KafkaConsumer = {
     connect: () => Promise<void>
+    disconnect: () => Promise<void>
     subscribe: (arg: {topic: string; fromBeginning: boolean}) => Promise<void>
     run: (arg: {
         autoCommit: boolean
@@ -100,27 +101,24 @@ export const createTopic = async (topic: string, partition?: number, replicas?:
 
 let consumers = []
 
-const start_consumer = async function (topic: string, groupId: string | KafkaCallback, callback?: KafkaCallback) {
-    callback = typeof groupId === "string" ? callback : groupId
-    groupId = typeof groupId === "string" ? `${appName}.${groupId}` : `${appName}.${topic}`
+const start_consumer = async function<T>(topic: string, groupId: string, callback: KafkaCallback<T>): Promise<KafkaConsumer> {
+    groupId = groupId ? `${appName}.${groupId}` : `${appName}.${topic}`
 
-    const consumer = kafka.consumer({ groupId: groupId })
+    const consumer: KafkaConsumer = kafka.consumer({ groupId: groupId })
     consumers.push(consumer)
 
     await consumer.connect()
     await consumer.subscribe({ topic: topic, fromBeginning: true })
 
-    const fn = callback!
-
     await consumer.run({
         autoCommit: false,
         eachMessage: async ({ topic, partition, message }) => {
-            // fn(topic, message.value.toString(), partition) // toString() returns array so won't parse if json.
+            // callback(topic, message.value.toString(), partition) // toString() returns array so won't parse if json.
             let success
             try {
-                success = await fn(topic, JSON.parse(message.value as string), partition)
+                success = await callback(topic, JSON.parse(message.value as string), partition)
             } catch (ex) {
-                success = await fn(topic, message.value, partition)
+                success = await callback(topic, message.value as T, partition) // NOTE: Beware: Ensure Kafka.receive<T> can handle raw message type that is not json parseable
             }
 
             await consumer.commitOffsets([{ topic, partition, offset: (Number(message.offset) + 1).toString() }]);
@@ -160,9 +158,9 @@ const start_producer = async function (topic: string, message: object, key: stri
 // start_consumer('quickstart-events')
 // start_producer('quickstart-events', 'Hello KafkaJS user! Little')
 
-export const receive = async function(topic: string, groupId: string, callback?: KafkaCallback) {
+export const receive = async function<T>(topic: string, groupId: string, callback: KafkaCallback<T>): Promise<KafkaConsumer> {
     topic = topicPrefix + topic
-    return (await start_consumer(topic, groupId, callback))
+    return await start_consumer(topic, groupId, callback)
 }
 
 

package/libs/messaging.ts

@@ -59,7 +59,7 @@ type UserPool = {
     listeners: UserListeners
 }
 
-type ExpressApp = {
+export type ExpressApp = {
     get: (
         path: string,
         handler: (req: Request, res: Response) => Promise<void>

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "corebasic",
   "type": "module",
-  "version": "1.0.231",
+  "version": "1.0.232",
   "description": "",
   "main": "dist/index.js",
   "types": "./index.ts",