corebasic 1.0.231 → 1.0.233

package/dist/libs/features.js

@@ -16,10 +16,13 @@ import axios from 'axios';
 // @ts-ignore
 import jwt from 'jsonwebtoken';
 let features = {};
-let apis = {};
-const getFeatureMethod = api => api.split(' ')[0].toLowerCase();
-const getFeatureUrl = api => api.split(' ')[1];
-const getFeature = name => features[name];
+let apis = {
+// <api>: {<feature>: FeatureEntry}
+};
+const getFeatureMethod = (api) => api.split(' ')[0].toLowerCase();
+const getFeatureUrl = (api) => { const url = api.split(' ')[1]; if (!url)
+    throw new Error(`Invalid Feature url in api: ${api}`); return url; };
+const getFeature = (name) => features[name];
 const DEPLOY_TOKEN_SECRET = process.env.DEPLOY_TOKEN_SECRET || "MY_SECRET_DEPLOY_TOKEN";
 const SERVICE_ACCESS_TOKEN = jwt.sign({ app: process.env.APP_DEPLOYMENT_NAME }, DEPLOY_TOKEN_SECRET, { expiresIn: '365d' });
 const NDCURVE_DEVELOPER_LICENSE_ACCESS_TOKEN = await (async () => { if (process.env.NDCURVE_DEVELOPER_SERVICE)
@@ -50,7 +53,7 @@ async function loadLocalFeatures() {
 let SLYP_FEATURES_LIST = await loadLocalFeatures();
 let appId = Utils.uid();
 let SERVICE_ADDRESS = process.env.APP_ENDPOINT || 'http://127.0.0.1:3000';
-export const get = feature => {
+export const get = (feature) => {
     const throwError = () => { throw new Error(`Feature ${feature} not found in internal or external list during Features.subscribe call`); };
     const baseFeature = getFeature(feature);
     let { api, service = SERVICE_ADDRESS, topic } = baseFeature ? baseFeature : (SLYP_FEATURES_LIST[feature] ?? throwError());
@@ -68,16 +71,27 @@ export const send = async (meta, feature, data, params) => {
         if (url.includes(':'))
             throw new Error(`Error: Internal feature call send on feature: ${feature} when api/params/substitution`);
     }
-    let payload = { ...meta, data, feature, txn: Utils.uid() };
+    const payload = { ...meta, data, feature, txn: Utils.uid() };
     service = service.replace('http://slyp.app', 'https://slyp.app');
     // return  (await axios[method](`${service}${url}`, { data: payload, headers: {jwt: headers.jwt}, timeout: 1000 })).data
     // Used to work. But request to 127.0.0.1 fails to send the specified headers along with the request.
     // return  (await axios[method](`${service}${url}`, payload, {headers: {jwt: SERVICE_ACCESS_TOKEN, service: true}, timeout: 1000 })).data // Worked Earlier, but issue spotted
-    if (getFeature(feature)) { // Local call
+    if (baseFeature) { // Local call
         let response;
-        let req = { meta, body: payload, params, method };
+        let req = { body: payload, params: params ?? {}, method, path: url, url: '', headers: {}, query: {}, on: (_event, _callback) => { } };
         req = JSON.parse(JSON.stringify(req));
-        let res = { json: payload => response = payload, end: () => true };
+        const callback = (payload) => { response = payload; };
+        const res = {
+            json: callback,
+            send: callback,
+            end: () => { },
+            status: (_status) => { return { send: callback, json: callback }; },
+            sendStatus: (_status) => { },
+            setHeader: (_header, _value) => { },
+            flushHeaders: () => { },
+            headersSent: false,
+            writableEnded: false,
+        };
         await apiHandler(req, res);
         return response;
     }
@@ -95,7 +109,7 @@ async function announce() {
     let exp_features = {};
     for (let [key, { api, subscribe }] of Object.entries(features))
         exp_features[key] = { api, service: SERVICE_ADDRESS, subscribe, topic: "" };
-    await Messaging.subscribe(`${process.env.REDIS_CHANNEL_PREFIX}_SLYP_FEATURES_LIST`, async (message, channel) => {
+    await Messaging.subscribe(`${process.env.REDIS_CHANNEL_PREFIX}_SLYP_FEATURES_LIST`, async (message, _channel) => {
         let { uid, ...msg } = JSON.parse(message);
         if (uid !== appId && !appids[uid]) {
             appids[uid] = true;
@@ -109,11 +123,13 @@ async function announce() {
 function getFeaturelessFeature(req) {
     if (Session.ALLOWED_URLS.includes(req.path)) {
         for (let key in features) {
-            if (features[key].featureless && features[key].api === req.method.toUpperCase() + ' ' + req.path) {
+            const feature = features[key];
+            if (feature.featureless && feature.api === req.method.toUpperCase() + ' ' + req.path) {
                 return getFeature(key);
             }
         }
     }
+    return undefined;
 }
 const apiHandler = async (req, res) => {
     let method = req.method.toLowerCase();
@@ -129,44 +145,77 @@ const apiHandler = async (req, res) => {
             console.warn(`Feature: ${req.body.feature} not available`);
             throw { status: 404, message: `Resource not found. Feature ${req.body.feature} not available.` };
         }
+        feature = feature; // typescript validation with !
         let topic = feature.topic;
-        let params = getFeatureUrl(feature.api).split("/").filter(item => item.startsWith(":")).map(item => item.replace(":", ""));
+        let params = getFeatureUrl(feature.api).split("/").filter((item) => item.startsWith(":")).map((item) => item.replace(":", ""));
         for (let param of params)
             if (!req.params[param])
                 throw { status: 404, message: "Resource not found. One or more url parameter not specified." };
-        let meta = { ...req.body, data: undefined };
-        req.meta = meta;
+        let meta = { ...req.body, data: undefined }; // TODO: Must also include invoiceTxn
         if (process.env.USE_DEFAULT_COMPANY) {
-            req.meta.company = 'DEFAULT_COMPANY';
-            req.meta.outlet = 'DEFAULT_OUTLET';
+            meta.company = 'DEFAULT_COMPANY';
+            meta.outlet = 'DEFAULT_OUTLET';
         }
+        meta.txn = meta.txn ?? Utils.uid();
+        const message = {
+            meta,
+            data: req.body.data,
+            params: req.params,
+            // To be removed 1
+            date: new Date().getTime(),
+            // To be removed 2
+            topic,
+            feature: req.body.feature,
+            user: req.body.user,
+            txn: meta.txn,
+        };
+        const appReq = {
+            meta: message.meta,
+            body: message,
+            params: message.params,
+            method: method,
+            path: req.path,
+            url: req.url,
+            headers: req.headers,
+            query: req.query,
+            on: req.on,
+        };
+        const appRes = res;
         if (method === "get") {
             try {
-                req.body = { ...req.body, topic };
-                await feature.handler(req, res);
+                // await feature.handler({...req, headers: req.headers, body: {...req.body, topic} }, res)
+                // req.body = {...req.body, topic}
+                await feature.handler(appReq, appRes);
             }
             catch (err) {
                 if (process.env.DEBUG_MODE)
-                    console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
+                    console.error('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
                 throw { status: 500, message: "Failed to GET feature", ...err };
             }
         }
         else if (method !== "get" && feature.bypass) {
             try {
-                await feature.handler(topic, prepareMessage(req), req, res);
+                await feature.handler(topic, message, appReq, appRes);
             }
             catch (err) {
                 if (process.env.DEBUG_MODE)
-                    console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
+                    console.error('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
                 throw { status: 500, message: "Failed to POST feature", ...err };
             }
         }
-        else
-            await commandAction(req, res, topic);
+        else {
+            try {
+                await Kafka.send(`Features.${topic}`, message, message.user, { compression: Kafka.CompressionTypes.GZIP });
+            }
+            catch {
+                throw { status: 500, message: "Failed to queue the transaction" };
+            }
+            appRes.json({ data: { txn: message.txn, success: true, status: "Queued", featureQueued: true } });
+        }
     }
     catch (err) {
         if (process.env.DEBUG_MODE)
-            console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
+            console.error('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err);
         try { // Sometimes error occurs when disconnecting stream from front end
             res.status(err.status ?? 500).json(err);
         }
@@ -203,6 +252,8 @@ function registerApi() {
     for (let api in apis) {
         let method = getFeatureMethod(api);
         let url = api.split(' ')[1];
+        if (!url)
+            throw new Error(`No url path found in api ${api} during Features.registerApi()`);
         if (api === 'GET /transactions/:id')
             continue;
         ExpressApp[method](url, apiHandler);
@@ -216,13 +267,13 @@ export const start = async (app, url, file) => {
     PROJECT_ROOT_URL = url;
     features = await Utils.fileToJson(url, file);
     await announce();
-    app.get('/features', async (req, res) => {
+    app.get('/features', async (_req, res) => {
         let exp_features = {};
         for (let [key, { api }] of Object.entries(features))
-            exp_features[key] = { api, service: `${SERVICE_ADDRESS}` };
+            exp_features[key] = { api, service: `${SERVICE_ADDRESS}`, subscribe: '', topic: '' };
         if (process.env.LOAD_LOCAL_FEATURES) {
             for (let key in SLYP_FEATURES_LIST)
-                SLYP_FEATURES_LIST[key].headers = { JWT: SERVICE_ACCESS_TOKEN, service: true, NDCURVE_DEVELOPER_LICENSE_ACCESS_TOKEN };
+                SLYP_FEATURES_LIST[key].headers = { JWT: SERVICE_ACCESS_TOKEN, service: true, NDCURVE_DEVELOPER_LICENSE_ACCESS_TOKEN }; // TODO: CRITICAL: Security issue
         }
         res.json({ data: { ...SLYP_FEATURES_LIST, ...exp_features } });
     });
@@ -230,20 +281,21 @@ export const start = async (app, url, file) => {
     await registerFeatures(features);
     // Get unique kafka topics list from features
     let kafkaTopics = [];
-    for (let name in features) {
+    for (const name in features) {
         let feature = features[name];
-        if (feature.api.split(' ')[0].toLowerCase() === "get")
+        if (feature.api.split(' ')[0]?.toLowerCase() === "get")
             continue;
         kafkaTopics = [...new Set(kafkaTopics.concat([feature.topic]))];
     }
     // Subscribe to each topic
     for (let topic of kafkaTopics) {
-        Kafka.receive(`Features.${topic}`, async (topic, message) => {
+        const groupId = '';
+        Kafka.receive(`Features.${topic}`, groupId, async (topic, message) => {
             if (Utils.isEmpty(message.meta?.company))
                 message.meta = { ...message.meta, company: "GLOBAL", outlet: "GLOBAL" };
             if (!message?.feature)
                 return;
-            const timer = ms => new Promise(res => setTimeout(res, ms)); // A promise that resolves after "ms" Milliseconds
+            const timer = (ms) => new Promise(res => setTimeout(res, ms)); // A promise that resolves after "ms" Milliseconds
             topic = topic.replace(/^.*Features./, '');
             while (true) {
                 try {
@@ -251,7 +303,12 @@ export const start = async (app, url, file) => {
                     // // TODO: use $useChunks: [] once support is added in dip insert
                     // await Dip.insert(globalMeta, `users.txns`, { _id: `${message.user}_${iso_date}_${message.txn}`, user: message.user, feature: message.feature, date: message.date, created: message.date, updated: message.date, status: "Queued" }, {idempotent: true}) // Can always update the status later
                     // TODO: Avoid duplicate processing
-                    await features[message.feature].handler(topic, message);
+                    const feature = features[message.feature];
+                    if (!feature) {
+                        console.warn(`Feature ${message.feature} missing, in executing handler during Kafka.receive(topic: ${topic})`);
+                        throw new Error(`Feature ${message.feature} missing, in executing handler during Kafka.receive(topic: ${topic})`);
+                    }
+                    await feature.handler(topic, message);
                     await Dip.insert(globalMeta, "Features.txns", { _id: message.txn, status: "Processed" }, { idempotent: true });
                     break;
                 }
@@ -296,31 +353,13 @@ export const start = async (app, url, file) => {
         console.log('Messaging failed to start. Maybe missing redis');
     }
 };
-function prepareMessage(req) {
-    let txn = req.body.txn ?? Utils.uid();
-    let _id = req.body.data?._id ?? txn;
-    let { data, feature, app, user, outlet, company, client, version } = req.body;
-    let meta = req.meta;
-    let params = req.params;
-    return { data, params, meta, feature, app, user, outlet, company, client, version, txn, id: _id, date: new Date().getTime() };
-}
-const commandAction = async (req, res, topic) => {
-    let message = prepareMessage(req);
-    try {
-        await Kafka.send(`Features.${topic}`, message, message.user, { compression: Kafka.CompressionTypes.GZIP });
-    }
-    catch {
-        throw { status: 500, message: "Failed to queue the transaction" };
-    }
-    res.json({ data: { txn: message.txn, success: true, status: "Queued", featureQueued: true } });
-};
 let subscriptions = {
 // "coins.query.hello": "invoices.command.add",     // Consumer : Feature of Topic
 };
 let subscribed_consumers = {};
 async function subscribe() {
     let Features = { get, send };
-    for (let [key, { subscribe }] of Object.entries(features))
+    for (let [key, { subscribe }] of Object.entries(features)) {
         if (!Utils.isEmpty(subscribe) && subscriptions[key] !== subscribe && subscribed_consumers[key]) {
             try {
                 await subscribed_consumers[key].disconnect();
@@ -328,30 +367,37 @@ async function subscribe() {
             catch (_) { }
             delete subscribed_consumers[key];
         }
-    for (let key in SLYP_FEATURES_LIST)
-        if (!Utils.isEmpty(SLYP_FEATURES_LIST[key].subscribe) && subscriptions[key] !== SLYP_FEATURES_LIST[key].subscribe && subscribed_consumers[key]) {
+    }
+    for (let key in SLYP_FEATURES_LIST) {
+        const feature = SLYP_FEATURES_LIST[key];
+        if (!Utils.isEmpty(feature?.subscribe) && subscriptions[key] !== feature?.subscribe && subscribed_consumers[key]) {
             try {
                 await subscribed_consumers[key].disconnect();
             }
             catch (_) { }
             delete subscribed_consumers[key];
         }
-    for (let key in SLYP_FEATURES_LIST)
-        if (!Utils.isEmpty(SLYP_FEATURES_LIST[key].subscribe)) {
-            subscriptions[key] = SLYP_FEATURES_LIST[key].subscribe;
+    }
+    for (let key in SLYP_FEATURES_LIST) {
+        const feature = SLYP_FEATURES_LIST[key];
+        if (!Utils.isEmpty(feature?.subscribe)) {
+            subscriptions[key] = feature.subscribe;
         }
-    for (let [key, { subscribe }] of Object.entries(features))
+    }
+    for (let [key, { subscribe }] of Object.entries(features)) {
         if (!Utils.isEmpty(subscribe)) {
             subscriptions[key] = subscribe;
         }
+    }
     for (let consumer in subscriptions) {
         let publisher = subscriptions[consumer];
-        let { topic } = getFeature(publisher);
-        if (subscribed_consumers[consumer] || !getFeature(publisher))
+        const publisherFeature = getFeature(publisher);
+        if (subscribed_consumers[consumer] || !publisherFeature)
             continue;
-        let kafka_consumer = Kafka.receive(`Features.${topic}`, consumer, async (topic, message) => {
+        let { topic } = publisherFeature;
+        let kafka_consumer_promise = Kafka.receive(`Features.${topic}`, consumer, async (topic, message) => {
             topic = topic.replace(/^.*Features./, '');
-            const timer = ms => new Promise(res => setTimeout(res, ms)); // A promise that resolves after "ms" Milliseconds
+            const timer = (ms) => new Promise(res => setTimeout(res, ms)); // A promise that resolves after "ms" Milliseconds
             while (true) {
                 try {
                     await Features.send(message.meta, consumer, message.data, message.params);
@@ -363,6 +409,11 @@ async function subscribe() {
                 await timer(10000);
             }
         });
-        subscribed_consumers[consumer] = kafka_consumer;
+        kafka_consumer_promise
+            .then(kafka_consumer => {
+            subscribed_consumers[consumer] = kafka_consumer;
+        }).catch(() => {
+            throw new Error("Error: Remote service to service kafka Feature subscription.");
+        });
     }
 }

package/dist/libs/kafka.js

@@ -33,23 +33,20 @@ export const createTopic = async (topic, partition, replicas) => {
 // ===========
 let consumers = [];
 const start_consumer = async function (topic, groupId, callback) {
-    callback = typeof groupId === "string" ? callback : groupId;
-    groupId = typeof groupId === "string" ? `${appName}.${groupId}` : `${appName}.${topic}`;
+    groupId = groupId ? `${appName}.${groupId}` : `${appName}.${topic}`;
     const consumer = kafka.consumer({ groupId: groupId });
     consumers.push(consumer);
     await consumer.connect();
     await consumer.subscribe({ topic: topic, fromBeginning: true });
-    const fn = callback;
     await consumer.run({
         autoCommit: false,
         eachMessage: async ({ topic, partition, message }) => {
-            // fn(topic, message.value.toString(), partition) // toString() returns array so won't parse if json.
-            let success;
+            // callback(topic, message.value.toString(), partition) // toString() returns array so won't parse if json.
             try {
-                success = await fn(topic, JSON.parse(message.value), partition);
+                await callback(topic, JSON.parse(message.value), partition);
             }
             catch (ex) {
-                success = await fn(topic, message.value, partition);
+                await callback(topic, message.value, partition); // NOTE: Beware: Ensure Kafka.receive<T> can handle raw message type that is not json parseable
             }
             await consumer.commitOffsets([{ topic, partition, offset: (Number(message.offset) + 1).toString() }]);
         },
@@ -82,7 +79,7 @@ const start_producer = async function (topic, message, key, options) {
 // start_producer('quickstart-events', 'Hello KafkaJS user! Little')
 export const receive = async function (topic, groupId, callback) {
     topic = topicPrefix + topic;
-    return (await start_consumer(topic, groupId, callback));
+    return await start_consumer(topic, groupId, callback);
 };
 export const send = async function (topic, message, key, options) {
     topic = topicPrefix + topic;

package/dist/libs/messaging.js

@@ -34,7 +34,7 @@ async function connect({ user, req, res, uid }) {
                 message = JSON.parse(message);
             }
             catch { }
-            for (let { req, res } of users_pool[user].reqres) {
+            for (let { res } of users_pool[user].reqres) {
                 res.write(JSON.stringify({ message, channel }) + "\n");
                 if (res.flush) // If compression enabled
                     res.flush();

package/dist/libs/utils.js

@@ -26,7 +26,7 @@ export function isEmpty(str) {
 }
 export let GLOBAL_META = { company: "GLOBAL", outlet: "GLOBAL" };
 export function isEmptyJson(json) {
-    for (let i in json)
+    for (let _i in json)
         return false;
     return true;
 }

package/libs/auth.ts

@@ -47,7 +47,7 @@ type SuccessCallback = (
 let validateFn: ValidateFunction | undefined
 let validateErrMessage: string | undefined
 
-export const validate = (callback: ValidateFunction, errMessage?: string) => {
+export const validate = (callback: ValidateFunction, _errMessage?: string) => {
     validateFn = callback
 }
 
@@ -58,7 +58,7 @@ export const start = (app: AuthApp, successCallback?: SuccessCallback) => {
             return
         }
         try {
-            let login = await attemptLogin(req, res)
+            let login = await attemptLogin(req)
             if (login.mode === 'verify' && login.success) {
                 let tokens = Session.generateAccessToken(login.userId, req.body.clientId ?? '')
                 let response = {...login, tokens}
@@ -74,7 +74,7 @@ export const start = (app: AuthApp, successCallback?: SuccessCallback) => {
     })
 }
 
-async function attemptLogin(req: AuthRequest, res: AuthResponse) {
+async function attemptLogin(req: AuthRequest) {
     let meta = {company: "GLOBAL", outlet: "GLOBAL"}
 
     let expiry = 300000

package/libs/dip/suffix/index.ts

@@ -89,7 +89,7 @@ function flatten_value(target_key: string, value: unknown, flattened: unknown[],
 }
 
 
-export function entries(target_key: string, query: unknown, binary_slice?: Buffer): unknown[] {
+export function entries(target_key: string, query: unknown, _binary_slice?: Buffer): unknown[] {
     let result  = flatten_id_values(target_key, query); // extract all valid _id
 
     // let cows = result.map(v => extract_direct_id(v, binary_slice))

package/libs/dip/suffix/policy.ts

@@ -140,7 +140,7 @@ type SuffixItem = { key?: string; value?: string; type: string; ls_threshold?: n
 type CollectionsJsonType = Record<string, {policy?: {when: object, suffix: SuffixItem[]}[]}>
 type Arg = { db: string; collection: string[]; }
 
-export async function applySuffixPolicy(COLLECTIONS_JSON: CollectionsJsonType, cols: string | string[], query: Record<string, any>, insertMode?: boolean, arg?: Arg) {
+export async function applySuffixPolicy(COLLECTIONS_JSON: CollectionsJsonType, cols: string | string[], query: Record<string, any>, insertMode: boolean, arg: Arg) {
     const collections = Array.isArray(cols) ? cols : [cols]
     let suffixes: string[] = []
 

package/libs/dip/suffix/suffix.ts

@@ -6,7 +6,7 @@ type SuffixItem = { key?: string; value?: string; type: string; ls_threshold?: n
 type Arg = { db: string; collection: string[]; }
 
 const EXPLICIT_SUFFIX_POLICY_TYPES = new Set([  "string", "date", "number" ]);
-export async function suffix(doc: Record<string, any>, policies: SuffixItem[], insertMode?: boolean, arg?: Arg) {
+export async function suffix(doc: Record<string, any>, policies: SuffixItem[], insertMode: boolean, arg: Arg) {
     let suffixes = [""]
     for (const policy of policies) {
 
@@ -92,9 +92,8 @@ export async function suffix(doc: Record<string, any>, policies: SuffixItem[], i
         if (!keys.length && !insertMode) {
             ls = true
             for (const suffix of suffixes) {
-                const collections = arg?.collection ?? []
-                for (const collection of collections) {
-                    const dirs = (await Dip.operation("ls", { db: arg?.db, collection: collection, suffix })).filter((dir: string) => !dir.startsWith('chunk-'))
+                for (const collection of arg.collection) {
+                    const dirs = (await Dip.operation("ls", { db: arg.db, collection: collection, suffix })).filter((dir: string) => !dir.startsWith('chunk-'))
                     suffixAssociatedKeys[suffix] = suffixAssociatedKeys[suffix] ?? []
                     suffixAssociatedKeys[suffix].push(...dirs)
                 }