corebasic 1.0.51 → 1.0.53

package/libs/features.js

@@ -1,57 +1,38 @@
 import * as Elabase from './elabase.js'
 import * as Kafka from './kafka.js'
-import { createRequire } from 'module'; 
-const require = createRequire(import.meta.url);
+import * as Utils from './utils.js'
 
 
 let  features = {}
 let apis = {}
 
-export function start(app, url) {
-    let file = url.toString().replace('file://','')
-    features = require(file)
+export const start = async (app, url, file) => {
+    features = await Utils.fileToJson(url, file)
     
-    Object.entries(features).forEach(async ([name,feature]) => {
+
+    // Registering handlers
+    for (let name in features) {
+        let feature = features[name]
         apis[feature.api] = apis[feature.api] ?? {}
         apis[feature.api][name] = feature
         
-        let featureName = name
-
         name = name.split('.')
         let handler = name.pop()
         name = `src/${name.join('/')}`
 
         if (name === 'src/transactions/query')
-            return
+            continue
 
         feature.handler = (await import(`${new URL(`${name}.js`, url).toString().replace('file://', '')}`))[handler]
+    }
 
-        if (feature.api.split(' ')[0].toLowerCase() !== "get") {
-            let topic = feature.topic
-            Kafka.receive(`Features.${topic}`, `${featureName}.${topic}`, async (topic, message) => {
-
-                const timer = ms => new Promise(res => setTimeout(res, ms)) // A promise that resolves after "ms" Milliseconds
-                topic = topic.replace('Features.','')
-                while (true) {
-                    try {
-                        await features[message.feature].handler(topic, message)
-                        await Elabase.insert("Features.txns", {_id: message.txn, status: "Processed"})
-                        break
-                    } catch {
-                        console.warn(`Feature: ${featureName}, error in executing handler during Kafka.receive(topic: ${topic})`)
-                    }
-                    await timer(10000); 
-                }
-            })
-        }
-
-    })
-
-    Object.entries(apis).forEach(([api,features]) => {
+    // Registering apis
+    for (let api in apis) {
         let method = api.split(' ')[0].toLowerCase()
         let url = api.split(' ')[1]
         if (api === 'GET /transactions/:id')
-            return
+            continue
+
         app[method](url, async (req, res) => {
             let feature = features[req.body.feature]
             let topic = feature.topic
@@ -68,9 +49,38 @@ export function start(app, url) {
                 res.status(err.status ?? 500).json(err)
             }
         })
-    })
+    }
+
+
+    // Get unique kafka topics list from features
+    let kafkaTopics = []
+    for (let name in features) {
+        let feature = features[name]
+        if (feature.api.split(' ')[0].toLowerCase() === "get" )
+            continue
+        kafkaTopics = [... new Set(kafkaTopics.concat([feature.topic]))]
+    }
 
+    // Subscribe to each topic
+    for (let topic of kafkaTopics) {
+        Kafka.receive(`Features.${topic}`, async (topic, message) => {
+
+            const timer = ms => new Promise(res => setTimeout(res, ms)) // A promise that resolves after "ms" Milliseconds
+            topic = topic.replace('Features.','')
+            while (true) {
+                try {
+                    await features[message.feature].handler(topic, message)
+                    await Elabase.insert("Features.txns", {_id: message.txn, status: "Processed"})
+                    break
+                } catch {
+                    console.warn(`Feature: ${message.feature}, error in executing handler during Kafka.receive(topic: ${topic})`)
+                }
+                await timer(10000); 
+            }
+        })
+    }
 
+    // Transaction status api
     app.get('/transactions/:id', async (req, res) => {
         try {
             let items = await Elabase.query("Features.txns", {_id: req.params.id})
@@ -90,7 +100,7 @@ const commandAction = async (req, res, topic) => {
     try {
         let {data, feature, app, user, client, version} = req.body
         let params = req.params
-        await Kafka.send(`Features.${topic}`, { data, params, feature, app, user, client, version, txn, id: _id, date: new Date().getTime() }, user)
+        await Kafka.send(`Features.${topic}`, { data, params, feature, app, user, client, version, txn, id: _id, date: new Date().getTime() }, user, { compression: Kafka.CompressionTypes.GZIP })
 
     } catch {
         throw {status: 500, message: "Failed to queue the transaction"}

package/libs/kafka.js

@@ -1,6 +1,7 @@
-import { Kafka, logLevel as KafkaLogLevel } from 'kafkajs'
+import { Kafka, logLevel as KafkaLogLevel, CompressionTypes as KafkaCompressionTypes } from 'kafkajs'
 
 export const logLevel = KafkaLogLevel
+export const CompressionTypes = KafkaCompressionTypes
 
 let kafka, producer, admin
 
@@ -68,7 +69,7 @@ const start_consumer = async function (topic, groupId, callback) {
 // Producer
 // ===========
 let producerInvoked = false
-const start_producer = async function (topic, message, key) {
+const start_producer = async function (topic, message, key, options) {
 
     let isJson = Object.prototype.toString.call(message) === '[object Object]' || Object.prototype.toString.call(message) === '[object Array]'
 
@@ -79,6 +80,7 @@ const start_producer = async function (topic, message, key) {
     let value = isJson ? JSON.stringify(message) : message
     await producer.send({
         topic: topic,
+        ...(options ?? {}),
         messages: [
             key ? { key, value} : {value},
         ],
@@ -99,9 +101,9 @@ export const receive = async function(topic, groupId, callback) {
 }
 
 
-export const send = async function(topic, message, key) {
+export const send = async function(topic, message, key, options) {
     topic = topicPrefix + topic
-    await start_producer(topic, message, key)
+    await start_producer(topic, message, key, options)
 }
 
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "corebasic",
   "type": "module",
-  "version": "1.0.51",
+  "version": "1.0.53",
   "description": "",
   "main": "index.js",
   "scripts": {