@samanbayaka/core 0.0.1

package/index.mjs

@@ -0,0 +1,427 @@
+import { ServiceBroker, Errors } from "moleculer"
+
+import {SBK_GLOBAL_CONFIGS} from '#hUti/global-configs-validator.mjs'
+import * as configHdl from '#hUti/config-handler.mjs'
+
+import {AjvValidator} from "#hMol/AjvValidator.mjs"
+import CustomLogger from "#hMol/CustomLogger.mjs"
+import HybridCacher  from "#hMol/HybridCacher.mjs"
+
+import { gracefulShutdown, validateServiceName, ErrorFormatterMW } from '#hUti/error-handler.mjs'
+import { createOpenTelemetryExporters, OpenTelemetryMW } from '#hUti/telemetry.mjs'
+import { openApiToMoleculerParams } from '#hUti/openapi-to-mol-params.mjs'
+
+/**
+ * Global Configs
+ */
+const BROKER_CONFIG = await configHdl.getConfigs('/config/sbk/global/broker/yaml/nats')
+const CATCHER_CONFIG = await configHdl.getConfigs('/config/sbk/global/catcher/yaml/redis')
+const TELEMETRY_CONFIG = await configHdl.getConfigs('/config/sbk/global/telemetry/yaml/openobserve')
+
+/**
+ * Local configs
+ */
+const LC_LOG_LEVEL = await configHdl.getConfigs(`/config/sbk/members/${SBK_GLOBAL_CONFIGS.serviceDtls.name}/envs/LOG_LEVEL`)
+const LC_TELEMETRY = await configHdl.getConfigs(`/config/sbk/members/${SBK_GLOBAL_CONFIGS.serviceDtls.name}/envs/TELEMETRY`)
+const LC_MAX_L1_TTL = await configHdl.getConfigs(`/config/sbk/members/${SBK_GLOBAL_CONFIGS.serviceDtls.name}/envs/MAX_L1_TTL`)
+const LC_MAX_L2_TTL = await configHdl.getConfigs(`/config/sbk/members/${SBK_GLOBAL_CONFIGS.serviceDtls.name}/envs/MAX_L2_TTL`)
+
+/**
+ * Service Broker initialization params with default values
+ */
+const LOG_LEVEL = LC_LOG_LEVEL || SBK_GLOBAL_CONFIGS.logLevel || "debug"
+const TELEMETRY = LC_TELEMETRY || SBK_GLOBAL_CONFIGS.telemetry || true
+const MAX_L1_TTL = LC_MAX_L1_TTL || SBK_GLOBAL_CONFIGS.maxL1TTL || 120
+const MAX_L2_TTL = LC_MAX_L2_TTL || SBK_GLOBAL_CONFIGS.maxL2TTL || 600
+
+
+/**
+ * Initialize OTLP metrics and telemetry exporter
+ */
+const nodeSDK = await createOpenTelemetryExporters(TELEMETRY_CONFIG)
+/**
+ * create file watcher instance
+ */
+const fileWatcherHdl = SBK_GLOBAL_CONFIGS.createFileWatcher()
+/**
+ * Declared broker
+ */
+let broker
+
+/**
+ * Listen for force stop signals
+ */
+process.on("beforeExit", async () => {
+  await configHdl.client.close()
+  await nodeSDK.shutdown()
+  await fileWatcherHdl.close()
+})
+
+process.on("SIGINT", async (signal) => {
+  await gracefulShutdown(broker, signal)
+})   // Ctrl+C
+process.on("SIGTERM", async (signal) => {
+  await gracefulShutdown(broker, signal)
+})  // kill command
+
+
+/**
+ * Optional: handle uncaught errors
+ */
+process.on("uncaughtException", async (err) => {
+  await gracefulShutdown(broker, "Uncaught Exception", err)
+})
+process.on("unhandledRejection", async (err) => {
+  await gracefulShutdown(broker, "Unhandled Rejection", err)
+})
+
+
+/**
+ * Event listner for configuration revision
+ * @type {Object}
+ */
+const configRevisionListner = {
+  "config.revisions": (ctx) => {
+    setImmediate(async () => {
+      ctx.broker.logger.debug({tag: "SBK", message: "Config revision received"}, ctx.params)
+
+      /**
+       * Restart the service when the global configuration is updated
+       */
+      if( SBK_GLOBAL_CONFIGS.configRevision.pub != ctx.params.pub ){
+        ctx.broker.logger.warn({tag: "SBK", message: `Global configuration deployed by CONFIGADMIN and the service "${SBK_GLOBAL_CONFIGS.serviceDtls.name}" is restarting in progress...`})
+        await ctx.broker.stop()
+        process.exit(0)
+      }
+
+      /**
+       * Restart the service if it is an edge service in response to an edge configuration update
+       */
+      if( 
+          SBK_GLOBAL_CONFIGS.configRevision.edg != ctx.params.edg 
+          && schema?.settings?.port != undefined
+      ){
+        ctx.broker.logger.warn({tag: "SBK", message: `Edge configuration deployed by CONFIGADMIN and the service "${SBK_GLOBAL_CONFIGS.serviceDtls.name}" is restarting in progress...`})
+        await ctx.broker.stop()
+        process.exit(0)
+      }
+
+      /**
+       * Restart the service when the corresponding configuration is updated
+       */
+      if(
+        SBK_GLOBAL_CONFIGS.configRevision.mem[SBK_GLOBAL_CONFIGS.serviceDtls.name] != ctx.params.mem[SBK_GLOBAL_CONFIGS.serviceDtls.name]
+        && ctx.params.mem[SBK_GLOBAL_CONFIGS.serviceDtls.name] >= 0
+      ){
+        ctx.broker.logger.warn({tag: "SBK", message: `Configuration deployed by CONFIGADMIN and the service "${SBK_GLOBAL_CONFIGS.serviceDtls.name}" is restarting in progress...`})
+        await ctx.broker.stop()
+        process.exit(0)
+      }
+
+      /**
+       * Stop the service when CONFIGADMIN removes it from the trusted list
+       */
+      if( ctx.params.mem[SBK_GLOBAL_CONFIGS.serviceDtls.name] < 0  && SBK_GLOBAL_CONFIGS.configRevision.mem[SBK_GLOBAL_CONFIGS.serviceDtls.name] >= 0 ){
+        ctx.broker.logger.error({tag: "SBK", message: `The service "${SBK_GLOBAL_CONFIGS.serviceDtls.name}" was stopped by CONFIGADMIN.`})
+        await ctx.broker.stop()
+        process.exit(1)
+      }
+    })        
+  }
+
+}
+
+/**
+ * Initalized service broker, validate service and create service
+ */
+const createSbkServices = async(schemas, isREPL = false) => {
+  /**
+   * Moleculer srvice broker configurations
+   * @type {ServiceBroker}
+   */
+  broker = new ServiceBroker({
+    ...BROKER_CONFIG,
+    namespace: SBK_GLOBAL_CONFIGS.namespaceTxt,
+    nodeID: SBK_GLOBAL_CONFIGS.serviceDtls.id,
+    validator: new AjvValidator(),
+    logLevel: LOG_LEVEL,
+    logger: CustomLogger({
+      logger: BROKER_CONFIG.logger,
+      logLevel: LOG_LEVEL
+    }),
+    cacher: new HybridCacher({
+      ...CATCHER_CONFIG,
+      ttl: [MAX_L2_TTL, MAX_L1_TTL]
+    }),
+    middlewares: [ErrorFormatterMW, OpenTelemetryMW(TELEMETRY)],
+  })
+
+  if(isREPL){
+    /**
+     * Start broker with repl mode
+     */
+    await broker.start().then(() => broker.repl())
+  }
+  else{
+    /**
+     * Start services
+     */
+    await broker.start()
+
+    const allServices = ( await broker.call("$node.services")).map(el => el.name )
+    if( allServices.includes("system") && SBK_GLOBAL_CONFIGS.serviceDtls.name == "system" ){
+      broker.logger.error({tag: "SBK", message: 'The service "system" is already registered in the Moleculer cluster.'})
+      await broker.stop()
+      process.exit(1)
+    }
+  }
+
+  /**
+   * Creating moleculer services
+   */
+  schemas = Array.isArray(schemas) ? schemas : [schemas]
+  for ( const [index, schema] of schemas.entries() ) {
+
+    // /**
+    //  * Listner for configuration revision
+    //  * @type {Object}
+    //  */
+    // const configRevisionListner = {
+    //   "config.revisions": (ctx) => {
+    //     setImmediate(async () => {
+    //       ctx.broker.logger.debug({tag: "SBK", message: "Config revision received"}, ctx.params)
+
+    //       /**
+    //        * Restart the service when the global configuration is updated
+    //        */
+    //       if( SBK_GLOBAL_CONFIGS.configRevision.pub != ctx.params.pub ){
+    //         ctx.broker.logger.warn({tag: "SBK", message: `Global configuration deployed by CONFIGADMIN and the service "${SBK_GLOBAL_CONFIGS.serviceDtls.name}" is restarting in progress...`})
+    //         await ctx.broker.stop()
+    //         process.exit(0)
+    //       }
+
+    //       /**
+    //        * Restart the service if it is an edge service in response to an edge configuration update
+    //        */
+    //       if( 
+    //           SBK_GLOBAL_CONFIGS.configRevision.edg != ctx.params.edg 
+    //           && schema?.settings?.port != undefined
+    //       ){
+    //         ctx.broker.logger.warn({tag: "SBK", message: `Edge configuration deployed by CONFIGADMIN and the service "${SBK_GLOBAL_CONFIGS.serviceDtls.name}" is restarting in progress...`})
+    //         await ctx.broker.stop()
+    //         process.exit(0)
+    //       }
+
+    //       /**
+    //        * Restart the service when the corresponding configuration is updated
+    //        */
+    //       if(
+    //         SBK_GLOBAL_CONFIGS.configRevision.mem[SBK_GLOBAL_CONFIGS.serviceDtls.name] != ctx.params.mem[SBK_GLOBAL_CONFIGS.serviceDtls.name]
+    //         && ctx.params.mem[SBK_GLOBAL_CONFIGS.serviceDtls.name] >= 0
+    //       ){
+    //         ctx.broker.logger.warn({tag: "SBK", message: `Configuration deployed by CONFIGADMIN and the service "${SBK_GLOBAL_CONFIGS.serviceDtls.name}" is restarting in progress...`})
+    //         await ctx.broker.stop()
+    //         process.exit(0)
+    //       }
+
+    //       /**
+    //        * Stop the service when CONFIGADMIN removes it from the trusted list
+    //        */
+    //       if( ctx.params.mem[SBK_GLOBAL_CONFIGS.serviceDtls.name] < 0  && SBK_GLOBAL_CONFIGS.configRevision.mem[SBK_GLOBAL_CONFIGS.serviceDtls.name] >= 0 ){
+    //         ctx.broker.logger.error({tag: "SBK", message: `The service "${SBK_GLOBAL_CONFIGS.serviceDtls.name}" was stopped by CONFIGADMIN.`})
+    //         await ctx.broker.stop()
+    //         process.exit(1)
+    //       }
+    //     })        
+    //   }
+
+    // }
+
+    /**
+     * merge configuration listner with user defined events
+     * @type {Object}
+     */
+    schema.events = {
+      ...schema?.events,
+      ...configRevisionListner
+    }
+
+    /**
+     * Override params according to the OpenAPI schema definition
+     */
+    if( schema?.actions ) {
+      Object.keys(schema.actions).map((el) => {
+        if( schema.actions[el] ){
+          openApiToMoleculerParams(schema.actions[el])
+
+        }
+      })
+    }
+    
+
+    validateServiceName(
+      schema, 
+      (index == 0 
+        ? SBK_GLOBAL_CONFIGS.serviceDtls.name
+        : schema.name
+      )
+    )
+    broker.createService(schema)
+  }
+
+
+  /**
+   * Notify on changes to files in the service directory.
+   */
+  fileWatcherHdl.on("add", (fp) => {
+    broker.logger.warn({tag: "SBK", message: `The file "${fp}" has been added to your service folder. Restart your service for the changes to take effect.`})
+  })
+  fileWatcherHdl.on("addDir", (fp) => {
+    broker.logger.warn({tag: "SBK", message: `The DIR "${fp}" has been added to your service folder. Restart your service for the changes to take effect.`})
+  })
+  fileWatcherHdl.on("change", (fp) => {
+    broker.logger.warn({tag: "SBK", message: `The file "${fp}" in your service folder has been modified. Restart your service for the changes to take effect.`})
+  })
+  fileWatcherHdl.on("unlink", (fp) => {
+    broker.logger.warn({tag: "SBK", message: `The file "${fp}" in your service folder has been removed. Restart your service for the changes to take effect.`})
+  })
+  fileWatcherHdl.on("unlinkDir", (fp) => {
+    broker.logger.warn({tag: "SBK", message: `The DIR "${fp}" in your service folder has been removed. Restart your service for the changes to take effect.`})
+  })
+
+}
+
+
+/**
+ * Export samanbayaka object
+ */
+export default {
+  Errors,
+  getConfigs: configHdl.getConfigs,
+  createSbkServices,
+
+  // /**
+  //  * Loading Demo
+  //  */
+  // createDemo: async() => {
+  //   const { default: apiGateway } = await import('#sApi/index.mjs')
+  //   const demo = await import('#sDmo/index.mjs')
+
+  //   const gtWy = {...apiGateway}
+  //   gtWy.name = "demo"
+  //   gtWy.settings.port = 3000
+  //   gtWy.settings.routes = gtWy.settings.routes.map((rt) => {
+  //     rt.path = `/demo${rt.path}`
+  //     return rt
+  //   })
+  //   createSbkServices([
+  //     gtWy,
+  //     demo.user, 
+  //     demo.role, 
+  //     demo.mail, 
+  //     demo.sms
+  //   ], true)
+
+  // },
+
+  // /**
+  //  * system services
+  //  * @return {void}
+  //  */
+  // registerSystemAPI: async () => {
+  //   const { default: sysApi } = await import('#sSys/index.mjs')
+  //   const { default: authMgr } = await import('#sAut/auth.mjs')
+  //   const { default: oidc } = await import('#sYdc/index.mjs')
+  //   const { default: asset } = await import('#sAst/index.mjs')
+  //   const { default: dbs } = await import('#sDbs/index.mjs')
+
+  //   const pgDb = await dbs('pg', {})
+
+  //   createSbkServices([sysApi, authMgr, oidc, asset, pgDb])
+  //   // createSbkServices([sysApi, authMgr, asset])
+
+  // },
+
+  // registerAuthAPI: async () => {
+  //   const { default: coreAuth } = await import('#sAut/index.mjs')
+  //   createSbkServices(coreAuth)
+
+  // },
+
+  // registerEdgeAPI: async () => {
+  //   const { default: apiGateway } = await import('#sApi/index.mjs')
+  //   createSbkServices(apiGateway)
+
+  // },
+
+  
+  // registerServices: async (schema) => {
+  //   createSbkServices(schema)
+
+  // },
+  
+
+  // registerBridgeServices: async( type, brokerOpts = {}, callback = () => {}) => {
+  //   const { auxBrokerParamsValidator } = await import('#hUti/aux-broker-params-validator.mjs')
+  //   const {logLevel, gzip, msgPack, rest, ...opts} = brokerOpts
+  //   const paramObj = {type, opts, callback}
+
+  //   auxBrokerParamsValidator(paramObj)
+  //   callback = paramObj.callback
+
+  //   const auxBroker = /kafka/.test(type)
+  //     ? await import('#sKfk/index.mjs')
+  //     : /mqtt/.test(type)
+  //       ? undefined // await import('#sKfk/index.mjs')
+  //       : /amqp/.test(type)
+  //         ? undefined // await import('#sKfk/index.mjs')
+  //         : undefined
+
+  //   if ( !auxBroker ) {
+  //     // throw new Error("The \"type\" value must be either \"kafka\" or \"mqtt\" or \"amqp\"")
+  //     throw new Error("The \"type\" value must be either \"kafka\"")
+  //   }
+
+  //   opts.logLevel = logLevel
+  //   opts.gzip = gzip
+  //   opts.msgPack = msgPack
+  //   opts.rest = rest
+
+  //   /**
+  //    * Service create for kafka producer
+  //    * @type {object}
+  //    */
+  //   if( opts.name.startsWith("producer") ){
+  //     createSbkServices(auxBroker.producer( opts, callback ))
+
+  //   }
+  //   else {
+  //     createSbkServices(auxBroker.consumer( opts, callback ))
+
+  //   }
+
+  // },
+
+  // registerAuthAPI: async () => {
+  //   const { default: authMgr } = await import('#sAut/index.mjs')
+  //   createSbkServices([authMgr])
+
+  // },
+
+  // registerYdcAPI: async () => {
+  //   const { default: oidc } = await import('#sYdc/index.mjs')
+  //   createSbkServices([oidc])
+
+  // },
+
+  // registerDbsAPI: async (type, opts) => {
+  //   const { default: dbs } = await import('#sDbs/index.mjs')
+  //   createSbkServices(await dbs(type, opts))
+
+  // },
+ 
+}
+
+
+// rm -rf node_modules/samanbayaka/ && cp -r ../../samanbayaka/. node_modules/samanbayaka/ && SBK_CONFIG_CRED='<your_etcd_user> <your_etcd_pass> <your_etcd_port>' node index.mjs
+// pm2 stop "$(basename "$PWD")" && rm -rf node_modules/samanbayaka/ && cp -r ../../samanbayaka/. node_modules/samanbayaka/ && pm2 start "$(basename "$PWD")" && pm2 logs | grep '━━━━━━━❖❖ SBK ❖❖━━━━━━━'

package/package.json

@@ -0,0 +1,78 @@
+{
+  "name": "@samanbayaka/core",
+  "version": "0.0.1",
+  "description": "Moleculer Gateway service with kafka transporter",
+  "homepage": "https://gitlab.com/dalal.suvendu/samanbayaka#readme",
+  "bugs": {
+    "url": "https://gitlab.com/dalal.suvendu/samanbayaka/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://gitlab.com/dalal.suvendu/samanbayaka.git"
+  },
+  "license": "ISC",
+  "author": "dalal.suvendu",
+  "type": "module",
+  "imports": {
+    "#hFil/*": "./helper/file/*",
+    "#hUti/*": "./helper/utility/*",
+    "#hMol/*": "./helper/mol-built-in/*",
+    "#sAut/*": "./services/auth/*",
+    "#sApi/*": "./services/gateway/*",
+    "#sDmo/*": "./services/demo/*",
+    "#sKfk/*": "./services/kafkajs/*",
+    "#sSys/*": "./services/system/*",
+    "#sYdc/*": "./services/oidc/*",
+    "#sDbs/*": "./services/dbs/*",
+    "#sAst/*": "./services/asset/*"
+  },
+  "main": "index.mjs",
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.1112.0",
+    "@opentelemetry/api": "^1.9.1",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.216.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.216.0",
+    "@opentelemetry/resources": "^2.7.1",
+    "@opentelemetry/sdk-metrics": "^2.7.1",
+    "@opentelemetry/sdk-node": "^0.216.0",
+    "@opentelemetry/sdk-trace-base": "^2.7.1",
+    "@opentelemetry/semantic-conventions": "^1.40.0",
+    "@smithy/node-http-handler": "^4.11.3",
+    "acorn": "^8.18.0",
+    "ajv": "^8.18.0",
+    "ajv-formats": "^3.0.1",
+    "approve-builds": "^1.0.0",
+    "body-parser": "^2.3.0",
+    "chokidar": "^5.0.0",
+    "compression": "^1.8.1",
+    "cookie-parser": "^1.4.7",
+    "etcd3": "^1.1.2",
+    "helmet": "^8.1.0",
+    "ioredis": "^5.10.1",
+    "jsonwebtoken": "^9.0.3",
+    "jwks-rsa": "^4.0.1",
+    "kafkajs": "^2.2.4",
+    "lru-cache": "^11.3.5",
+    "moleculer": "0.15.0",
+    "moleculer-auto-openapi": "^1.1.7",
+    "moleculer-repl": "^0.8.0",
+    "moleculer-web": "^0.11.0",
+    "msgpack5": "^6.0.2",
+    "pg": "^8.23.0",
+    "pino": "^10.3.1",
+    "pnpm": "^10.34.1",
+    "sinon": "^21.1.2",
+    "swagger-stats": "^0.99.7",
+    "uuid": "^14.0.1",
+    "yaml": "^2.9.0"
+  },
+  "devDependencies": {
+    "chai": "^6.2.2",
+    "husky": "^9.1.7",
+    "mocha": "^11.7.5"
+  },
+  "scripts": {
+    "test": "mocha",
+    "start": "node index.mjs"
+  }
+}