@nsshunt/stsappframework 3.0.103 → 3.0.105

package/src_new/kafka/IMKafkaManager.ts

@@ -1,154 +0,0 @@
-/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
-import { InstrumentPayload } from './../commonTypes'
-import { Gauge } from '@nsshunt/stsinstrumentation'
-import { KAFKA_PREFIX } from '@nsshunt/stssocketio-client'
-
-import { KafkaManager, IKafkaManagerConfig } from './kafkamanager'
-import { KafkaProducer } from './kafkaproducer'
-
-import { JSONObject } from '@nsshunt/stsutils'
-import { v4 as uuidv4 } from 'uuid';
-
-import chalk from 'chalk';
-
-import { $Options } from '@nsshunt/stsconfig'
-const goptions = $Options()
-
-const _logPrefix = 'IMKafkaManager.'
-
-// Manage the publication of InstrumentPayload to Kafka
-export class IMKafkaManager {
-    #shuttingDown: boolean = false;
-    #producer: KafkaProducer | null = null;
-
-    #km: KafkaManager;
-    #topics: JSONObject = { }
-
-    constructor()
-    {
-        //super(options);
-
-        const kmc: IKafkaManagerConfig = {
-            clientId: `${goptions.kafka_clientId}_${uuidv4()}`,
-            brokers: goptions.kafka_brokers.split(','),
-            keepAlive: goptions.kafka_keep_alive,
-            adminTimeout: goptions.kafka_admin_timeout,
-            connectionTimeout: goptions.kafka_connection_timeout,
-            logLevel: goptions.kafka_log_level,
-            useSSL: goptions.kafka_use_ssl,
-            requestTimeout: goptions.kafka_request_timeout
-        }
-        if (goptions.kafka_use_ssl) {
-            kmc.ssl = {
-                rejectUnauthorized: goptions.kafka_ssl_rejectUnauthorized,
-                cafile: goptions.kafka_ssl_cafile,
-                certfileFile: goptions.kafka_ssl_certfile,
-                keyfile: goptions.kafka_ssl_keyfile
-            }
-        }
-
-        this.#km = new KafkaManager(kmc);
-    }
-
-    get km(): KafkaManager {
-        return this.#km;
-    }
-
-    Start = async (): Promise<void> => {
-        this.#producer = this.#km.CreateProducer();
-        await this.#producer.Connect()
-    }
-
-    Terminate = async (): Promise<void> => {
-        try {
-            if (this.#shuttingDown) {
-                console.log(`IMKafkaManager:Terminate: Terminate already called. Ignoring.`.yellow);
-            } else {
-                this.#shuttingDown = true;
-                console.log(`ProducerDisconnect`.yellow);
-                if (this.#producer) {
-                    await this.#producer.Disconnect();
-                }
-            }
-        } catch (error) {
-            console.error(chalk.red(`${_logPrefix}#Terminate: Error: [${error}]`));
-        }
-    }
-
-    async #OutputLogsToKafkaTopic(topic: string, logMessages: string[]) {
-        if (!this.#topics[topic]) {
-            console.log(`Setting up kafka topic: [${topic}] Starting`.yellow);
-            this.#topics[topic] = true;
-            const result = await this.#km.CreateTopic(topic, 1);
-            if (result) {
-                console.log(`Setting up kafka topic: [${topic}] Completed`.yellow);
-            } else {
-                console.log(`Setting up kafka topic: [${topic}] Completed - topic already exists`.cyan);
-            }
-        }
-
-        const messages: JSONObject[] = [ ];
-        logMessages.forEach((m) => {
-            messages.push({key: topic, value: m})
-        });
-
-        // Don't wait for these to return ...
-        if (this.#producer) {
-            this.#producer.SendMessages(topic, messages as any);
-        }
-    }
-
-    async OutputLogs(instrumentPayload: InstrumentPayload): Promise<boolean> {
-        try {
-            if (this.#shuttingDown) {
-                return false;
-            }
-
-            let topic: string = '';
-            if (instrumentPayload.instruments[Gauge.LOGGER]) {
-                const loggerMessages = [ ...instrumentPayload.instruments[Gauge.LOGGER].val as string[] ];
-                if (loggerMessages.length > 0) {
-                    if (instrumentPayload.context.agentName) {
-                        // Processing agent payload
-                        const { agentName, threadId, asyncRunnerId } = instrumentPayload.context;
-        
-                        topic = '';
-                        if (agentName && threadId && asyncRunnerId) {
-                            // Output logs for a specific asyncRunnerId
-                            topic = `${KAFKA_PREFIX}_${agentName}_${threadId}_${asyncRunnerId}`.replace(/@/g, '_').replace(/\|/g, '_');
-                            this.#OutputLogsToKafkaTopic(topic, loggerMessages); // no need to await this ...
-                        }
-                        if (agentName && threadId) {
-                            // Output logs for a specific thread (worker) within an agent
-                            topic = `${KAFKA_PREFIX}_${agentName}_${threadId}`.replace(/@/g, '_').replace(/\|/g, '_');
-                            this.#OutputLogsToKafkaTopic(topic, loggerMessages); // no need to await this ...
-                        }
-                        if (agentName) {
-                            // Output logs for a specific thread (worker) within an agent
-                            topic = `${KAFKA_PREFIX}_${agentName}`.replace(/@/g, '_').replace(/\|/g, '_');
-                            this.#OutputLogsToKafkaTopic(topic, loggerMessages); // no need to await this ...
-                        }
-                    } else {
-                        // Processing service payload
-                        const { serviceInstanceId, serviceInstanceProcessId, pid, ppid } = instrumentPayload.context;
-                        if (serviceInstanceId && serviceInstanceProcessId && pid && ppid) {
-                            // Output to the specific thread within an instance
-                            topic = `${KAFKA_PREFIX}_${serviceInstanceId}_${serviceInstanceProcessId}`.replace(/@/g, '_').replace(/\|/g, '_');
-                            this.#OutputLogsToKafkaTopic(topic, loggerMessages); // no need to await this ...
-
-                            if (pid === ppid) {
-                                // Output only to the main thread on the service, i.e. this is for O/A service logging
-                                topic = `${KAFKA_PREFIX}_${serviceInstanceId}`.replace(/@/g, '_').replace(/\|/g, '_');
-                                this.#OutputLogsToKafkaTopic(topic, loggerMessages); // no need to await this ...
-                            }
-                        }
-                    }
-                }
-            }
-            return true;
-        } catch (error: any) {
-            console.error(chalk.red(`${_logPrefix}OutputLogs: Could not output log data to kafka: [${error}]`));
-            return false;
-        }
-    }
-}

package/src_new/kafka/kafkaconsumer.ts

@@ -1,82 +0,0 @@
-import { Kafka, Consumer, IHeaders } from 'kafkajs'
-import { IKafkaConsumer } from './../commonTypes'
-
-export class KafkaConsumer implements IKafkaConsumer {
-    #id: string
-    #groupId: string
-    #consumer: Consumer;
-    #kafka: Kafka;
-    #connected: boolean = false;
-
-    constructor(kafka: Kafka, id: string, groupId: string) {
-        this.#id = id;
-        this.#groupId = groupId;
-        this.#kafka = kafka;
-        this.#consumer = this.#kafka.consumer({ groupId: this.#groupId })
-    }
-
-    get consumer() {
-        return this.#consumer;
-    }
-    
-    get id(): string {
-        return this.#id;
-    }
-
-    async Connect(): Promise<void> {
-        if (!this.#connected) {
-            await this.#consumer.connect()
-            this.#connected = true;
-        }
-    }
-
-    async Disconnect(): Promise<void> {
-        if (this.#connected) {
-            await this.#consumer.disconnect()
-            this.#connected = false;
-        }
-    }
-
-    Subscribe = async(topics: string[], fromBeginning: boolean): Promise<void> => {
-        if (this.#connected) {
-            await this.#consumer.subscribe({ topics, fromBeginning })
-        } else {
-            throw new Error(`[Subscribe] Consumer not connected.`);
-        }
-    }
-
-    Stop = async(): Promise<void> => {
-        if (this.#connected) {
-            await this.#consumer.stop();
-        }
-    }
-
-    StartConsumingMessages = async (cb: (topic: string, key: string, partition: number, value: string, headers: IHeaders | undefined) => void): Promise<void> => {
-        if (this.#connected) {
-            await this.#consumer.run({
-                // eslint-disable-next-line @typescript-eslint/no-unused-vars
-                eachMessage: async ({ topic, partition, message, heartbeat, pause }) => {
-                    try {
-                        if (message.key) {
-                            if (message.value) {
-                                cb(topic.toString(), message.key.toString(), partition, message.value.toString(), message.headers)
-                            } else {
-                                cb(topic.toString(), message.key.toString(), partition, "", message.headers)
-                            }
-                        } else {
-                            if (message.value) {
-                                cb(topic.toString(), "", partition, message.value?.toString(), message.headers)
-                            } else {
-                                cb(topic.toString(), "", partition, "", message.headers)
-                            }
-                        }
-                    } catch (err) {
-                        console.log(err);
-                    }
-                }
-            })
-        } else {
-            throw new Error(`[StartConsumingMessages] Consumer not connected.`);
-        }
-    }
-}

package/src_new/kafka/kafkamanager.ts

@@ -1,186 +0,0 @@
-/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
-/*
-
-kafka example server #01 - Docker Compose File
-----------------------------------------------
-Note: In this example, the log retention is set to 24 hours (rather than default to 1 week)
-https://www.conduktor.io/kafka/kafka-topic-configuration-log-retention/
-
-version: '2'
-services:
-  zookeeper:
-    image: wurstmeister/zookeeper
-    ports:
-      - "2181:2181"
-    restart: unless-stopped
-
-  kafka:
-    image: wurstmeister/kafka
-    ports:
-      - "9092:9092"
-    environment:
-      DOCKER_API_VERSION: 1.22
-      KAFKA_ADVERTISED_HOST_NAME: 192.168.14.92
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_CREATE_TOPICS: "topic-name2:3:1"
-      KAFKA_LOG_RETENTION_MS: 86400000
-      KAFKA_LOG_RETENTION_BYTES: -1
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-    restart: unless-stopped
-
-
-kafka example server #02 - Docker Compose File
-----------------------------------------------
-version: "3.9"  # optional since v1.27.0
-
-networks:
-  app-tier:
-    driver: bridge
-
-services:
-  kafka:
-    image: 'bitnami/kafka:latest'
-    ports:
-     - '9092:9092'
-    networks:
-      - app-tier     
-    environment:
-      - ALLOW_PLAINTEXT_LISTENER=yes
-      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
-      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://192.168.14.92:9092
-
-*/
-import { STSOptionsBase } from '@nsshunt/stsutils'
-
-import { Kafka, KafkaConfig, logLevel } from 'kafkajs'
-
-import { v4 as uuidv4 } from 'uuid';
-import fs from 'node:fs'
-
-import { KafkaConsumer } from './kafkaconsumer'
-import { KafkaProducer } from './kafkaproducer'
-
-import net from 'net'
-import tls from 'tls'
-
-const KEEP_ALIVE_DELAY = 60000 //@@ in ms
-
-// https://kafka.js.org/docs/configuration
-export interface IKafkaManagerConfig {
-    clientId: string // A logical identifier of an application. Can be used by brokers to apply quotas or trace requests to a specific application. Example: booking-events-processor.
-    brokers: string[] // List of Kafka brokers
-    adminTimeout: number // Time in milliseconds to wait for a successful admin operation. The default value is: 5000.
-    connectionTimeout: number // Time in milliseconds to wait for a successful connection. The default value is: 1000.
-    requestTimeout: number // Time in milliseconds to wait for a successful request. The default value is: 30000.
-    logLevel: string // There are 5 log levels available: NOTHING, ERROR, WARN, INFO, and DEBUG. INFO is configured by default.
-    keepAlive?: number // When specified, the number of ms for socket keep alive processing.
-    useSSL: boolean // Use SSL
-    ssl?: { // Must be specified if useSSL is true
-      rejectUnauthorized: boolean
-      cafile: string
-      keyfile: string
-      certfileFile: string
-    }
-}
-
-// https://kafka.js.org/docs/configuration
-// https://github.com/tulios/kafkajs/blob/master/src/network/socketFactory.js
-declare interface ICustomSocketFactory {
-    host: any,
-    port: any,
-    ssl: any,
-    onConnect: any
-}
-
-export class KafkaManager extends STSOptionsBase {
-    #kafka: Kafka
-
-    constructor(options: IKafkaManagerConfig) {
-        super(options);
-
-        const kc: KafkaConfig = {
-            clientId: options.clientId,
-            brokers: options.brokers, //brokers: ['localhost:9092', 'kafka2:9092'],
-            connectionTimeout: options.connectionTimeout,
-            requestTimeout: options.requestTimeout
-        }
-        // NOTHING, ERROR, WARN, INFO, and DEBUG. INFO is configured by default.
-        switch (options.logLevel) {
-        case 'NOTHING' :
-            kc.logLevel = logLevel.NOTHING;
-            break;
-        case 'ERROR' :
-            kc.logLevel = logLevel.ERROR;
-            break;
-        case 'WARN' :
-            kc.logLevel = logLevel.WARN;
-            break;
-        case 'INFO' :
-            kc.logLevel = logLevel.INFO;
-            break;
-        case 'DEBUG' :
-            kc.logLevel = logLevel.DEBUG;
-            break;
-        default :
-            kc.logLevel = logLevel.NOTHING;
-        }
-        if (options.useSSL && options.ssl) {
-            kc.ssl = {
-                ca: [fs.readFileSync(options.ssl.cafile as string, { encoding: 'utf8'})],
-                key: fs.readFileSync(options.ssl.keyfile, { encoding: 'utf8'}),
-                cert: fs.readFileSync(options.ssl.certfileFile, { encoding: 'utf8'}),
-            }
-        }
-        if (options.keepAlive) {
-            //const myCustomSocketFactory = ({ host, port, ssl, onConnect }) => {
-            const myCustomSocketFactory = (config: ICustomSocketFactory) => {
-                const socket = config.ssl
-                    ? tls.connect(
-                        Object.assign({ host: config.host, port: config.port }, !net.isIP(config.host) ? { servername: config.host } : {}, config.ssl),
-                        config.onConnect
-                    )
-                    : net.connect({ host: config.host, port: config.port }, config.onConnect)
-        
-                socket.setKeepAlive(true, options.keepAlive)
-                return socket
-            }
-            kc.socketFactory = myCustomSocketFactory;
-        }
-
-        this.#kafka = new Kafka(kc);
-    }
-
-    get kafka() {
-        return this.#kafka;
-    }
-
-    CreateProducer(): KafkaProducer {
-        return new KafkaProducer(this.#kafka, uuidv4());
-    }
-
-    CreateConsumer(groupId: string) {
-        return new KafkaConsumer(this.#kafka, uuidv4(), groupId);
-    }
-
-    CreateTopic = async (topic: string, partitions: number): Promise<boolean> => {
-        const admin = this.#kafka.admin()
-        await admin.connect()
-        const result = await admin.createTopics({
-            validateOnly: false,
-            waitForLeaders: true,
-            timeout: this.options?.timeout,
-            topics: [
-                {
-                    topic: topic,
-                    numPartitions: partitions,     // default: -1 (uses broker `num.partitions` configuration)
-                    //replicationFactor: <Number>, // default: -1 (uses broker `default.replication.factor` configuration)
-                    //replicaAssignment: <Array>,  // Example: [{ partition: 0, replicas: [0,1,2] }] - default: []
-                    //configEntries: <Array>       // Example: [{ name: 'cleanup.policy', value: 'compact' }] - default: []
-                }            
-            ]
-        })
-        await admin.disconnect()
-        return result;
-    }
-}

package/src_new/kafka/kafkaproducer.ts

@@ -1,58 +0,0 @@
-import { Kafka, Producer, RecordMetadata } from 'kafkajs'
-
-export class KafkaProducer {
-    #id: string
-    #producer: Producer;
-    #kafka: Kafka;
-    #connected: boolean = false;
-
-    constructor(kafka: Kafka, id: string) {
-        this.#id = id;
-        this.#kafka = kafka;
-        this.#producer = this.#kafka.producer()
-    }
-
-    get producer() {
-        return this.#producer;
-    }
-    
-    get id(): string {
-        return this.#id;
-    }
-
-    async Connect(): Promise<void> {
-        if (!this.#connected) {
-            await this.#producer.connect();
-            this.#connected = true;
-        }
-    }
-
-    async Disconnect(): Promise<void> {
-        if (this.#connected) {
-            await this.#producer.disconnect()
-            this.#connected = false;
-        }
-    }
-
-    SendMessage = async(topic: string, message: { key: string, value: string} ): Promise<RecordMetadata[]> => {
-        if (this.#connected) {
-            return this.#producer.send({
-                topic,
-                messages: [ message ]
-            })   
-        } else {
-            throw new Error(`[SendMessage] Producer not connected.`);
-        }
-    }
-
-    SendMessages = async(topic: string, messages: { key: string, value: string}[] ): Promise<RecordMetadata[]> => {
-        if (this.#connected) {
-            return this.#producer.send({
-                topic,
-                messages
-            })   
-        } else {
-            throw new Error(`[SendMessages] Producer not connected.`);
-        }
-    }
-}

package/src_new/kafkatesting/config.ts

@@ -1,10 +0,0 @@
-export const TOPIC = 'appframework-test-logs';
-export const PARTITIONS = 3;
-export const TIMEOUT = 5000;
-
-export const GROUP_ID = 'my-group';
-
-export const CLIENT_ID = 'my-app';
-
-export const BROKERS = ['192.168.14.92:9092'];
-

package/src_new/kafkatesting/consume.ts

@@ -1,116 +0,0 @@
-/* eslint @typescript-eslint/no-unused-vars: 0 */  // --> OFF
-import { TOPIC, GROUP_ID, CLIENT_ID, BROKERS, TIMEOUT } from './config'
-
-import { KafkaManager } from './../kafka/kafkamanager'
-import { JSONObject } from '@nsshunt/stsutils';
-
-async function Sleep(milliseconds = 1000) { 
-    return new Promise(resolve => setTimeout(resolve, milliseconds))
-}
-
-const km = new KafkaManager({
-    clientId: CLIENT_ID + process.env.CLIENT_ID,
-    brokers: BROKERS,
-    adminTimeout: TIMEOUT,
-    connectionTimeout: TIMEOUT,
-    requestTimeout: TIMEOUT,
-    logLevel: 'NOTHING',
-    useSSL: false
-});
-
-const runme = async () => {
-
-    const fromBeginning = false;
-
-    const consumer = km.CreateConsumer(GROUP_ID + process.env.GROUP_ID);
-
-    await consumer.Connect();
-
-    await consumer.Subscribe([TOPIC], fromBeginning);
-
-    await consumer.StartConsumingMessages((topic: string, key: string, partition: number, value: string, headers: JSONObject | undefined) => {
-        console.log({
-            key,
-            partition,
-            value,
-            headers
-        });
-    });
-
-    await consumer.Stop();
-
-    await consumer.Subscribe(['zzz'], fromBeginning);
-
-    await consumer.StartConsumingMessages((topic: string, key: string, partition: number, value: string, headers: JSONObject | undefined) => {
-        console.log({
-            key,
-            partition,
-            value,
-            headers
-        });
-    });
-
-    await consumer.Stop();
-
-    await consumer.Subscribe(['yyy'], fromBeginning);
-
-    await consumer.StartConsumingMessages((topic: string, key: string, partition: number, value: string, headers: JSONObject | undefined) => {
-        /*
-        console.log({
-            key,
-            partition,
-            value,
-            headers
-        });
-        */
-        console.log(`key: [${key}]   value: [${value}]   partition: [${partition}]  headers: [${headers}]`);
-    });
-
-    //await km.Subscribe(['zzz'], cb);
-
-    process.on("SIGINT", async () => {
-        console.log('=========SIGTERM START =======================')
-        await consumer.Disconnect();
-        console.log('=========SIGTERM END =======================')
-        process.exit();
-    });
-
-    let iteration = 0;
-    for (;;) {
-        console.log('sleep: ' + iteration++);
-        await Sleep(1000);
-    }
-}
-
-/*
-const errorTypes = ['unhandledRejection', 'uncaughtException']
-const signalTraps = ['SIGTERM', 'SIGINT', 'SIGUSR2']
-
-errorTypes.forEach(type => {
-    process.on(type, async () => {
-        try {
-            console.log(`process.on ${type}`)
-            console.log('=========consumer.disconnect() START =======================')
-            await consumer.disconnect()
-            console.log('=========consumer.disconnect() END =======================')
-            process.exit(0)
-        } catch (_) {
-            process.exit(1)
-        }
-    })
-})
-
-signalTraps.forEach(type => {
-    process.once(type, async () => {
-        try {
-            console.log('=========consumer.disconnect() START [2] =======================')
-            await consumer.disconnect()
-            console.log('=========consumer.disconnect() END [2] =======================')
-        } finally {
-            process.kill(process.pid, type)
-        }
-    })
-})
-*/
-
-runme().catch(e => console.error(`[example/producer] ${e.message}`, e))