@nsshunt/stsappframework 3.0.95 → 3.0.96

package/src_working/influxdb/influxDBManagerAgent.ts

@@ -0,0 +1,314 @@
+/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
+// https://www.influxdata.com/blog/tldr-influxdb-tech-tips-multiple-aggregations-yield-flux/
+// https://www.influxdata.com/blog/top-5-hurdles-for-intermediate-flux-users-and-resources-for-optimizing-flux/
+
+/*
+option task = {name: "task-sts-agent-stats", every: 1s}
+
+data =
+    from(bucket: "TestBucket01")
+        |> range(start: -5s)
+        |> last()
+        |> filter(fn: (r) => r["_measurement"] == "agent")
+
+r1 =
+    data
+        |> filter(fn: (r) => r["_field"] == "requestCount" 
+		or r["_field"] == "errorCount"
+		or r["_field"] == "retryCount"
+	    or r["_field"] == "authenticationCount" 
+        or r["_field"] == "activeRequestCount" 
+        or r["_field"] == "coreCount" 
+        or r["_field"] == "velocity"
+        or r["_field"] == "rx"
+        or r["_field"] == "tx"
+        or r["_field"] == "childCount"
+        or r["_field"] == "timer")
+
+r2 =
+    data
+        |> filter(fn: (r) => float(v: r["_value"]) > 0.0 and (r["_field"] == "duration" 
+            or r["_field"] == "latency"))
+
+byagentthreadasyncunnersum =
+    r1
+        |> group(columns: ["agentName", "threadId", "asyncRunnerId", "_field"])
+        |> sum()
+        |> toFloat()
+
+byagentthreadasyncunnermean =
+    r2
+        |> group(columns: ["agentName", "threadId", "asyncRunnerId", "_field"])
+        |> mean()
+        |> toFloat()
+
+union(tables: [byagentthreadasyncunnersum, byagentthreadasyncunnermean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-by-agentthreadasyncunner"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+byagentthreadsum =
+    r1
+        |> group(columns: ["agentName", "threadId", "_field"])
+        |> sum()
+        |> toFloat()
+
+byagentthreadmean =
+    r2
+        |> group(columns: ["agentName", "threadId", "_field"])
+        |> mean()
+        |> toFloat()
+
+union(tables: [byagentthreadsum, byagentthreadmean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-by-agentthread"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+byagentsum =
+    r1
+        |> group(columns: ["agentName", "_field"])
+        |> sum()
+        |> toFloat()
+
+byagentmean =
+    r2
+        |> group(columns: ["agentName", "_field"])
+        |> mean()
+        |> toFloat()
+
+union(tables: [byagentsum, byagentmean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-by-agent"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+globalagentsum =
+    r1
+        |> group(columns: ["_field"])
+        |> sum()
+        |> toFloat()
+
+globalagentmean =
+    r2
+        |> group(columns: ["_field"])
+        |> mean()
+        |> toFloat()
+
+union(tables: [globalagentsum, globalagentmean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-globalagent"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+*/
+
+import { Point, WriteApi, QueryApi } from '@influxdata/influxdb-client'
+
+import { Gauge, InstrumentVelocity } from '@nsshunt/stsinstrumentation'
+
+import { InfluxDBManagerBase } from './influxDBManagerBase'
+
+import { IInfluxDBManagerOptions, InstrumentPayload } from './../commonTypes'
+import { ISubscriptionPayload, ISubscriptionKey } from '@nsshunt/stssocketio-client'
+
+const AGENT_STATS_BY_AGENT_THREAD_ASYNCRUNNER = "sts-stats-by-agentthreadasyncunner";
+const AGENT_STATS_BY_AGENT_THREAD = "sts-stats-by-agentthread";
+const AGENT_STATS_BY_AGENT = "sts-stats-by-agent";
+const AGENT_STATS_GLOBALAGENT = "sts-stats-globalagent";
+
+const _logPrefix = 'InfluxDBManagerService:'
+
+export class InfluxDBManagerAgent extends InfluxDBManagerBase
+{
+    constructor(options: IInfluxDBManagerOptions, queryApi: QueryApi) {
+        super(options, queryApi);
+    }
+
+    override get modelType(): string {
+        return 'agent'
+    }
+
+    // Queries --------------------------------------------------------------------------------------------------------
+
+    // Counter metrics
+    #GetSTSCountGenericAgent = async (measurement: string, filterClause: string, showOutput: boolean = false) => {
+        try {
+            const query = `from(bucket: "${this.options.bucket}") 
+            |> range(start: -5s) 
+            |> last()
+            |> filter(fn: (r) => r["_measurement"] == "${measurement}" ${filterClause})`;
+
+            if (showOutput) {
+                console.log(query);
+            }
+
+            return this.queryApi.collectRows(query)
+        } catch (error) {
+            console.error(`${_logPrefix}#GetSTSCountGeneric: Error: [${error}]`.red);
+        }
+    }
+
+    // Histo metrics
+    #GetSTSHistoGenericService = async (measurement: string, filterClause: string): Promise<any> => {
+        try {
+            const query = `import "math"
+            from(bucket: "${this.options.bucket}") 
+            |> range(start: -10m) 
+            |> filter(fn: (r) => r["_measurement"] == "${measurement}" and (r["_field"] =="latency"
+                or r["_field"] =="duration") ${filterClause})
+            |> histogram(bins: [0.0, 10.0, 20.0, 50.0, 100.0, 1000.0, 50000.0, math.mInf(sign: 1) ])
+            |> difference()`;
+            return this.queryApi.collectRows(query)
+        } catch (error) {
+            console.error(`${_logPrefix}#GetSTSHistoGeneric: Error: [${error}]`.red);
+        }
+    }
+
+    // Quantile metrics
+    #GetSTSQuantileGenericAgent = async (measurement: string, filterClause: string) => {
+        try {
+            const query = `data = from(bucket: "${this.options.bucket}") 
+            |> range(start: -10m) 
+            |> filter(fn: (r) => r["_measurement"] == "${measurement}" and (r["_field"] == "latency"
+                or r["_field"] == "duration") ${filterClause})
+            |> aggregateWindow(every: 5s, fn: max, createEmpty: false)
+
+            dostsquantileex = (q) =>
+            data
+            |> quantile(q: q, method: "estimate_tdigest", compression: 1000.0)
+            |> set(key: "quantile", value: string(v:q))
+            |> group(columns: ["quantile"])
+            
+            union(tables: [
+                dostsquantileex(q: 0.5),
+                dostsquantileex(q: 0.8),
+                dostsquantileex(q: 0.9),
+                dostsquantileex(q: 0.95),
+                dostsquantileex(q: 0.99)
+            ])`;
+            
+            return this.queryApi.collectRows(query)
+        } catch (error) {
+            console.error(`${_logPrefix}#GetSTSQuantileGeneric: Error: [${error}]`.red);
+        }
+    }
+
+    // Metric queries -------------------------------------------------------------------------------------------------
+    // Root level metrics
+    async GetInfluxDBResultsRootAgent(subscriptionKey: ISubscriptionKey): Promise<ISubscriptionPayload> { // ISubscriptionPayload
+        let retVal = null;
+        try {
+            retVal = await this.ProcessInfluxDBResults([
+                this.#GetSTSCountGenericAgent(AGENT_STATS_GLOBALAGENT, ''),
+                this.#GetSTSQuantileGenericAgent(AGENT_STATS_GLOBALAGENT, ''),
+                this.#GetSTSHistoGenericService(AGENT_STATS_GLOBALAGENT, '')],
+            [ ])
+        } catch (error) {
+            console.error(`${_logPrefix}GetInfluxDBResultsRootAgent: Error: [${error}]`.red);
+        }
+        return { 
+            subscriptionKey,
+            data: retVal 
+        };
+    }
+
+    // Service level metrics
+    async GetInfluxDBResultsAgent(subscriptionKey: ISubscriptionKey): Promise<ISubscriptionPayload> {
+        let retVal = null;
+        try {
+            retVal = await this.ProcessInfluxDBResults([
+                this.#GetSTSCountGenericAgent(AGENT_STATS_BY_AGENT, ''),
+                this.#GetSTSQuantileGenericAgent(AGENT_STATS_BY_AGENT, ''),
+                this.#GetSTSHistoGenericService(AGENT_STATS_BY_AGENT, '')],
+            ['agentName'])
+        } catch (error) {
+            console.error(`${_logPrefix}GetInfluxDBResultsAgent: Error: [${error}]`.red);
+        }
+        return { 
+            subscriptionKey,
+            data: retVal 
+        };
+    }
+
+    // Service instance metrics for a particular service type
+    async GetInfluxDBResultsAgentThreads(subscriptionKey: ISubscriptionKey): Promise<ISubscriptionPayload> {
+        let retVal = null;
+        try {
+            const agentName = subscriptionKey.key as string;
+            retVal = await this.ProcessInfluxDBResults([
+                this.#GetSTSCountGenericAgent(AGENT_STATS_BY_AGENT_THREAD, `and r["agentName"] == "${agentName}"`),
+                this.#GetSTSQuantileGenericAgent(AGENT_STATS_BY_AGENT_THREAD, `and r["agentName"] == "${agentName}"`),
+                this.#GetSTSHistoGenericService(AGENT_STATS_BY_AGENT_THREAD, `and r["agentName"] == "${agentName}"`)],
+            ['agentName', 'threadId'])
+        } catch (error) {
+            console.error(`${_logPrefix}GetInfluxDBResultsAgentThreads: Error: [${error}]`.red);
+        }
+        return { 
+            subscriptionKey,
+            data: retVal 
+        };
+    }
+
+    // Service instance thread metrics for a particular service instance
+    async GetInfluxDBResultsAgentThread(subscriptionKey: ISubscriptionKey): Promise<ISubscriptionPayload> {
+        let retVal = null;
+        try {
+            const agentName = subscriptionKey.key as string;
+            const threadId = subscriptionKey.subkey as string;
+            retVal = await this.ProcessInfluxDBResults([
+                this.#GetSTSCountGenericAgent(AGENT_STATS_BY_AGENT_THREAD_ASYNCRUNNER, `and r["agentName"] == "${agentName}" and r["threadId"] == "${threadId}"`),
+                this.#GetSTSQuantileGenericAgent(AGENT_STATS_BY_AGENT_THREAD_ASYNCRUNNER, `and r["agentName"] == "${agentName}" and r["threadId"] == "${threadId}"`),
+                this.#GetSTSHistoGenericService(AGENT_STATS_BY_AGENT_THREAD_ASYNCRUNNER, `and r["agentName"] == "${agentName}" and r["threadId"] == "${threadId}"`)],
+            ['agentName', 'threadId', 'asyncRunnerId'])
+        } catch (error) {
+            console.error(`${_logPrefix}GetInfluxDBResultsAgentThread: Error: [${error}]`.red);
+        }
+        return { 
+            subscriptionKey,
+            data: retVal 
+        };
+    }
+    
+    // Agent context payload
+    // {"nid":"somehost@e58f5d75-6ff6-4e04-92a4-f2bcd722fec0-someuseragent|MainProcess|0","id":"somehost@e58f5d75-6ff6-4e04-92a4-f2bcd722fec0-someuseragent|MainProcess|0","hostName":"somehost"
+    // ,"agentName":"e58f5d75-6ff6-4e04-92a4-f2bcd722fec0-someuseragent","threadId":"MainProcess","asyncRunnerId":"0"}
+    async OutputInfluxDB(writeApi: WriteApi, instrumentPayload: InstrumentPayload): Promise<boolean> {
+        if (writeApi) {
+            try {
+                const { nid, id, hostName, agentName, threadId, asyncRunnerId } = instrumentPayload.context;
+
+                let systemcpu = 0.0;
+                if (instrumentPayload.instruments[Gauge.CPU_SYSTEM_LOAD_GAUGE]) {
+                    systemcpu = (instrumentPayload.instruments[Gauge.CPU_SYSTEM_LOAD_GAUGE].val as number);
+                }
+
+                const point = new Point('all')
+                    .measurement('agent')
+                    // Context settings
+                    .tag('id', id as string) // Must be unique (uuid)
+                    .tag('hostName', hostName as string) // Must be unique (uuid)
+                    .tag('agentName', agentName as string) // Must be unique (uuid)
+                    .tag('threadId', threadId as string) // Must be unique (uuid)
+                    .tag('asyncRunnerId', asyncRunnerId as string)
+                    // Data fields
+                    .intField('requestCount', instrumentPayload.instruments[Gauge.REQUEST_COUNT_GAUGE].val) // q
+                    .intField('errorCount', instrumentPayload.instruments[Gauge.ERROR_COUNT_GAUGE].val) // j
+                    .intField('retryCount', instrumentPayload.instruments[Gauge.RETRY_COUNT_GAUGE].val) // r
+                    .intField('authenticationCount', instrumentPayload.instruments[Gauge.AUTHENTICATION_COUNT_GAUGE].val) // b
+                    .floatField('velocity', (instrumentPayload.instruments[Gauge.VELOCITY_GAUGE] as InstrumentVelocity).va) // t - Note: Using va not val here
+                    .intField('coreCount', instrumentPayload.instruments[Gauge.CORE_COUNT_GAUGE].val) // x
+                    .floatField('rx', (instrumentPayload.instruments[Gauge.NETWORK_RX_GAUGE] as InstrumentVelocity).va) // Note: Using va not val here
+                    .floatField('tx', (instrumentPayload.instruments[Gauge.NETWORK_TX_GAUGE] as InstrumentVelocity).va) // Note: Using va not val here
+                    .floatField('timer', instrumentPayload.instruments[Gauge.TIMER_GAUGE].val) // s
+                    .floatField('duration', instrumentPayload.instruments[Gauge.DURATION_GAUGE].val) // h
+                    .floatField('latency', instrumentPayload.instruments[Gauge.LATENCY_GAUGE].val) // h
+                    .intField('activeRequestCount', instrumentPayload.instruments[Gauge.ACTIVE_REQUEST_GAUGE].val) // a
+                    .intField('childCount', instrumentPayload.instruments[Gauge.CHILD_COUNT].val) // y
+                    // logger // m
+                    // duration histo // i
+                writeApi.writePoint(point);
+                return true;
+            } catch (error: any) {
+                console.error(`${_logPrefix}OutputInfluxDBAgent: Could not write data point: [${error}]`.red);
+                return false;
+            }
+        } else {
+            console.error(`${_logPrefix}OutputInfluxDBAgent: Could not write data point as writeClient is null`.red);
+            return false;
+        }
+    }
+}

package/src_working/influxdb/influxDBManagerBase.ts

@@ -0,0 +1,109 @@
+/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
+// https://www.influxdata.com/blog/tldr-influxdb-tech-tips-multiple-aggregations-yield-flux/
+// https://www.influxdata.com/blog/top-5-hurdles-for-intermediate-flux-users-and-resources-for-optimizing-flux/
+import { WriteApi, QueryApi } from '@influxdata/influxdb-client'
+
+import { IInfluxDBManagerOptions, InstrumentPayload } from './../commonTypes'
+
+import { JSONObject } from '@nsshunt/stsutils'
+
+const _logPrefix = 'InfluxDBManager:'
+
+export class InfluxDBManagerBase
+{
+    #queryApi: QueryApi;
+    #shuttingDown: boolean = false;
+    #options: IInfluxDBManagerOptions;
+
+    constructor(options: IInfluxDBManagerOptions, queryApi: QueryApi) {
+        this.#queryApi = queryApi;
+        this.#options = options;
+    }
+
+    get options() {
+        return this.#options;
+    }
+    get queryApi() {
+        return this.#queryApi;
+    }
+
+    get shuttingDown() {
+        return this.#shuttingDown;
+    }
+    set shuttingDown(val: boolean) {
+        this.#shuttingDown = val;
+    }
+
+    get modelType(): string {
+        throw new Error('modelType(): Must override in sub-class')
+    }
+
+    // Helpers --------------------------------------------------------------------------------------------------------
+    async ProcessInfluxDBResults(promArray: Promise<any>[], fields: string[]) {
+        try {
+            const promRetVal: any[] = await Promise.all(promArray).catch(error => {
+                console.error(error)
+            }) as any[];
+
+            const combinedResults: any = { };
+
+            if (promRetVal) {
+                const stscount: any[] = promRetVal[0];
+                const stsquantile: any[] = promRetVal[1];
+                const stshisto: any[] = promRetVal[2];
+
+                stscount.forEach((countRecord) => {
+                    const values: any[] = fields.map(field => countRecord[field])
+                    const { _field, _value } = countRecord;
+                    this.CreateNestedObject(combinedResults, values)[_field] = _value;
+                });
+
+                stsquantile.forEach((stsquantileRecord) => {
+                    const values: any[] = fields.map(field => stsquantileRecord[field])
+                    const { _field, quantile, _value} = stsquantileRecord;
+                    const useField = _field + '_quantile'
+                    this.CreateNestedObject(combinedResults, [...values, useField])[quantile] = _value;
+                });
+
+                stshisto.forEach((stshistoRecord) => {
+                    const values: any[] = fields.map(field => stshistoRecord[field])
+                    const { _field, le, _value} = stshistoRecord;
+                    const useField = _field + '_histo'
+                    this.CreateNestedObject(combinedResults, [...values, useField])[le] = _value;
+                });
+            }
+            return combinedResults;
+        } catch (error) {
+            console.error(`${_logPrefix}ProcessInfluxDBResults: Error: [${error}]`.red);
+        }
+    }
+    
+    CreateGlobalCountModel = (stscount: any[]) => {
+        try {
+            const results: any = { };
+            stscount.forEach((o: any) => {
+                results[o._field] = o._value;
+            });
+            return results;
+        } catch (error) {
+            console.error(`${_logPrefix}#CreateGlobalCountModel: Error: [${error}]`.red);
+        }
+    }
+
+    CreateNestedObject(combinedResults: JSONObject, keyList: string[]): JSONObject {
+        let workingObject = combinedResults;
+        keyList.forEach((key) => {
+            if (!workingObject[key]) {
+                workingObject[key] = { };
+            }
+            workingObject = workingObject[key];
+        });
+        return workingObject;
+    }
+
+    // Agent context payload
+    // {"nid":"somehost@e58f5d75-6ff6-4e04-92a4-f2bcd722fec0-someuseragent|MainProcess|0","id":"somehost@e58f5d75-6ff6-4e04-92a4-f2bcd722fec0-someuseragent|MainProcess|0","hostName":"somehost","agentName":"e58f5d75-6ff6-4e04-92a4-f2bcd722fec0-someuseragent","threadId":"MainProcess","asyncRunnerId":"0"}
+    async OutputInfluxDB(writeApi: WriteApi, instrumentPayload: InstrumentPayload): Promise<boolean> {
+        throw new Error('Must override in sub-class');
+    }
+}