@nsshunt/stsappframework 3.0.59 → 3.0.61

package/src/influxdb/influxDBManagerAgent.ts

@@ -1,6 +1,99 @@
 /* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
 // https://www.influxdata.com/blog/tldr-influxdb-tech-tips-multiple-aggregations-yield-flux/
 // https://www.influxdata.com/blog/top-5-hurdles-for-intermediate-flux-users-and-resources-for-optimizing-flux/
+
+/*
+option task = {name: "task-sts-agent-stats", every: 1s}
+
+data =
+    from(bucket: "TestBucket01")
+        |> range(start: -5s)
+        |> last()
+        |> filter(fn: (r) => r["_measurement"] == "agent")
+
+r1 =
+    data
+        |> filter(fn: (r) => r["_field"] == "requestCount" 
+		or r["_field"] == "errorCount"
+		or r["_field"] == "retryCount"
+	    or r["_field"] == "authenticationCount" 
+        or r["_field"] == "activeRequestCount" 
+        or r["_field"] == "coreCount" 
+        or r["_field"] == "velocity"
+        or r["_field"] == "childCount"
+        or r["_field"] == "timer")
+
+r2 =
+    data
+        |> filter(fn: (r) => float(v: r["_value"]) > 0.0 and (r["_field"] == "duration" 
+            or r["_field"] == "latency"))
+
+byagentthreadasyncunnersum =
+    r1
+        |> group(columns: ["agentName", "threadId", "asyncRunnerId", "_field"])
+        |> sum()
+        |> toFloat()
+
+byagentthreadasyncunnermean =
+    r2
+        |> group(columns: ["agentName", "threadId", "asyncRunnerId", "_field"])
+        |> mean()
+        |> toFloat()
+
+union(tables: [byagentthreadasyncunnersum, byagentthreadasyncunnermean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-by-agentthreadasyncunner"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+byagentthreadsum =
+    r1
+        |> group(columns: ["agentName", "threadId", "_field"])
+        |> sum()
+        |> toFloat()
+
+byagentthreadmean =
+    r2
+        |> group(columns: ["agentName", "threadId", "_field"])
+        |> mean()
+        |> toFloat()
+
+union(tables: [byagentthreadsum, byagentthreadmean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-by-agentthread"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+byagentsum =
+    r1
+        |> group(columns: ["agentName", "_field"])
+        |> sum()
+        |> toFloat()
+
+byagentmean =
+    r2
+        |> group(columns: ["agentName", "_field"])
+        |> mean()
+        |> toFloat()
+
+union(tables: [byagentsum, byagentmean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-by-agent"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+globalagentsum =
+    r1
+        |> group(columns: ["_field"])
+        |> sum()
+        |> toFloat()
+
+globalagentmean =
+    r2
+        |> group(columns: ["_field"])
+        |> mean()
+        |> toFloat()
+
+union(tables: [globalagentsum, globalagentmean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-globalagent"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+*/
+
 import { Point, WriteApi, QueryApi } from '@influxdata/influxdb-client'
 
 import { Gauge, InstrumentVelocity } from '@nsshunt/stsinstrumentation'
@@ -10,6 +103,11 @@ import { InfluxDBManagerBase } from './influxDBManagerBase'
 import { IInfluxDBManagerOptions, InstrumentPayload } from './../commonTypes'
 import { ISubscriptionPayload, ISubscriptionKey } from '@nsshunt/stssocketio-client'
 
+const AGENT_STATS_BY_AGENT_THREAD_ASYNCRUNNER = "sts-stats-by-agentthreadasyncunner";
+const AGENT_STATS_BY_AGENT_THREAD = "sts-stats-by-agentthread";
+const AGENT_STATS_BY_AGENT = "sts-stats-by-agent";
+const AGENT_STATS_GLOBALAGENT = "sts-stats-globalagent";
+
 const _logPrefix = 'InfluxDBManagerService:'
 
 export class InfluxDBManagerAgent extends InfluxDBManagerBase
@@ -25,60 +123,12 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
     // Queries --------------------------------------------------------------------------------------------------------
 
     // Counter metrics
-    #GetSTSCountGenericAgent = async (filterClause: string, groupClause: string, showOutput: boolean = false) => {
+    #GetSTSCountGenericAgent = async (measurement: string, filterClause: string, showOutput: boolean = false) => {
         try {
             const query = `from(bucket: "${this.options.bucket}") 
             |> range(start: -5s) 
             |> last()
-            |> filter(fn: (r) => r["_measurement"] == "agent" and (r["_field"] == "errorCount"
-                or r["_field"] == "retryCount"
-                or r["_field"] == "authenticationCount"
-                or r["_field"] == "velocity"
-                or r["_field"] == "coreCount"
-                or r["_field"] == "timer"
-                or r["_field"] == "duration"
-                or r["_field"] == "latency"
-                or r["_field"] == "activeRequestCount"
-                or r["_field"] == "childCount") ${filterClause})
-            |> ${groupClause}
-            |> sum()`;
-
-            if (showOutput) {
-                console.log(query);
-            }
-
-            return this.queryApi.collectRows(query)
-        } catch (error) {
-            console.error(`${_logPrefix}#GetSTSCountGeneric: Error: [${error}]`.red);
-        }
-    }
-
-    #GetSTSCountGenericAgentOld = async (filterClause: string, groupClause: string, showOutput: boolean = false) => {
-        try {
-            const query = `dostscountex = (q, d) => 
-            from(bucket: "${this.options.bucket}") 
-            |> range(start: d) 
-            |> filter(fn: (r) => r["_measurement"] == "agent" ${filterClause}) 
-            |> filter(fn: (r) => r["_field"] == q) 
-            |> last()
-            |> ${groupClause}
-            |> sum()
-            
-            dostscount = (d) =>
-            union(tables: [
-                dostscountex(q: "requestCount", d: d),
-                dostscountex(q: "errorCount", d: d),
-                dostscountex(q: "retryCount", d: d),
-                dostscountex(q: "authenticationCount", d: d),
-                dostscountex(q: "velocity", d: d),
-                dostscountex(q: "coreCount", d: d),
-                dostscountex(q: "timer", d: d),
-                dostscountex(q: "duration", d: d),
-                dostscountex(q: "latency", d: d),
-                dostscountex(q: "activeRequestCount", d: d),
-                dostscountex(q: "childCount", d: d)
-            ])
-            dostscount(d: -5s)`;
+            |> filter(fn: (r) => r["_measurement"] == "${measurement}" ${filterClause})`;
 
             if (showOutput) {
                 console.log(query);
@@ -91,14 +141,13 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
     }
 
     // Histo metrics
-    #GetSTSHistoGenericService = async (filterClause: string, groupClause: string): Promise<any> => {
+    #GetSTSHistoGenericService = async (measurement: string, filterClause: string): Promise<any> => {
         try {
             const query = `import "math"
             from(bucket: "${this.options.bucket}") 
             |> range(start: -10m) 
-            |> filter(fn: (r) => r["_measurement"] == "agent" and (r["_field"] =="latency"
+            |> filter(fn: (r) => r["_measurement"] == "${measurement}" and (r["_field"] =="latency"
                 or r["_field"] =="duration") ${filterClause})
-            |> ${groupClause}
             |> histogram(bins: [0.0, 10.0, 20.0, 50.0, 100.0, 1000.0, 50000.0, math.mInf(sign: 1) ])
             |> difference()`;
             return this.queryApi.collectRows(query)
@@ -107,40 +156,13 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
         }
     }
 
-    #GetSTSHistoGenericServiceOld = async (filterClause: string, groupClause: string): Promise<any> => {
-        try {
-            const query = `import "math"
-
-            dostshistoex = (q, d) =>
-            from(bucket: "${this.options.bucket}") 
-            |> range(start: d) 
-            |> filter(fn: (r) => r["_measurement"] == "agent") 
-            |> filter(fn: (r) => r["_field"] == q ${filterClause}) 
-            |> ${groupClause}
-            |> histogram(bins: [0.0, 10.0, 20.0, 50.0, 100.0, 1000.0, 50000.0, math.mInf(sign: 1) ])
-            |> difference()
-            
-            dostshisto = (d) =>
-            union(tables: [
-                dostshistoex(q: "latency", d: d),
-                dostshistoex(q: "duration", d: d)
-            ])
-            
-            dostshisto(d: -10m)`;
-            return this.queryApi.collectRows(query)
-        } catch (error) {
-            console.error(`${_logPrefix}#GetSTSHistoGeneric: Error: [${error}]`.red);
-        }
-    }
-
     // Quantile metrics
-    #GetSTSQuantileGenericAgent = async (filterClause: string, groupClause: string) => {
+    #GetSTSQuantileGenericAgent = async (measurement: string, filterClause: string) => {
         try {
             const query = `data = from(bucket: "${this.options.bucket}") 
             |> range(start: -10m) 
-            |> filter(fn: (r) => r["_measurement"] == "agent" and (r["_field"] == "latency"
+            |> filter(fn: (r) => r["_measurement"] == "${measurement}" and (r["_field"] == "latency"
                 or r["_field"] == "duration") ${filterClause})
-            |> ${groupClause}
             |> aggregateWindow(every: 5s, fn: max, createEmpty: false)
 
             dostsquantileex = (q) =>
@@ -163,48 +185,15 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
         }
     }
 
-    #GetSTSQuantileGenericAgentOld = async (filterClause: string, groupClause: string) => {
-        try {
-            const query = `dostsquantileex = (q, d, i, f) =>
-            from(bucket: "${this.options.bucket}") 
-            |> range(start: d) 
-            |> filter(fn: (r) => r["_measurement"] == "agent")
-            |> filter(fn: (r) => r["_field"] == f ${filterClause})
-            |> ${groupClause}
-            |> aggregateWindow(every: i, fn: max, createEmpty: false)
-            |> quantile(q: q, method: "estimate_tdigest", compression: 1000.0)
-            |> set(key: "quantile", value: string(v:q))
-            |> group(columns: ["LatencyType","quantile"])
-            
-            dostsquantile = (d, i, f) =>
-            union(tables: [
-                dostsquantileex(q: 0.5, d: d, i: i, f: f),
-                dostsquantileex(q: 0.8, d: d, i: i, f: f),
-                dostsquantileex(q: 0.9, d: d, i: i, f: f),
-                dostsquantileex(q: 0.95, d: d, i: i, f: f),
-                dostsquantileex(q: 0.99, d: d, i: i, f: f)
-            ])
-            
-            union(tables: [
-                dostsquantile(d: -10m, i: 5s, f: "latency"),
-                dostsquantile(d: -10m, i: 5s, f: "duration")
-            ])`;
-            
-            return this.queryApi.collectRows(query)
-        } catch (error) {
-            console.error(`${_logPrefix}#GetSTSQuantileGeneric: Error: [${error}]`.red);
-        }
-    }
-
     // Metric queries -------------------------------------------------------------------------------------------------
     // Root level metrics
     async GetInfluxDBResultsRootAgent(subscriptionKey: ISubscriptionKey): Promise<ISubscriptionPayload> { // ISubscriptionPayload
         let retVal = null;
         try {
             retVal = await this.ProcessInfluxDBResults([
-                this.#GetSTSCountGenericAgent('', 'group(columns: ["_field"])'),
-                this.#GetSTSQuantileGenericAgent('', 'group(columns: ["_field"])'),
-                this.#GetSTSHistoGenericService('', 'group(columns: ["_field"])')],
+                this.#GetSTSCountGenericAgent(AGENT_STATS_GLOBALAGENT, ''),
+                this.#GetSTSQuantileGenericAgent(AGENT_STATS_GLOBALAGENT, ''),
+                this.#GetSTSHistoGenericService(AGENT_STATS_GLOBALAGENT, '')],
             [ ])
         } catch (error) {
             console.error(`${_logPrefix}GetInfluxDBResultsRootAgent: Error: [${error}]`.red);
@@ -220,9 +209,9 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
         let retVal = null;
         try {
             retVal = await this.ProcessInfluxDBResults([
-                this.#GetSTSCountGenericAgent('', 'group(columns: ["agentName", "_field"])'),
-                this.#GetSTSQuantileGenericAgent('', 'group(columns: ["agentName", "_field"])'),
-                this.#GetSTSHistoGenericService('', 'group(columns: ["agentName", "_field"])')],
+                this.#GetSTSCountGenericAgent(AGENT_STATS_BY_AGENT, ''),
+                this.#GetSTSQuantileGenericAgent(AGENT_STATS_BY_AGENT, ''),
+                this.#GetSTSHistoGenericService(AGENT_STATS_BY_AGENT, '')],
             ['agentName'])
         } catch (error) {
             console.error(`${_logPrefix}GetInfluxDBResultsAgent: Error: [${error}]`.red);
@@ -239,9 +228,9 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
         try {
             const agentName = subscriptionKey.key as string;
             retVal = await this.ProcessInfluxDBResults([
-                this.#GetSTSCountGenericAgent(`and r["agentName"] == "${agentName}"`, `group(columns: ["agentName", "threadId", "_field"])`),
-                this.#GetSTSQuantileGenericAgent(`and r["agentName"] == "${agentName}"`, 'group(columns: ["agentName", "threadId", "_field"])'),
-                this.#GetSTSHistoGenericService(`and r["agentName"] == "${agentName}"`, 'group(columns: ["agentName", "threadId", "_field"])')],
+                this.#GetSTSCountGenericAgent(AGENT_STATS_BY_AGENT_THREAD, `and r["agentName"] == "${agentName}"`),
+                this.#GetSTSQuantileGenericAgent(AGENT_STATS_BY_AGENT_THREAD, `and r["agentName"] == "${agentName}"`),
+                this.#GetSTSHistoGenericService(AGENT_STATS_BY_AGENT_THREAD, `and r["agentName"] == "${agentName}"`)],
             ['agentName', 'threadId'])
         } catch (error) {
             console.error(`${_logPrefix}GetInfluxDBResultsAgentThreads: Error: [${error}]`.red);
@@ -259,9 +248,9 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
             const agentName = subscriptionKey.key as string;
             const threadId = subscriptionKey.subkey as string;
             retVal = await this.ProcessInfluxDBResults([
-                this.#GetSTSCountGenericAgent(`and r["agentName"] == "${agentName}" and r["threadId"] == "${threadId}"`, `group(columns: ["agentName", "threadId", "asyncRunnerId", "_field"])`),
-                this.#GetSTSQuantileGenericAgent(`and r["agentName"] == "${agentName}" and r["threadId"] == "${threadId}"`, 'group(columns: ["agentName", "threadId", "asyncRunnerId", "_field"])'),
-                this.#GetSTSHistoGenericService(`and r["agentName"] == "${agentName}" and r["threadId"] == "${threadId}"`, 'group(columns: ["agentName", "threadId", "asyncRunnerId", "_field"])')],
+                this.#GetSTSCountGenericAgent(AGENT_STATS_BY_AGENT_THREAD_ASYNCRUNNER, `and r["agentName"] == "${agentName}" and r["threadId"] == "${threadId}"`),
+                this.#GetSTSQuantileGenericAgent(AGENT_STATS_BY_AGENT_THREAD_ASYNCRUNNER, `and r["agentName"] == "${agentName}" and r["threadId"] == "${threadId}"`),
+                this.#GetSTSHistoGenericService(AGENT_STATS_BY_AGENT_THREAD_ASYNCRUNNER, `and r["agentName"] == "${agentName}" and r["threadId"] == "${threadId}"`)],
             ['agentName', 'threadId', 'asyncRunnerId'])
         } catch (error) {
             console.error(`${_logPrefix}GetInfluxDBResultsAgentThread: Error: [${error}]`.red);