@nsshunt/stsappframework 3.0.95 → 3.0.96

package/src_working/influxdb/influxDBManager.ts

@@ -0,0 +1,970 @@
+/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
+import { InfluxDB, WriteApi, QueryApi, DEFAULT_WriteOptions, WriteOptions } from '@influxdata/influxdb-client'
+// Use below for production https mode - needs to be config item
+//import { Agent } from 'https'
+import { Agent } from 'http'
+
+import { $Options } from '@nsshunt/stsconfig'
+import { JSONObject } from '@nsshunt/stsutils'
+const goptions = $Options()
+
+import { InfluxDBManagerBase } from './influxDBManagerBase'
+import { InfluxDBManagerService} from './influxDBManagerService'
+import { InfluxDBManagerAgent } from './influxDBManagerAgent'
+import { IInfluxDBManagerOptions, InstrumentPayload } from './../commonTypes'
+
+import { TasksAPI, Tasks, Task } from '@influxdata/influxdb-client-apis'
+
+import debugModule from 'debug'
+const debug = debugModule(`proc:${process.pid}`);
+
+/*
+Manual docker run command and setup -------------------------------------------------------
+
+docker run -d -p 8086:8086 --name influxdb \
+      -v $PWD/data:/var/lib/influxdb2 \
+      -v $PWD/config:/etc/influxdb2 \
+      -e DOCKER_INFLUXDB_INIT_MODE=setup \
+      -e DOCKER_INFLUXDB_INIT_USERNAME=my-user \
+      -e DOCKER_INFLUXDB_INIT_PASSWORD=my-password \
+      -e DOCKER_INFLUXDB_INIT_ORG=my-org \
+      -e DOCKER_INFLUXDB_INIT_BUCKET=my-bucket \
+      -e DOCKER_INFLUXDB_INIT_RETENTION=1w \
+      -e DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token \
+      --restart unless-stopped \
+      influxdb:latest
+
+Then access admin portal via;
+https://192.168.14.92:8086
+
+Login using credentials above
+username: my-user 
+password: my-password
+
+Select Buckets from the menu
+Create buckets as required
+Example: TestBucket01
+Note: We will not use the default bucket as specified in the docker initial setup (my-bucket)
+
+Select API Tokens from the menu
+Generate an API token / custom API token for read/write access to the bucket about
+Bucket: TestBucket01 (from above bucket create)
+
+Copy the API token and use in the ENV file for access
+INFLUXDB_API_TOKEN=<< generated from InfluxDB web admin tool >> \
+INFLUXDB_URL=http://192.168.14.92:8086 \
+INFLUXDB_ORG=my-org \
+INFLUXDB_BUCKET=TestBucket01 \
+
+Docker compose file --------------------------------------------------------------
+
+version: '2'
+services:
+  influxdb:
+    image: influxdb:latest
+    ports:
+      - "8086:8086"
+    environment:
+      DOCKER_INFLUXDB_INIT_MODE: setup
+      DOCKER_INFLUXDB_INIT_USERNAME: my-user
+      DOCKER_INFLUXDB_INIT_PASSWORD: my-password
+      DOCKER_INFLUXDB_INIT_ORG: my-org
+      DOCKER_INFLUXDB_INIT_BUCKET: my-bucket
+      DOCKER_INFLUXDB_INIT_RETENTION: 1w
+      DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: my-super-secret-auth-token
+    volumes:
+      - /var/run/influxdb/data:/var/lib/influxdb2
+      - /var/run/influxdb/config:/etc/influxdb2
+    restart: unless-stopped
+
+
+// Exam Queries Below ----------------------------------------------------------------------
+
+from(bucket: "TestBucket01")
+  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
+  |> filter(fn: (r) => r["_measurement"] == "stsServicePointV3")
+  |> group(columns: ["serviceId"], mode: "by")
+  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)
+  |> yield(name: "mean")
+
+from(bucket: "TestBucket01")
+  |> range(start: -10m)
+  |> filter(fn: (r) => r["_measurement"] == "requestCount")
+  |> group(columns: ["serviceInstanceId"])
+  |> mean()
+
+from(bucket: "TestBucket01")
+  |> range(start: -10m)
+  |> filter(fn: (r) => r["_measurement"] == "cpu")
+  |> group(columns: ["serviceInstanceId"])
+  |> aggregateWindow(every: 1m, fn: mean, createEmpty: false)
+
+from(bucket: "TestBucket01")
+  |> range(start: -2m)
+  |> filter(fn: (r) => r["_measurement"] == "cpu")
+  |> group(columns: ["serviceInstanceId"])
+  |> aggregateWindow(every: 1m, fn: mean, createEmpty: false)
+
+from(bucket: "TestBucket01")
+  |> range(start: -10s)
+  |> filter(fn: (r) => r["_measurement"] == "all")
+  |> filter(fn: (r) => r["_field"] == "requestCount")
+  |> group(columns: ["serviceInstanceId"])
+  |> max()
+
+from(bucket: "TestBucket01")
+  |> range(start: -10s)
+  |> filter(fn: (r) => r["_measurement"] == "all")
+  |> filter(fn: (r) => r["_field"] == "requestCount" or r["_field"] == "errorCount")
+  |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")
+  |> group(columns: ["serviceInstanceId"])
+  |> max()
+
+from(bucket: "TestBucket01") 
+|> range(start: -5s) 
+|> filter(fn: (r) => r["_measurement"] == "all") |> filter(fn: (r) => r["_field"] == "cpu") 
+|> last()
+|> group(columns: ["serviceInstanceId", "serviceId", "_field"]) 
+|> sum()
+|> group(columns: ["_field"]) 
+|> sum()
+
+// Histo ---------------------------------------
+
+import "math"
+from(bucket: "TestBucket01") 
+|> range(start: -10m) 
+|> filter(fn: (r) => r["_measurement"] == "all") |> filter(fn: (r) => r["_field"] == "latency") 
+|> group(columns: ["serviceInstanceIsssd", "serviceId", "_field"]) 
+|> aggregateWindow(every: 1s, fn: mean, createEmpty: false)
+|> histogram(bins: [0.0, 5.0, 10.0, 20.0, 50.0, math.mInf(sign: 1) ])
+|> difference()
+
+// Velocity ------------------------------------------------------------------------
+
+dostsvelocityex = (r, ds, d) =>
+from(bucket: "TestBucket01") 
+|> range(start: r) 
+|> filter(fn: (r) => r["_measurement"] == "all") 
+|> filter(fn: (r) => r["_field"] == "requestCount") 
+|> group(columns: ["serviceInstanceIsssd", "serviceId", "_field", "serviceInstanceProcessId"]) 
+|> aggregateWindow(every: ds, fn: last, createEmpty: false)
+|> limit(n: 5)
+|> difference()
+|> last()
+|> group(columns: ["serviceInstanceIsssd", "serviceId", "_field"])
+|> sum()
+|> map(fn: (r) => ({r with _value: float(v: r._value) / d}))
+
+dostsvelocity = () =>
+    dostsvelocityex(r: -30s, ds: 5s, d: 5.0)
+
+dostsvelocity()
+
+// Velocity - for a specific service instance ----------------------------------
+
+dostsvelocitybyserviceinstanceex = (sid, r, ds, d) =>
+from(bucket: "TestBucket01") 
+|> range(start: r) 
+|> filter(fn: (r) => r["_measurement"] == "all") 
+|> filter(fn: (r) => r["_field"] == "requestCount") 
+|> filter(fn: (r) => r["serviceId"] == sid) 
+|> group(columns: ["serviceInstanceId", "serviceId", "_field", "serviceInstanceProcessId"]) 
+|> aggregateWindow(every: ds, fn: last, createEmpty: false)
+|> limit(n: 5)
+|> difference()
+|> last()
+|> sum()
+|> map(fn: (r) => ({r with _value: float(v: r._value) / d}))
+
+dostsvelocitybyserviceinstance = (sid) =>
+    dostsvelocitybyserviceinstanceex(sid: sid, r: -30s, ds: 5s, d: 5.0)
+
+dostsvelocitybyserviceinstance(sid: "STSAuth@1.0.0")
+
+// Percentile --------------------------------------------------------------
+
+from(bucket: "TestBucket01") 
+|> range(start: -10m) 
+|> filter(fn: (r) => r["_measurement"] == "all") |> filter(fn: (r) => r["_field"] == "latency") 
+|> group(columns: ["serviceInstanceIsssd", "serviceId", "_field"]) 
+|> aggregateWindow(every: 10s, fn: mean, createEmpty: false)
+|> quantile(q: 0.99, method: "exact_mean")
+*/
+
+// export INFLUXDB_TOKEN=W6GztqLmWEqqpP9hAOW9nucgVfESk32dC0JYzFCaYD2yJtS8Ox_WK_1hwSkPUaheWjwwcavkE7_J2aRzm3-E7w==
+
+
+const _logPrefix = 'InfluxDBManager:'
+
+export class InfluxDBManager
+{
+    #shuttingDown: boolean = false;
+
+    //#token = process.env.INFLUXDB_TOKEN;
+    #token = '';
+    #url = '' // 'http://192.168.14.92:8086'
+    #org = '' // `my-org`
+    #bucket = '' // `TestBucket01`
+    #client: InfluxDB;
+    #writeClient: WriteApi | null = null;
+    #queryApi: QueryApi;
+    #writeDataPointFlushTimeout: NodeJS.Timeout | null = null;
+    #agent: Agent | null = null;
+    #influxDBManagerClients: Record<string, InfluxDBManagerBase> = { };
+    #options: IInfluxDBManagerOptions;
+
+    constructor()
+    {
+        this.#options = {
+            token: goptions.influxDB_APIToken,
+            url: goptions.influxDB_Url,
+            org: goptions.influxDB_Org,
+            bucket:goptions.influxDB_Bucket,
+            agent: {
+                influxDB_keepAlive: goptions.influxDB_keepAlive,
+                influxDB_maxSockets: goptions.influxDB_maxSockets,
+                influxDB_maxTotalSockets: goptions.influxDB_maxTotalSockets,
+                influxDB_maxFreeSockets: goptions.influxDB_maxFreeSockets,
+                influxDB_timeout: goptions.influxDB_timeout,
+            }
+        };
+
+        this.#token = this.#options.token;
+        this.#url = this.#options.url;
+        this.#org = this.#options.org;
+        this.#bucket = this.#options.bucket;
+
+        if (this.#options.agent) {
+            this.#agent = new Agent({
+                keepAlive: this.#options.agent.influxDB_keepAlive,
+                keepAliveMsecs: 20000, //@@
+                maxSockets: this.#options.agent.influxDB_maxSockets,
+                maxTotalSockets: this.#options.agent.influxDB_maxTotalSockets,
+                maxFreeSockets: this.#options.agent.influxDB_maxFreeSockets,
+                timeout: this.#options.agent.influxDB_timeout,
+                //rejectUnauthorized: this.#options.agent.influxDB_rejectUnauthorized, // only for SSL
+            })
+
+            this.#client = new InfluxDB({ 
+                url: this.#url, 
+                token: this.#token,
+                transportOptions: { agent: this.#agent }
+            })
+        } else {
+            this.#agent = null;
+
+            this.#client = new InfluxDB({ 
+                url: this.#url, 
+                token: this.#token
+            })
+        }
+
+        this.#BoostrapInfluxDB();
+
+        this.#queryApi = this.#client.getQueryApi(this.#org);
+        
+        this.CreateInfluxDBManagerClient('service');
+        this.CreateInfluxDBManagerClient('agent');
+    }
+
+    #BoostrapInfluxDB = async(): Promise<void> => {
+        const tasksAPI = new TasksAPI(this.#client);
+        const tasks: Tasks = await tasksAPI.getTasks({
+            org: 'my-org'
+        });
+        if (tasks.tasks) {
+            console.log(`[InfluxDB Tasks] - Count: [${tasks.tasks.length}]`.cyan);
+            tasks.tasks.forEach((task: Task) => {
+                console.log(`-- [ Start Task Definition ]--`.cyan);
+                console.log(`${JSON.stringify(task)}`.green);
+                console.log(`authorizationID: [${task.authorizationID}]`.yellow);
+                console.log(`-- [ End Task Definition ]--`.cyan);
+            });
+        } else {
+            console.log(`[InfluxDB Tasks] - Could not get tasks from the InfluxDB API`.magenta);
+        }
+    }
+
+    CreateInfluxDBManagerClient(modelType: string) {
+        switch (modelType) {
+        case 'service':
+            this.#influxDBManagerClients[modelType] = new InfluxDBManagerService(this.#options, this.#queryApi);
+            break;
+        case 'agent':
+            this.#influxDBManagerClients[modelType] = new InfluxDBManagerAgent(this.#options, this.#queryApi);
+            break;
+        default:
+            throw new Error(`Model type: [${modelType} not supported.]`);
+        }
+    }
+
+    get serviceManager(): InfluxDBManagerService {
+        return this.#influxDBManagerClients['service'] as InfluxDBManagerService;
+    }
+
+    get agentManager(): InfluxDBManagerAgent {
+        return this.#influxDBManagerClients['agent'] as InfluxDBManagerAgent;
+    }
+
+    Start = async () => {
+        // perform any setup here
+    }
+
+    Terminate = async () => {
+        if (this.#shuttingDown) {
+            console.log(`${_logPrefix}Terminate: Terminate already called. Ignoring.`.yellow);
+        } else {
+            this.#shuttingDown = true;
+            if (this.#writeClient) {
+                try {
+                    this.#writeClient.flush();
+                    this.#shuttingDown = true;
+                    if (this.#writeDataPointFlushTimeout) {
+                        clearTimeout(this.#writeDataPointFlushTimeout);
+                        this.#writeDataPointFlushTimeout = null;
+                    }
+
+                    if (this.#agent) {
+                        this.#agent?.destroy();
+                        this.#agent = null;
+                    }
+
+                    this.#writeClient = null;
+
+                    console.log(`ProducerDisconnect`.yellow);
+                } catch (error) {
+                    console.error(`${_logPrefix}#Terminate: Error: [${error}]`.red);
+                }
+            }
+        }
+    }
+
+    /*
+    .floatField('timer', instrumentPayload.instruments[Gauge.TIMER_GAUGE].val)
+    .floatField('duration', instrumentPayload.instruments[Gauge.DURATION_GAUGE].val)
+    .floatField('latency', instrumentPayload.instruments[Gauge.LATENCY_GAUGE].val)
+    */
+
+    CreateGlobalCountModel = (stscount: any[]) => {
+        try {
+            const results: any = { };
+            stscount.forEach((o: any) => {
+                results[o._field] = o._value;
+            });
+            return results;
+        } catch (error) {
+            console.error(`${_logPrefix}#CreateGlobalCountModel: Error: [${error}]`.red);
+        }
+    }
+
+
+    CreateNestedObject(combinedResults: JSONObject, keyList: string[]): JSONObject {
+        let workingObject = combinedResults;
+        keyList.forEach((key) => {
+            if (!workingObject[key]) {
+                workingObject[key] = { };
+                workingObject = workingObject[key];
+            }
+        });
+        return combinedResults;
+    }
+
+    get writeClient(): WriteApi {
+        if (!this.#writeClient) {
+            /* Defaults from the library as at 12/11/2023
+            // default RetryDelayStrategyOptions
+            export const DEFAULT_RetryDelayStrategyOptions = {
+                retryJitter: 200,
+                minRetryDelay: 5000,
+                maxRetryDelay: 125000,
+                exponentialBase: 5,
+                randomRetry: true,
+            }
+            
+            // default writeOptions
+            export const DEFAULT_WriteOptions: WriteOptions = {
+                batchSize: 1000,
+                maxBatchBytes: 50_000_000, // default max batch size in the cloud
+                flushInterval: 60000,
+                writeFailed: function () {},
+                writeSuccess: function () {},
+                writeRetrySkipped: function () {},
+                maxRetries: 5,
+                maxRetryTime: 180_000,
+                maxBufferLines: 32_000,
+                // a copy of DEFAULT_RetryDelayStrategyOptions, so that DEFAULT_WriteOptions could be tree-shaken
+                retryJitter: 200,
+                minRetryDelay: 5000,
+                maxRetryDelay: 125000,
+                exponentialBase: 2,
+                gzipThreshold: 1000,
+                randomRetry: true,
+            }
+            */
+    
+            const flushBatchSize = DEFAULT_WriteOptions.batchSize;
+
+            const writeOptions: Partial<WriteOptions> = {
+                /* the maximum points/lines to send in a single batch to InfluxDB server */
+                batchSize: flushBatchSize + 1, // don't let automatically flush data
+                /* default tags to add to every point */
+                //defaultTags: {location: hostname},
+                /* maximum time in millis to keep points in an unflushed batch, 0 means don't periodically flush */
+                flushInterval: 0,
+                // max size of a batch in bytes
+                maxBatchBytes: 50_000_000, // default max batch size in the cloud
+
+                // Retry Options
+
+                /* maximum size of the retry buffer - it contains items that could not be sent for the first time */
+                maxBufferLines: 30000,
+                /* the count of internally-scheduled retries upon write failure, the delays between write attempts follow an exponential backoff strategy if there is no Retry-After HTTP header */
+                maxRetries: 5, // do not retry writes
+                // max time (millis) that can be spent with retries
+                maxRetryTime: 180_000,
+
+                // ... there are more write options that can be customized, see
+                // https://influxdata.github.io/influxdb-client-js/influxdb-client.writeoptions.html and
+                // https://influxdata.github.io/influxdb-client-js/influxdb-client.writeretryoptions.html
+                writeFailed(error, lines, attempt, expires) {
+                    debug(`writeOptions:writeSuccess(): Error: [${error}], Attempt: [${attempt}], Expires: [${expires}], Lines: [${lines}]`.red);
+                },
+                writeSuccess(lines) {
+                    debug(`writeOptions:writeSuccess(): Lines: [${lines}]`.green);
+                },
+                writeRetrySkipped(entry) {
+                    debug(`writeOptions:writeSuccess(): Expires: [${entry.expires}], Lines: [${entry.lines}]`.magenta);
+                }
+            }
+    
+            this.#writeClient = this.#client.getWriteApi(this.#org, this.#bucket, 'ns', writeOptions);
+
+            let attemptCount = 0;
+            const maxAttemptCount = 5;
+            const normalFlushDelay = 1000;
+            const retryFlushDelay = 2500;
+    
+            const StartWriteClient = (delay: number) => {
+                this.#writeDataPointFlushTimeout = setTimeout(async () => {
+                    if (this.#writeClient) {
+                        try {
+                            attemptCount++;
+                            const start = performance.now();
+                            await this.#writeClient.flush();
+                            const end = performance.now();
+                            debug(`InfluxDB Flush time: [${end-start}]`.grey)
+                            attemptCount = 0;
+                            StartWriteClient(normalFlushDelay); 
+                        } catch (error) {
+                            console.error(`InfluxDBManager:StartWriteClient(): Attempt: [${attemptCount}], Error: [${error}]`.red)
+                            if (attemptCount === maxAttemptCount) {
+                                console.error(`InfluxDBManager:StartWriteClient(): Max Attempts reached. Will no longer retry.`.red);
+                                console.error(`InfluxDBManager:StartWriteClient(): InfluxDB data capture ended in permanent faiilure.`.red);
+                            } else {
+                                StartWriteClient(retryFlushDelay);
+                            }
+                        }
+                    }
+                }, delay).unref();
+            }
+    
+            StartWriteClient(normalFlushDelay);
+        }
+        return this.#writeClient;
+    }
+
+    // Agent context payload
+    // {"nid":"somehost@e58f5d75-6ff6-4e04-92a4-f2bcd722fec0-someuseragent|MainProcess|0","id":"somehost@e58f5d75-6ff6-4e04-92a4-f2bcd722fec0-someuseragent|MainProcess|0","hostName":"somehost","agentName":"e58f5d75-6ff6-4e04-92a4-f2bcd722fec0-someuseragent","threadId":"MainProcess","asyncRunnerId":"0"}
+    async OutputInfluxDB(instrumentPayload: InstrumentPayload): Promise<boolean> {
+        try {
+            if (this.#shuttingDown) {
+                return false;
+            }
+            if (instrumentPayload.context.agentName) {
+                return this.#influxDBManagerClients['agent'].OutputInfluxDB(this.writeClient, instrumentPayload);
+            } else {
+                return this.#influxDBManagerClients['service'].OutputInfluxDB(this.writeClient, instrumentPayload);
+            }
+        } catch (error) {
+            console.error(`${_logPrefix}OutputInfluxDB: Could not write data point: [${error}]`.red);
+            return false;
+        }
+    }
+}
+
+
+
+/*
+    |> histogram(bins: [0.0, 10.0, 20.0, 50.0, 100.0, 1000.0, 50000.0, math.mInf(sign: 1) ])
+
+
+    |> histogramQuantile(quantile: 0.99999)    
+
+
+
+option task = {name: "downsample-daily", every: 1d}
+
+
+option task = {name: "downsample-daily", every: 1d}
+
+from(bucket: "my-db/")
+    |> range(start: -task.every)
+    |> filter(fn: (r) => r._measurement == "example-measurement")
+    |> filter(fn: (r) => r._field == "example-field")
+    |> aggregateWindow(every: 1h, fn: mean)
+    |> set(key: "_measurement", value: "average-example-measurement")
+    |> to(org: "example-org", bucket: "my-db/example-rp")
+
+
+
+
+    |> set(key: "_measurement", value: "sts01")
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+
+
+
+
+
+
+
+
+
+from(bucket: "TestBucket01") 
+    |> range(start: -5s) 
+    |> last()
+    |> filter(fn: (r) => r["_measurement"] == "service")
+    |> filter(fn: (r) => r["_field"] != "memory")
+    |> group(columns: ["_field"])
+    |> sum()
+    |> map(
+        fn: (r) => ({r with _time: now(), _measurement: "sts01"}),
+    )    
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+
+
+
+group(columns: ["serviceId", "serviceInstanceId", "_field"])
+
+option task = {name: "task-sts-stats-sum-byserviceInstanceprocessid", every: 1s}
+from(bucket: "TestBucket01")
+    |> range(start: -5s)
+    |> last()
+    |> filter(fn: (r) => r["_measurement"] == "service" and r["_field"] != "memory")
+    |> group(columns: ["serviceId", "serviceInstanceId", "serviceInstanceProcessId", "_field"])
+    |> sum()
+    |> toFloat()
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-sum-byserviceInstanceprocessid"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+option task = {name: "task-sts-stats-sum-byserviceinstance", every: 1s}
+from(bucket: "TestBucket01")
+    |> range(start: -5s)
+    |> last()
+    |> filter(fn: (r) => r["_measurement"] == "service" and r["_field"] != "memory")
+    |> group(columns: ["serviceId", "serviceInstanceId", "_field"])
+    |> sum()
+    |> toFloat()
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-sum-byserviceinstance"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+option task = {name: "task-sts-stats-sum-byservice", every: 1s}
+from(bucket: "TestBucket01")
+    |> range(start: -5s)
+    |> last()
+    |> filter(fn: (r) => r["_measurement"] == "service" and r["_field"] != "memory")
+    |> group(columns: ["serviceId", "_field"])
+    |> sum()
+    |> toFloat()
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-sum-byservice"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+option task = {name: "task-sts-stats-sum", every: 1s}
+from(bucket: "TestBucket01")
+    |> range(start: -5s)
+    |> last()
+    |> filter(fn: (r) => r["_measurement"] == "service" and r["_field"] != "memory")
+    |> group(columns: ["_field"])
+    |> sum()
+    |> toFloat()
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-sum"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+
+
+
+
+option task = {name: "task-sts-stats-sum-byagentthreadasyncrunner-agent", every: 1s}
+from(bucket: "TestBucket01")
+    |> range(start: -5s)
+    |> last()
+    |> filter(fn: (r) => r["_measurement"] == "agent")
+    |> group(columns: ["agentName", "threadId", "asyncRunnerId", "_field"])
+    |> sum()
+    |> toFloat()
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-sum-byagentthreadasyncunner-agent"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+option task = {name: "task-sts-stats-sum-byagentthread-agent", every: 1s}
+from(bucket: "TestBucket01")
+    |> range(start: -5s)
+    |> last()
+    |> filter(fn: (r) => r["_measurement"] == "agent")
+    |> group(columns: ["agentName", "threadId", "_field"])
+    |> sum()
+    |> toFloat()
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-sum-byagentthread-agent"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+option task = {name: "task-sts-stats-sum-byagent-agent", every: 1s}
+from(bucket: "TestBucket01")
+    |> range(start: -5s)
+    |> last()
+    |> filter(fn: (r) => r["_measurement"] == "agent")
+    |> group(columns: ["agentName", "_field"])
+    |> sum()
+    |> toFloat()
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-sum-byagent-agent"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+option task = {name: "task-sts-stats-sum-agent", every: 1s}
+from(bucket: "TestBucket01")
+    |> range(start: -5s)
+    |> last()
+    |> filter(fn: (r) => r["_measurement"] == "agent")
+    |> group(columns: ["_field"])
+    |> sum()
+    |> toFloat()
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-sum-agent"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+
+
+
+
+
+
+
+import "math"
+    from(bucket: "TestBucket01") 
+    |> range(start: -10m) 
+    |> filter(fn: (r) => r["_measurement"] == "sts-stats-sum-byservice" and r["serviceId"] == "STSBroker@1.0.0" and r["_field"] == "cpu")
+    |> histogram(bins: [0.0, 10.0, 20.0, 50.0, 100.0, 200.0, 250.0, 1000.0, 50000.0, math.mInf(sign: 1) ])
+	|> difference()
+
+
+
+import "math"
+    from(bucket: "TestBucket01") 
+    |> range(start: -10m) 
+    |> filter(fn: (r) => r["_measurement"] == "sts-stats-sum" and r["_field"] == "cpu")
+    |> histogram(bins: [0.0, 10.0, 20.0, 50.0, 100.0, 200.0, 250.0, 1000.0, 50000.0, math.mInf(sign: 1) ])
+	|> difference()
+
+
+
+
+// More accurate
+
+data = from(bucket: "TestBucket01") 
+            |> range(start: -10m) 
+            |> filter(fn: (r) => r["_measurement"] == "sts-stats-sum-byservice" and r["serviceId"] == "STSBroker@1.0.0" and r["_field"] == "cpu")
+            |> aggregateWindow(every: 5s, fn: max, createEmpty: false)
+            
+            dostsquantileex = (q) =>
+            data
+            |> quantile(q: q, method: "estimate_tdigest", compression: 1000.0)
+            |> set(key: "quantile", value: string(v:q))
+            |> group(columns: ["quantile"])
+            
+            union(tables: [
+                dostsquantileex(q: 0.5),
+                dostsquantileex(q: 0.8),
+                dostsquantileex(q: 0.9),
+                dostsquantileex(q: 0.95),
+                dostsquantileex(q: 0.99)
+            ])
+
+
+
+
+// Less accurate
+
+import "math"
+data = from(bucket: "TestBucket01") 
+    |> range(start: -10m) 
+    |> filter(fn: (r) => r["_measurement"] == "sts-stats-sum-byservice" and r["serviceId"] == "STSBroker@1.0.0" and r["_field"] == "cpu")
+    |> histogram(bins: [0.0, 10.0, 20.0, 50.0, 100.0, 200.0, 250.0, 1000.0, 50000.0, math.mInf(sign: 1) ])
+
+
+    dostsquantileex = (q) =>
+        data
+            |> histogramQuantile(quantile: q)
+            |> set(key: "quantile", value: string(v:q))
+            |> group(columns: ["quantile"])
+
+union(tables: [
+    dostsquantileex(q: 0.5),
+    dostsquantileex(q: 0.8),
+    dostsquantileex(q: 0.9),
+    dostsquantileex(q: 0.95),
+    dostsquantileex(q: 0.99)
+])
+
+
+
+
+
+
+activeRequestCount	0	2023-11-11T06:23:52.761Z	2023-11-11T06:33:52.761Z	2023-11-11T06:33:52.000Z
+1	sts-stats-sum	authenticationCount	0	2023-11-11T06:23:52.761Z	2023-11-11T06:33:52.761Z	2023-11-11T06:33:52.000Z
+2	sts-stats-sum	connectionCount	40	2023-11-11T06:23:52.761Z	2023-11-11T06:33:52.761Z	2023-11-11T06:33:52.000Z
+3	sts-stats-sum	connectionIdleCount	12	2023-11-11T06:23:52.761Z	2023-11-11T06:33:52.761Z	2023-11-11T06:33:52.000Z
+4	sts-stats-sum	connectionPoolCount	12	2023-11-11T06:23:52.761Z	2023-11-11T06:33:52.761Z	2023-11-11T06:33:52.000Z
+5	sts-stats-sum	connectionWaitingCount	0	2023-11-11T06:23:52.761Z	2023-11-11T06:33:52.761Z	2023-11-11T06:33:52.000Z
+6	sts-stats-sum	coreCount
+
+
+cpu	50.91	2023-11-11T06:23:52.761Z	2023-11-11T06:33:52.761Z	2023-11-11T06:33:52.000Z
+8	sts-stats-sum	duration	3.6900000000000004	2023-11-11T06:23:52.761Z	2023-11-11T06:33:52.761Z	2023-11-11T06:33:52.000Z
+9	sts-stats-sum	errorCount	0	2023-11-11T06:23:52.761Z	2023-11-11T06:33:52.761Z	2023-11-11T06:33:52.000Z
+10	sts-stats-sum	latency	22.82	2023-11-11T06:23:52.761Z	2023-11-11T06:33:52.761Z	2023-11-11T06:33:52.000Z
+11	sts-stats-sum	requestCount	4485612	2023-11-11T06:23:52.761Z	2023-11-11T06:33:52.761Z	2023-11-11T06:33:52.000Z
+12	sts-stats-sum	retryCount	0	2023-11-11T06:23:52.761Z	2023-11-11T06:33:52.761Z	2023-11-11T06:33:52.000Z
+13	sts-stats-sum	systemcpu
+
+timer	72578365.22	2023-11-11T06:23:52.761Z	2023-11-11T06:33:52.761Z	2023-11-11T06:33:52.000Z
+15	sts-stats-sum	velocity
+
+
+
+
+
+
+
+
+
+
+
+
+
+option task = { 
+  name: "sts-service-stats",
+  every: 1s,
+}
+
+data = from(bucket: "TestBucket01")
+    |> range(start: -5s)
+    |> last()
+    |> filter(fn: (r) => r["_measurement"] == "service" and r["_field"] != "memory")
+
+r1 = data
+    |> filter(fn: (r) => r["_field"] == "requestCount"
+		or r["_field"] == "errorCount"
+        or r["_field"] == "retryCount"
+        or r["_field"] == "authenticationCount"
+        or r["_field"] == "activeRequestCount"
+        or r["_field"] == "connectionCount"
+        or r["_field"] == "connectionPoolCount"
+        or r["_field"] == "connectionIdleCount"
+        or r["_field"] == "connectionWaitingCount"
+        or r["_field"] == "coreCount"
+        or r["_field"] == "cpu"
+        or r["_field"] == "systemcpu"
+        or r["_field"] == "velocity"
+        or r["_field"] == "timer")
+    |> group(columns: ["_field"])
+    |> sum()
+    |> toFloat()
+
+r2 = data
+    |> filter(fn: (r) => float(v: r["_value"]) > 0.0 and (r["_field"] == "duration"
+		or r["_field"] == "latency"))
+
+serviceInstanceProcessSum = r1
+    |> group(columns: ["serviceId", "serviceInstanceId", "serviceInstanceProcessId", "_field"])
+    |> sum()
+    |> toFloat()
+
+serviceInstanceProcessMean = r2
+    |> group(columns: ["serviceId", "serviceInstanceId", "serviceInstanceProcessId", "_field"])
+    |> mean()
+    |> toFloat()
+
+union(tables: [serviceInstanceProcessSum, serviceInstanceProcessMean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-by-serviceinstanceprocess"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+serviceInstanceSum = r1
+    |> group(columns: ["serviceId", "serviceInstanceId", "_field"])
+    |> sum()
+    |> toFloat()
+
+serviceInstanceMean = r2
+    |> group(columns: ["serviceId", "serviceInstanceId", "_field"])
+    |> mean()
+    |> toFloat()
+
+union(tables: [serviceInstanceSum, serviceInstanceMean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-by-serviceinstance"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+serviceSum = r1
+    |> group(columns: ["serviceId", "_field"])
+    |> sum()
+    |> toFloat()
+
+serviceMean = r2
+    |> group(columns: ["serviceId", "_field"])
+    |> mean()
+    |> toFloat()
+
+union(tables: [serviceSum, serviceMean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-by-service"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+globalServiceSum = r1
+    |> group(columns: ["_field"])
+    |> sum()
+    |> toFloat()
+
+globalServiceMean = r2
+    |> group(columns: ["_field"])
+    |> mean()
+    |> toFloat()
+
+union(tables: [globalServiceSum, globalServiceMean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-globalservice"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+option task = {name: "task-sts-service-stats", every: 1s}
+
+data =
+    from(bucket: "TestBucket01")
+        |> range(start: -5s)
+        |> last()
+        |> filter(fn: (r) => r["_measurement"] == "service" and r["_field"] != "memory")
+
+r1 =
+    data
+        |> filter(
+            fn: (r) =>
+                r["_field"] == "requestCount" or r["_field"] == "errorCount" or r["_field"]
+                    ==
+                    "retryCount" or r["_field"] == "authenticationCount" or r["_field"]
+                    ==
+                    "activeRequestCount" or r["_field"] == "connectionCount" or r["_field"]
+                    ==
+                    "connectionPoolCount" or r["_field"] == "connectionIdleCount" or r["_field"]
+                    ==
+                    "connectionWaitingCount" or r["_field"] == "coreCount" or r["_field"] == "cpu"
+                    or
+                    r["_field"] == "systemcpu" or r["_field"] == "velocity" or r["_field"]
+                    ==
+                    "timer",
+        )
+
+r2 =
+    data
+        |> filter(
+            fn: (r) =>
+                float(v: r["_value"]) > 0.0 and (r["_field"] == "duration" or r["_field"]
+                        ==
+                        "latency"),
+        )
+
+serviceInstanceProcessSum =
+    r1
+        |> group(columns: ["serviceId", "serviceInstanceId", "serviceInstanceProcessId", "_field"])
+        |> sum()
+        |> toFloat()
+
+serviceInstanceProcessMean =
+    r2
+        |> group(columns: ["serviceId", "serviceInstanceId", "serviceInstanceProcessId", "_field"])
+        |> mean()
+        |> toFloat()
+
+union(tables: [serviceInstanceProcessSum, serviceInstanceProcessMean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-by-serviceinstanceprocess"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+serviceInstanceSum =
+    r1
+        |> group(columns: ["serviceId", "serviceInstanceId", "_field"])
+        |> sum()
+        |> toFloat()
+
+serviceInstanceMean =
+    r2
+        |> group(columns: ["serviceId", "serviceInstanceId", "_field"])
+        |> mean()
+        |> toFloat()
+
+union(tables: [serviceInstanceSum, serviceInstanceMean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-by-serviceinstance"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+serviceSum =
+    r1
+        |> group(columns: ["serviceId", "_field"])
+        |> sum()
+        |> toFloat()
+
+serviceMean =
+    r2
+        |> group(columns: ["serviceId", "_field"])
+        |> mean()
+        |> toFloat()
+
+union(tables: [serviceSum, serviceMean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-by-service"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+globalServiceSum =
+    r1
+        |> group(columns: ["_field"])
+        |> sum()
+        |> toFloat()
+
+globalServiceMean =
+    r2
+        |> group(columns: ["_field"])
+        |> mean()
+        |> toFloat()
+
+union(tables: [globalServiceSum, globalServiceMean])
+    |> map(fn: (r) => ({r with _time: now(), _measurement: "sts-stats-globalservice"}))
+    |> to(org: "my-org", bucket: "TestBucket01")
+
+
+
+
+    
+*/