@nsshunt/stsappframework 3.1.50 → 3.1.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +0 -1
- package/dist/index.js.map +1 -1
- package/dist/influxdb/influxDBManagerAgent.js +0 -144
- package/dist/influxdb/influxDBManagerAgent.js.map +1 -1
- package/dist/influxdb/influxDBManagerService.js +0 -170
- package/dist/influxdb/influxDBManagerService.js.map +1 -1
- package/package.json +1 -1
- package/src/index.ts +0 -1
- package/src/influxdb/influxDBManagerAgent.ts +3 -0
- package/src/influxdb/influxDBManagerService.ts +6 -22
- package/types/index.d.ts +0 -1
- package/types/index.d.ts.map +1 -1
- package/types/influxdb/influxDBManagerAgent.d.ts +0 -6
- package/types/influxdb/influxDBManagerAgent.d.ts.map +1 -1
- package/types/influxdb/influxDBManagerService.d.ts +0 -6
- package/types/influxdb/influxDBManagerService.d.ts.map +1 -1
- package/dist/instrumentationsubscriber.js +0 -304
- package/dist/instrumentationsubscriber.js.map +0 -1
- package/src/instrumentationsubscriber.ts +0 -336
- package/types/instrumentationsubscriber.d.ts +0 -18
- package/types/instrumentationsubscriber.d.ts.map +0 -1
package/dist/index.js
CHANGED
|
@@ -37,7 +37,6 @@ __exportStar(require("./influxdb/influxDBManagerAgent"), exports);
|
|
|
37
37
|
__exportStar(require("./influxdb/influxDBManagerLambda"), exports);
|
|
38
38
|
__exportStar(require("./publishertransports/publishTransportUtils"), exports);
|
|
39
39
|
__exportStar(require("./publishertransports/publishTransportDirect"), exports);
|
|
40
|
-
__exportStar(require("./instrumentationsubscriber"), exports);
|
|
41
40
|
__exportStar(require("./stsTransportWinston"), exports);
|
|
42
41
|
__exportStar(require("./stsTransportLoggerWinston"), exports);
|
|
43
42
|
__exportStar(require("./globalServiceDefinitions"), exports);
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,gDAA6B;AAC7B,mDAAgC;AAChC,6CAA0B;AAC1B,kDAA+B;AAC/B,kDAA+B;AAC/B,sDAAmC;AACnC,sDAAmC;AACnC,sDAAmC;AACnC,sDAAmC;AACnC,yDAAsC;AACtC,yDAAsC;AACtC,oDAAiC;AACjC,uDAAoC;AACpC,wDAAqC;AACrC,wDAAqC;AACrC,yDAAsC;AACtC,6DAA0C;AAC1C,iEAA8C;AAC9C,oEAAiD;AACjD,kEAA+C;AAC/C,mEAAgD;AAChD,8EAA2D;AAC3D,+EAA4D;AAC5D,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,gDAA6B;AAC7B,mDAAgC;AAChC,6CAA0B;AAC1B,kDAA+B;AAC/B,kDAA+B;AAC/B,sDAAmC;AACnC,sDAAmC;AACnC,sDAAmC;AACnC,sDAAmC;AACnC,yDAAsC;AACtC,yDAAsC;AACtC,oDAAiC;AACjC,uDAAoC;AACpC,wDAAqC;AACrC,wDAAqC;AACrC,yDAAsC;AACtC,6DAA0C;AAC1C,iEAA8C;AAC9C,oEAAiD;AACjD,kEAA+C;AAC/C,mEAAgD;AAChD,8EAA2D;AAC3D,+EAA4D;AAC5D,wDAAqC;AACrC,8DAA2C;AAC3C,6DAA0C"}
|
|
@@ -105,7 +105,6 @@ union(tables: [globalagentsum, globalagentmean])
|
|
|
105
105
|
const debug_1 = __importDefault(require("debug"));
|
|
106
106
|
const debug = (0, debug_1.default)(`proc:${process.pid}`);
|
|
107
107
|
const influxDBManagerBase_1 = require("./influxDBManagerBase");
|
|
108
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
109
108
|
const AGENT_STATS_BY_AGENT_THREAD_ASYNCRUNNER = "sts-stats-by-agentthreadasyncunner";
|
|
110
109
|
const AGENT_STATS_BY_AGENT_THREAD = "sts-stats-by-agentthread";
|
|
111
110
|
const AGENT_STATS_BY_AGENT = "sts-stats-by-agent";
|
|
@@ -118,149 +117,6 @@ class InfluxDBManagerAgent extends influxDBManagerBase_1.InfluxDBManagerBase {
|
|
|
118
117
|
get modelType() {
|
|
119
118
|
return 'agent';
|
|
120
119
|
}
|
|
121
|
-
#LogErrorMessage(message) {
|
|
122
|
-
this.options.logger.error(message);
|
|
123
|
-
}
|
|
124
|
-
#LogDebugMessage(message) {
|
|
125
|
-
this.options.logger.debug(message);
|
|
126
|
-
}
|
|
127
|
-
// Queries --------------------------------------------------------------------------------------------------------
|
|
128
|
-
// Counter metrics
|
|
129
|
-
#GetSTSCountGenericAgent = async (measurement, filterClause, showOutput = false) => {
|
|
130
|
-
try {
|
|
131
|
-
const query = `from(bucket: "${this.options.bucket}")
|
|
132
|
-
|> range(start: -5s)
|
|
133
|
-
|> last()
|
|
134
|
-
|> filter(fn: (r) => r["_measurement"] == "${measurement}" ${filterClause})`;
|
|
135
|
-
if (showOutput) {
|
|
136
|
-
this.#LogDebugMessage(`GetSTSCountGenericAgent(): query: [${query}]`);
|
|
137
|
-
}
|
|
138
|
-
return this.queryApi.collectRows(query);
|
|
139
|
-
}
|
|
140
|
-
catch (error) {
|
|
141
|
-
this.#LogErrorMessage(chalk_1.default.red(`${_logPrefix}#GetSTSCountGeneric: Error: [${error}]`));
|
|
142
|
-
}
|
|
143
|
-
};
|
|
144
|
-
// Histo metrics
|
|
145
|
-
#GetSTSHistoGenericService = async (measurement, filterClause) => {
|
|
146
|
-
try {
|
|
147
|
-
const query = `import "math"
|
|
148
|
-
from(bucket: "${this.options.bucket}")
|
|
149
|
-
|> range(start: -10m)
|
|
150
|
-
|> filter(fn: (r) => r["_measurement"] == "${measurement}" and (r["_field"] =="latency"
|
|
151
|
-
or r["_field"] =="duration") ${filterClause})
|
|
152
|
-
|> histogram(bins: [0.0, 10.0, 20.0, 50.0, 100.0, 1000.0, 50000.0, math.mInf(sign: 1) ])
|
|
153
|
-
|> difference()`;
|
|
154
|
-
return this.queryApi.collectRows(query);
|
|
155
|
-
}
|
|
156
|
-
catch (error) {
|
|
157
|
-
this.#LogErrorMessage(chalk_1.default.red(`${_logPrefix}#GetSTSHistoGeneric: Error: [${error}]`));
|
|
158
|
-
}
|
|
159
|
-
};
|
|
160
|
-
// Quantile metrics
|
|
161
|
-
#GetSTSQuantileGenericAgent = async (measurement, filterClause) => {
|
|
162
|
-
try {
|
|
163
|
-
const query = `data = from(bucket: "${this.options.bucket}")
|
|
164
|
-
|> range(start: -10m)
|
|
165
|
-
|> filter(fn: (r) => r["_measurement"] == "${measurement}" and (r["_field"] == "latency"
|
|
166
|
-
or r["_field"] == "duration") ${filterClause})
|
|
167
|
-
|> aggregateWindow(every: 5s, fn: max, createEmpty: false)
|
|
168
|
-
|
|
169
|
-
dostsquantileex = (q) =>
|
|
170
|
-
data
|
|
171
|
-
|> quantile(q: q, method: "estimate_tdigest", compression: 1000.0)
|
|
172
|
-
|> set(key: "quantile", value: string(v:q))
|
|
173
|
-
|> group(columns: ["quantile"])
|
|
174
|
-
|
|
175
|
-
union(tables: [
|
|
176
|
-
dostsquantileex(q: 0.5),
|
|
177
|
-
dostsquantileex(q: 0.8),
|
|
178
|
-
dostsquantileex(q: 0.9),
|
|
179
|
-
dostsquantileex(q: 0.95),
|
|
180
|
-
dostsquantileex(q: 0.99)
|
|
181
|
-
])`;
|
|
182
|
-
return this.queryApi.collectRows(query);
|
|
183
|
-
}
|
|
184
|
-
catch (error) {
|
|
185
|
-
this.#LogErrorMessage(chalk_1.default.red(`${_logPrefix}#GetSTSQuantileGeneric: Error: [${error}]`));
|
|
186
|
-
}
|
|
187
|
-
};
|
|
188
|
-
// Metric queries -------------------------------------------------------------------------------------------------
|
|
189
|
-
// Root level metrics
|
|
190
|
-
async GetInfluxDBResultsRootAgent(subscriptionKey) {
|
|
191
|
-
let retVal = null;
|
|
192
|
-
try {
|
|
193
|
-
retVal = await this.ProcessInfluxDBResults([
|
|
194
|
-
this.#GetSTSCountGenericAgent(AGENT_STATS_GLOBALAGENT, ''),
|
|
195
|
-
this.#GetSTSQuantileGenericAgent(AGENT_STATS_GLOBALAGENT, ''),
|
|
196
|
-
this.#GetSTSHistoGenericService(AGENT_STATS_GLOBALAGENT, '')
|
|
197
|
-
], []);
|
|
198
|
-
}
|
|
199
|
-
catch (error) {
|
|
200
|
-
this.#LogErrorMessage(chalk_1.default.red(`${_logPrefix}GetInfluxDBResultsRootAgent: Error: [${error}]`));
|
|
201
|
-
}
|
|
202
|
-
return {
|
|
203
|
-
subscriptionKey,
|
|
204
|
-
data: retVal
|
|
205
|
-
};
|
|
206
|
-
}
|
|
207
|
-
// Service level metrics
|
|
208
|
-
async GetInfluxDBResultsAgent(subscriptionKey) {
|
|
209
|
-
let retVal = null;
|
|
210
|
-
try {
|
|
211
|
-
retVal = await this.ProcessInfluxDBResults([
|
|
212
|
-
this.#GetSTSCountGenericAgent(AGENT_STATS_BY_AGENT, ''),
|
|
213
|
-
this.#GetSTSQuantileGenericAgent(AGENT_STATS_BY_AGENT, ''),
|
|
214
|
-
this.#GetSTSHistoGenericService(AGENT_STATS_BY_AGENT, '')
|
|
215
|
-
], ['agentName']);
|
|
216
|
-
}
|
|
217
|
-
catch (error) {
|
|
218
|
-
this.#LogErrorMessage(chalk_1.default.red(`${_logPrefix}GetInfluxDBResultsAgent: Error: [${error}]`));
|
|
219
|
-
}
|
|
220
|
-
return {
|
|
221
|
-
subscriptionKey,
|
|
222
|
-
data: retVal
|
|
223
|
-
};
|
|
224
|
-
}
|
|
225
|
-
// Service instance metrics for a particular service type
|
|
226
|
-
async GetInfluxDBResultsAgentThreads(subscriptionKey) {
|
|
227
|
-
let retVal = null;
|
|
228
|
-
try {
|
|
229
|
-
const agentName = subscriptionKey.key;
|
|
230
|
-
retVal = await this.ProcessInfluxDBResults([
|
|
231
|
-
this.#GetSTSCountGenericAgent(AGENT_STATS_BY_AGENT_THREAD, `and r["agentName"] == "${agentName}"`),
|
|
232
|
-
this.#GetSTSQuantileGenericAgent(AGENT_STATS_BY_AGENT_THREAD, `and r["agentName"] == "${agentName}"`),
|
|
233
|
-
this.#GetSTSHistoGenericService(AGENT_STATS_BY_AGENT_THREAD, `and r["agentName"] == "${agentName}"`)
|
|
234
|
-
], ['agentName', 'threadId']);
|
|
235
|
-
}
|
|
236
|
-
catch (error) {
|
|
237
|
-
this.#LogErrorMessage(chalk_1.default.red(`${_logPrefix}GetInfluxDBResultsAgentThreads: Error: [${error}]`));
|
|
238
|
-
}
|
|
239
|
-
return {
|
|
240
|
-
subscriptionKey,
|
|
241
|
-
data: retVal
|
|
242
|
-
};
|
|
243
|
-
}
|
|
244
|
-
// Service instance thread metrics for a particular service instance
|
|
245
|
-
async GetInfluxDBResultsAgentThread(subscriptionKey) {
|
|
246
|
-
let retVal = null;
|
|
247
|
-
try {
|
|
248
|
-
const agentName = subscriptionKey.key;
|
|
249
|
-
const threadId = subscriptionKey.subkey;
|
|
250
|
-
retVal = await this.ProcessInfluxDBResults([
|
|
251
|
-
this.#GetSTSCountGenericAgent(AGENT_STATS_BY_AGENT_THREAD_ASYNCRUNNER, `and r["agentName"] == "${agentName}" and r["threadId"] == "${threadId}"`),
|
|
252
|
-
this.#GetSTSQuantileGenericAgent(AGENT_STATS_BY_AGENT_THREAD_ASYNCRUNNER, `and r["agentName"] == "${agentName}" and r["threadId"] == "${threadId}"`),
|
|
253
|
-
this.#GetSTSHistoGenericService(AGENT_STATS_BY_AGENT_THREAD_ASYNCRUNNER, `and r["agentName"] == "${agentName}" and r["threadId"] == "${threadId}"`)
|
|
254
|
-
], ['agentName', 'threadId', 'asyncRunnerId']);
|
|
255
|
-
}
|
|
256
|
-
catch (error) {
|
|
257
|
-
this.#LogErrorMessage(chalk_1.default.red(`${_logPrefix}GetInfluxDBResultsAgentThread: Error: [${error}]`));
|
|
258
|
-
}
|
|
259
|
-
return {
|
|
260
|
-
subscriptionKey,
|
|
261
|
-
data: retVal
|
|
262
|
-
};
|
|
263
|
-
}
|
|
264
120
|
}
|
|
265
121
|
exports.InfluxDBManagerAgent = InfluxDBManagerAgent;
|
|
266
122
|
//# sourceMappingURL=influxDBManagerAgent.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"influxDBManagerAgent.js","sourceRoot":"","sources":["../../src/influxdb/influxDBManagerAgent.ts"],"names":[],"mappings":";AAAA,wFAAwF,CAAE,UAAU;AACpG,4FAA4F;AAC5F,+GAA+G;;;;;;AAE/G;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA8FE;AAEF,kDAA+B;AAC/B,MAAM,KAAK,GAAG,IAAA,eAAW,EAAC,QAAQ,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;AAMjD,+DAA2D;
|
|
1
|
+
{"version":3,"file":"influxDBManagerAgent.js","sourceRoot":"","sources":["../../src/influxdb/influxDBManagerAgent.ts"],"names":[],"mappings":";AAAA,wFAAwF,CAAE,UAAU;AACpG,4FAA4F;AAC5F,+GAA+G;;;;;;AAE/G;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA8FE;AAEF,kDAA+B;AAC/B,MAAM,KAAK,GAAG,IAAA,eAAW,EAAC,QAAQ,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;AAMjD,+DAA2D;AAQ3D,MAAM,uCAAuC,GAAG,oCAAoC,CAAC;AACrF,MAAM,2BAA2B,GAAG,0BAA0B,CAAC;AAC/D,MAAM,oBAAoB,GAAG,oBAAoB,CAAC;AAClD,MAAM,uBAAuB,GAAG,uBAAuB,CAAC;AAExD,MAAM,UAAU,GAAG,yBAAyB,CAAA;AAE5C,MAAa,oBAAqB,SAAQ,yCAAmB;IAEzD,YAAY,OAAgC,EAAE,QAAkB;QAC5D,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAED,IAAa,SAAS;QAClB,OAAO,OAAO,CAAA;IAClB,CAAC;CAyJJ;AAjKD,oDAiKC"}
|
|
@@ -2,13 +2,9 @@
|
|
|
2
2
|
/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */ // --> OFF
|
|
3
3
|
// https://www.influxdata.com/blog/tldr-influxdb-tech-tips-multiple-aggregations-yield-flux/
|
|
4
4
|
// https://www.influxdata.com/blog/top-5-hurdles-for-intermediate-flux-users-and-resources-for-optimizing-flux/
|
|
5
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
6
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
7
|
-
};
|
|
8
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
6
|
exports.InfluxDBManagerService = void 0;
|
|
10
7
|
const influxDBManagerBase_1 = require("./influxDBManagerBase");
|
|
11
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
12
8
|
const _logPrefix = 'InfluxDBManagerService:';
|
|
13
9
|
/*
|
|
14
10
|
const SERVICE_STATS_BY_SERVICE_INSTANCE_PROCESS = "sts-stats-sum-byserviceInstanceprocessid";
|
|
@@ -24,175 +20,9 @@ class InfluxDBManagerService extends influxDBManagerBase_1.InfluxDBManagerBase {
|
|
|
24
20
|
constructor(options, queryApi) {
|
|
25
21
|
super(options, queryApi);
|
|
26
22
|
}
|
|
27
|
-
#LogErrorMessage(message) {
|
|
28
|
-
this.options.logger.error(message);
|
|
29
|
-
}
|
|
30
|
-
#LogDebugMessage(message) {
|
|
31
|
-
this.options.logger.debug(message);
|
|
32
|
-
}
|
|
33
23
|
get modelType() {
|
|
34
24
|
return 'service';
|
|
35
25
|
}
|
|
36
|
-
// Queries --------------------------------------------------------------------------------------------------------
|
|
37
|
-
/*
|
|
38
|
-
data = from(bucket: "my-bucket")
|
|
39
|
-
|> range(start: -task.every)
|
|
40
|
-
|> filter(fn: (r) => r._measurement == "my_measurement")
|
|
41
|
-
|
|
42
|
-
data
|
|
43
|
-
|> mean()
|
|
44
|
-
|> set(key: "agg_type",value: "mean_temp")
|
|
45
|
-
|> to(bucket: "downsampled", org: "my-org", tagColumns: ["agg_type"])
|
|
46
|
-
|
|
47
|
-
data
|
|
48
|
-
|> count()
|
|
49
|
-
|> set(key: "agg_type",value: "count_temp")
|
|
50
|
-
|> to(bucket: "downsampled", org: "my-org", tagColumns: ["agg_type"])
|
|
51
|
-
*/
|
|
52
|
-
#GetSTSCountGenericService = async (measurement, filterClause, showOutput = false) => {
|
|
53
|
-
try {
|
|
54
|
-
const query = `r1 = from(bucket: "${this.options.bucket}")
|
|
55
|
-
|> range(start: -5s)
|
|
56
|
-
|> last()
|
|
57
|
-
|> filter(fn: (r) => r["_measurement"] == "${measurement}" ${filterClause})
|
|
58
|
-
|
|
59
|
-
r2 = from(bucket: "${this.options.bucket}")
|
|
60
|
-
|> range(start: -5s)
|
|
61
|
-
|> last()
|
|
62
|
-
|> filter(fn: (r) => r["_measurement"] == "service" ${filterClause} and r["_field"] == "memory")
|
|
63
|
-
|
|
64
|
-
union(tables: [r1, r2])`;
|
|
65
|
-
if (showOutput) {
|
|
66
|
-
this.#LogDebugMessage(`InfluxDBManagerService:GetSTSCountGenericService(): query: [${query}]`);
|
|
67
|
-
}
|
|
68
|
-
return this.queryApi.collectRows(query);
|
|
69
|
-
}
|
|
70
|
-
catch (error) {
|
|
71
|
-
this.#LogErrorMessage(chalk_1.default.red(`${_logPrefix}#GetSTSCountGeneric: Error: [${error}]`));
|
|
72
|
-
}
|
|
73
|
-
};
|
|
74
|
-
// Histo metrics
|
|
75
|
-
#GetSTSHistoGenericService = async (measurement, filterClause) => {
|
|
76
|
-
try {
|
|
77
|
-
const query = `import "math"
|
|
78
|
-
from(bucket: "${this.options.bucket}")
|
|
79
|
-
|> range(start: -10m)
|
|
80
|
-
|> filter(fn: (r) => r["_measurement"] == "${measurement}" and (r["_field"] =="latency" or
|
|
81
|
-
r["_field"] =="duration") ${filterClause})
|
|
82
|
-
|> histogram(bins: [0.0, 10.0, 20.0, 50.0, 100.0, 1000.0, 50000.0, math.mInf(sign: 1) ])
|
|
83
|
-
|> difference()`;
|
|
84
|
-
return this.queryApi.collectRows(query);
|
|
85
|
-
}
|
|
86
|
-
catch (error) {
|
|
87
|
-
this.#LogErrorMessage(chalk_1.default.red(`${_logPrefix}#GetSTSHistoGeneric: Error: [${error}]`));
|
|
88
|
-
}
|
|
89
|
-
};
|
|
90
|
-
// Quantile metrics
|
|
91
|
-
#GetSTSQuantileGenericService = async (measurement, filterClause) => {
|
|
92
|
-
try {
|
|
93
|
-
const query = `data = from(bucket: "${this.options.bucket}")
|
|
94
|
-
|> range(start: -10m)
|
|
95
|
-
|> filter(fn: (r) => r["_measurement"] == "${measurement}" and (r["_field"] == "latency" or
|
|
96
|
-
r["_field"] == "duration" or
|
|
97
|
-
r["_field"] == "cpu" or
|
|
98
|
-
r["_field"] == "systemcpu") ${filterClause})
|
|
99
|
-
|> aggregateWindow(every: 5s, fn: max, createEmpty: false)
|
|
100
|
-
|
|
101
|
-
dostsquantileex = (q) =>
|
|
102
|
-
data
|
|
103
|
-
|> quantile(q: q, method: "estimate_tdigest", compression: 1000.0)
|
|
104
|
-
|> set(key: "quantile", value: string(v:q))
|
|
105
|
-
|> group(columns: ["quantile"])
|
|
106
|
-
|
|
107
|
-
union(tables: [
|
|
108
|
-
dostsquantileex(q: 0.5),
|
|
109
|
-
dostsquantileex(q: 0.8),
|
|
110
|
-
dostsquantileex(q: 0.9),
|
|
111
|
-
dostsquantileex(q: 0.95),
|
|
112
|
-
dostsquantileex(q: 0.99)
|
|
113
|
-
])`;
|
|
114
|
-
return this.queryApi.collectRows(query);
|
|
115
|
-
}
|
|
116
|
-
catch (error) {
|
|
117
|
-
this.#LogErrorMessage(chalk_1.default.red(`${_logPrefix}#GetSTSQuantileGeneric: Error: [${error}]`));
|
|
118
|
-
}
|
|
119
|
-
};
|
|
120
|
-
// Metric queries -------------------------------------------------------------------------------------------------
|
|
121
|
-
// Root level metrics
|
|
122
|
-
async GetInfluxDBResultsRootService(subscriptionKey) {
|
|
123
|
-
let retVal = null;
|
|
124
|
-
try {
|
|
125
|
-
retVal = await this.ProcessInfluxDBResults([
|
|
126
|
-
this.#GetSTSCountGenericService(SERVICE_STATS_GLOBAL, ''),
|
|
127
|
-
this.#GetSTSQuantileGenericService(SERVICE_STATS_GLOBAL, ''),
|
|
128
|
-
this.#GetSTSHistoGenericService(SERVICE_STATS_GLOBAL, '')
|
|
129
|
-
], []);
|
|
130
|
-
}
|
|
131
|
-
catch (error) {
|
|
132
|
-
this.#LogErrorMessage(chalk_1.default.red(`${_logPrefix}GetInfluxDBResultsRoot: Error: [${error}]`));
|
|
133
|
-
}
|
|
134
|
-
return {
|
|
135
|
-
subscriptionKey,
|
|
136
|
-
data: retVal
|
|
137
|
-
};
|
|
138
|
-
}
|
|
139
|
-
// Service level metrics
|
|
140
|
-
async GetInfluxDBResultsService(subscriptionKey) {
|
|
141
|
-
let retVal = null;
|
|
142
|
-
try {
|
|
143
|
-
retVal = await this.ProcessInfluxDBResults([
|
|
144
|
-
this.#GetSTSCountGenericService(SERVICE_STATS_BY_SERVICE, ''),
|
|
145
|
-
this.#GetSTSQuantileGenericService(SERVICE_STATS_BY_SERVICE, ''),
|
|
146
|
-
this.#GetSTSHistoGenericService(SERVICE_STATS_BY_SERVICE, '')
|
|
147
|
-
], ['serviceId']);
|
|
148
|
-
}
|
|
149
|
-
catch (error) {
|
|
150
|
-
this.#LogErrorMessage(chalk_1.default.red(`${_logPrefix}GetInfluxDBResultsService: Error: [${error}]`));
|
|
151
|
-
}
|
|
152
|
-
return {
|
|
153
|
-
subscriptionKey,
|
|
154
|
-
data: retVal
|
|
155
|
-
};
|
|
156
|
-
}
|
|
157
|
-
// Service instance metrics for a particular service type
|
|
158
|
-
async GetInfluxDBResultsServiceInstances(subscriptionKey) {
|
|
159
|
-
let retVal = null;
|
|
160
|
-
try {
|
|
161
|
-
const serviceId = subscriptionKey.key;
|
|
162
|
-
retVal = await this.ProcessInfluxDBResults([
|
|
163
|
-
this.#GetSTSCountGenericService(SERVICE_STATS_BY_SERVICE_INSTANCE, `and r["serviceId"] == "${serviceId}"`),
|
|
164
|
-
this.#GetSTSQuantileGenericService(SERVICE_STATS_BY_SERVICE_INSTANCE, `and r["serviceId"] == "${serviceId}"`),
|
|
165
|
-
this.#GetSTSHistoGenericService(SERVICE_STATS_BY_SERVICE_INSTANCE, `and r["serviceId"] == "${serviceId}"`)
|
|
166
|
-
], ['serviceId', 'serviceInstanceId']);
|
|
167
|
-
}
|
|
168
|
-
catch (error) {
|
|
169
|
-
this.#LogErrorMessage(chalk_1.default.red(`${_logPrefix}GetInfluxDBResultsServiceInstances: Error: [${error}]`));
|
|
170
|
-
}
|
|
171
|
-
return {
|
|
172
|
-
subscriptionKey,
|
|
173
|
-
data: retVal
|
|
174
|
-
};
|
|
175
|
-
}
|
|
176
|
-
// Service instance thread metrics for a particular service instance
|
|
177
|
-
async GetInfluxDBResultsServiceInstance(subscriptionKey) {
|
|
178
|
-
let retVal = null;
|
|
179
|
-
try {
|
|
180
|
-
//const serviceId = subscriptionKey.key as string; // Not required for the query
|
|
181
|
-
const serviceInstanceId = subscriptionKey.subkey;
|
|
182
|
-
retVal = await this.ProcessInfluxDBResults([
|
|
183
|
-
this.#GetSTSCountGenericService(SERVICE_STATS_BY_SERVICE_INSTANCE_PROCESS, `and r["serviceInstanceId"] == "${serviceInstanceId}"`),
|
|
184
|
-
this.#GetSTSQuantileGenericService(SERVICE_STATS_BY_SERVICE_INSTANCE_PROCESS, `and r["serviceInstanceId"] == "${serviceInstanceId}"`),
|
|
185
|
-
this.#GetSTSHistoGenericService(SERVICE_STATS_BY_SERVICE_INSTANCE_PROCESS, `and r["serviceInstanceId"] == "${serviceInstanceId}"`)
|
|
186
|
-
], ['serviceId', 'serviceInstanceId', 'serviceInstanceProcessId']);
|
|
187
|
-
}
|
|
188
|
-
catch (error) {
|
|
189
|
-
this.#LogErrorMessage(chalk_1.default.red(`${_logPrefix}GetInfluxDBResultsServiceInstance: Error: [${error}]`));
|
|
190
|
-
}
|
|
191
|
-
return {
|
|
192
|
-
subscriptionKey,
|
|
193
|
-
data: retVal
|
|
194
|
-
};
|
|
195
|
-
}
|
|
196
26
|
}
|
|
197
27
|
exports.InfluxDBManagerService = InfluxDBManagerService;
|
|
198
28
|
//# sourceMappingURL=influxDBManagerService.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"influxDBManagerService.js","sourceRoot":"","sources":["../../src/influxdb/influxDBManagerService.ts"],"names":[],"mappings":";AAAA,wFAAwF,CAAE,UAAU;AACpG,4FAA4F;AAC5F,+GAA+G
|
|
1
|
+
{"version":3,"file":"influxDBManagerService.js","sourceRoot":"","sources":["../../src/influxdb/influxDBManagerService.ts"],"names":[],"mappings":";AAAA,wFAAwF,CAAE,UAAU;AACpG,4FAA4F;AAC5F,+GAA+G;;;AAkH/G,+DAA2D;AAQ3D,MAAM,UAAU,GAAG,yBAAyB,CAAA;AAE5C;;;;;EAKE;AAEF,MAAM,yCAAyC,GAAG,qCAAqC,CAAC;AACxF,MAAM,iCAAiC,GAAG,8BAA8B,CAAC;AACzE,MAAM,wBAAwB,GAAG,sBAAsB,CAAC;AACxD,MAAM,oBAAoB,GAAG,yBAAyB,CAAC;AAKvD,MAAa,sBAAuB,SAAQ,yCAAmB;IAE3D,YAAY,OAAgC,EAAE,QAAkB;QAC5D,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAED,IAAa,SAAS;QAClB,OAAO,SAAS,CAAA;IACpB,CAAC;CA+JJ;AAvKD,wDAuKC"}
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -21,7 +21,6 @@ export * from './influxdb/influxDBManagerAgent'
|
|
|
21
21
|
export * from './influxdb/influxDBManagerLambda'
|
|
22
22
|
export * from './publishertransports/publishTransportUtils'
|
|
23
23
|
export * from './publishertransports/publishTransportDirect'
|
|
24
|
-
export * from './instrumentationsubscriber'
|
|
25
24
|
export * from './stsTransportWinston'
|
|
26
25
|
export * from './stsTransportLoggerWinston'
|
|
27
26
|
export * from './globalServiceDefinitions'
|
|
@@ -130,6 +130,8 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
|
|
|
130
130
|
return 'agent'
|
|
131
131
|
}
|
|
132
132
|
|
|
133
|
+
/*
|
|
134
|
+
|
|
133
135
|
#LogErrorMessage(message: any) {
|
|
134
136
|
this.options.logger.error(message);
|
|
135
137
|
}
|
|
@@ -278,4 +280,5 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
|
|
|
278
280
|
data: retVal
|
|
279
281
|
};
|
|
280
282
|
}
|
|
283
|
+
*/
|
|
281
284
|
}
|
|
@@ -145,6 +145,11 @@ export class InfluxDBManagerService extends InfluxDBManagerBase
|
|
|
145
145
|
super(options, queryApi);
|
|
146
146
|
}
|
|
147
147
|
|
|
148
|
+
override get modelType(): string {
|
|
149
|
+
return 'service'
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/*
|
|
148
153
|
#LogErrorMessage(message: any) {
|
|
149
154
|
this.options.logger.error(message);
|
|
150
155
|
}
|
|
@@ -153,28 +158,6 @@ export class InfluxDBManagerService extends InfluxDBManagerBase
|
|
|
153
158
|
this.options.logger.debug(message);
|
|
154
159
|
}
|
|
155
160
|
|
|
156
|
-
override get modelType(): string {
|
|
157
|
-
return 'service'
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
// Queries --------------------------------------------------------------------------------------------------------
|
|
161
|
-
|
|
162
|
-
/*
|
|
163
|
-
data = from(bucket: "my-bucket")
|
|
164
|
-
|> range(start: -task.every)
|
|
165
|
-
|> filter(fn: (r) => r._measurement == "my_measurement")
|
|
166
|
-
|
|
167
|
-
data
|
|
168
|
-
|> mean()
|
|
169
|
-
|> set(key: "agg_type",value: "mean_temp")
|
|
170
|
-
|> to(bucket: "downsampled", org: "my-org", tagColumns: ["agg_type"])
|
|
171
|
-
|
|
172
|
-
data
|
|
173
|
-
|> count()
|
|
174
|
-
|> set(key: "agg_type",value: "count_temp")
|
|
175
|
-
|> to(bucket: "downsampled", org: "my-org", tagColumns: ["agg_type"])
|
|
176
|
-
*/
|
|
177
|
-
|
|
178
161
|
#GetSTSCountGenericService = async (measurement: string, filterClause: string, showOutput: boolean = false): Promise<any> => {
|
|
179
162
|
try {
|
|
180
163
|
const query = `r1 = from(bucket: "${this.options.bucket}")
|
|
@@ -322,4 +305,5 @@ data
|
|
|
322
305
|
data: retVal
|
|
323
306
|
};
|
|
324
307
|
}
|
|
308
|
+
*/
|
|
325
309
|
}
|
package/types/index.d.ts
CHANGED
|
@@ -21,7 +21,6 @@ export * from './influxdb/influxDBManagerAgent';
|
|
|
21
21
|
export * from './influxdb/influxDBManagerLambda';
|
|
22
22
|
export * from './publishertransports/publishTransportUtils';
|
|
23
23
|
export * from './publishertransports/publishTransportDirect';
|
|
24
|
-
export * from './instrumentationsubscriber';
|
|
25
24
|
export * from './stsTransportWinston';
|
|
26
25
|
export * from './stsTransportLoggerWinston';
|
|
27
26
|
export * from './globalServiceDefinitions';
|
package/types/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAA;AAC7B,cAAc,kBAAkB,CAAA;AAChC,cAAc,YAAY,CAAA;AAC1B,cAAc,iBAAiB,CAAA;AAC/B,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,wBAAwB,CAAA;AACtC,cAAc,wBAAwB,CAAA;AACtC,cAAc,mBAAmB,CAAA;AACjC,cAAc,sBAAsB,CAAA;AACpC,cAAc,uBAAuB,CAAA;AACrC,cAAc,uBAAuB,CAAA;AACrC,cAAc,wBAAwB,CAAA;AACtC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,gCAAgC,CAAA;AAC9C,cAAc,mCAAmC,CAAA;AACjD,cAAc,iCAAiC,CAAA;AAC/C,cAAc,kCAAkC,CAAA;AAChD,cAAc,6CAA6C,CAAA;AAC3D,cAAc,8CAA8C,CAAA;AAC5D,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAA;AAC7B,cAAc,kBAAkB,CAAA;AAChC,cAAc,YAAY,CAAA;AAC1B,cAAc,iBAAiB,CAAA;AAC/B,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,wBAAwB,CAAA;AACtC,cAAc,wBAAwB,CAAA;AACtC,cAAc,mBAAmB,CAAA;AACjC,cAAc,sBAAsB,CAAA;AACpC,cAAc,uBAAuB,CAAA;AACrC,cAAc,uBAAuB,CAAA;AACrC,cAAc,wBAAwB,CAAA;AACtC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,gCAAgC,CAAA;AAC9C,cAAc,mCAAmC,CAAA;AACjD,cAAc,iCAAiC,CAAA;AAC/C,cAAc,kCAAkC,CAAA;AAChD,cAAc,6CAA6C,CAAA;AAC3D,cAAc,8CAA8C,CAAA;AAC5D,cAAc,uBAAuB,CAAA;AACrC,cAAc,6BAA6B,CAAA;AAC3C,cAAc,4BAA4B,CAAA"}
|
|
@@ -2,14 +2,8 @@
|
|
|
2
2
|
import { QueryApi } from '@influxdata/influxdb-client';
|
|
3
3
|
import { InfluxDBManagerBase } from './influxDBManagerBase';
|
|
4
4
|
import { IInfluxDBManagerOptions } from './../commonTypes';
|
|
5
|
-
import { ISubscriptionPayload, ISubscriptionKey } from '@nsshunt/stssocketio-client';
|
|
6
5
|
export declare class InfluxDBManagerAgent extends InfluxDBManagerBase {
|
|
7
|
-
#private;
|
|
8
6
|
constructor(options: IInfluxDBManagerOptions, queryApi: QueryApi);
|
|
9
7
|
get modelType(): string;
|
|
10
|
-
GetInfluxDBResultsRootAgent(subscriptionKey: ISubscriptionKey): Promise<ISubscriptionPayload>;
|
|
11
|
-
GetInfluxDBResultsAgent(subscriptionKey: ISubscriptionKey): Promise<ISubscriptionPayload>;
|
|
12
|
-
GetInfluxDBResultsAgentThreads(subscriptionKey: ISubscriptionKey): Promise<ISubscriptionPayload>;
|
|
13
|
-
GetInfluxDBResultsAgentThread(subscriptionKey: ISubscriptionKey): Promise<ISubscriptionPayload>;
|
|
14
8
|
}
|
|
15
9
|
//# sourceMappingURL=influxDBManagerAgent.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"influxDBManagerAgent.d.ts","sourceRoot":"","sources":["../../src/influxdb/influxDBManagerAgent.ts"],"names":[],"mappings":";AAuGA,OAAO,EAAmB,QAAQ,EAAE,MAAM,6BAA6B,CAAA;AAIvE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAE3D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAA;
|
|
1
|
+
{"version":3,"file":"influxDBManagerAgent.d.ts","sourceRoot":"","sources":["../../src/influxdb/influxDBManagerAgent.ts"],"names":[],"mappings":";AAuGA,OAAO,EAAmB,QAAQ,EAAE,MAAM,6BAA6B,CAAA;AAIvE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAE3D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAA;AAa1D,qBAAa,oBAAqB,SAAQ,mBAAmB;gBAE7C,OAAO,EAAE,uBAAuB,EAAE,QAAQ,EAAE,QAAQ;IAIhE,IAAa,SAAS,IAAI,MAAM,CAE/B;CAyJJ"}
|
|
@@ -2,14 +2,8 @@
|
|
|
2
2
|
import { QueryApi } from '@influxdata/influxdb-client';
|
|
3
3
|
import { InfluxDBManagerBase } from './influxDBManagerBase';
|
|
4
4
|
import { IInfluxDBManagerOptions } from './../commonTypes';
|
|
5
|
-
import { ISubscriptionPayload, ISubscriptionKey } from '@nsshunt/stssocketio-client';
|
|
6
5
|
export declare class InfluxDBManagerService extends InfluxDBManagerBase {
|
|
7
|
-
#private;
|
|
8
6
|
constructor(options: IInfluxDBManagerOptions, queryApi: QueryApi);
|
|
9
7
|
get modelType(): string;
|
|
10
|
-
GetInfluxDBResultsRootService(subscriptionKey: ISubscriptionKey): Promise<ISubscriptionPayload>;
|
|
11
|
-
GetInfluxDBResultsService(subscriptionKey: ISubscriptionKey): Promise<ISubscriptionPayload>;
|
|
12
|
-
GetInfluxDBResultsServiceInstances(subscriptionKey: ISubscriptionKey): Promise<ISubscriptionPayload>;
|
|
13
|
-
GetInfluxDBResultsServiceInstance(subscriptionKey: ISubscriptionKey): Promise<ISubscriptionPayload>;
|
|
14
8
|
}
|
|
15
9
|
//# sourceMappingURL=influxDBManagerService.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"influxDBManagerService.d.ts","sourceRoot":"","sources":["../../src/influxdb/influxDBManagerService.ts"],"names":[],"mappings":";AAgHA,OAAO,EAAmB,QAAQ,EAAE,MAAM,6BAA6B,CAAA;AAIvE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAE3D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAA;
|
|
1
|
+
{"version":3,"file":"influxDBManagerService.d.ts","sourceRoot":"","sources":["../../src/influxdb/influxDBManagerService.ts"],"names":[],"mappings":";AAgHA,OAAO,EAAmB,QAAQ,EAAE,MAAM,6BAA6B,CAAA;AAIvE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAE3D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAA;AAuB1D,qBAAa,sBAAuB,SAAQ,mBAAmB;gBAE/C,OAAO,EAAE,uBAAuB,EAAE,QAAQ,EAAE,QAAQ;IAIhE,IAAa,SAAS,IAAI,MAAM,CAE/B;CA+JJ"}
|
|
@@ -1,304 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.InstrumentationSubscriber = void 0;
|
|
7
|
-
const stssocketio_client_1 = require("@nsshunt/stssocketio-client");
|
|
8
|
-
const kafkamanager_1 = require("./kafka/kafkamanager");
|
|
9
|
-
const influxDBManager_1 = require("./influxdb/influxDBManager");
|
|
10
|
-
const uuid_1 = require("uuid");
|
|
11
|
-
const stsconfig_1 = require("@nsshunt/stsconfig");
|
|
12
|
-
const stsutils_1 = require("@nsshunt/stsutils");
|
|
13
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
14
|
-
const goptions = (0, stsconfig_1.$Options)();
|
|
15
|
-
/**
|
|
16
|
-
* Maintain list of instrumentation subscriptions.
|
|
17
|
-
*/
|
|
18
|
-
class InstrumentationSubscriber extends stsutils_1.STSOptionsBase {
|
|
19
|
-
#subscriptions = {};
|
|
20
|
-
#influxDBManager;
|
|
21
|
-
#timeout = null;
|
|
22
|
-
#km;
|
|
23
|
-
#stop = true;
|
|
24
|
-
constructor(options) {
|
|
25
|
-
super(options);
|
|
26
|
-
this.#influxDBManager = new influxDBManager_1.InfluxDBManager(this.options?.logger);
|
|
27
|
-
const kmc = {
|
|
28
|
-
clientId: `${goptions.kafka_clientId}_${(0, uuid_1.v4)()}`,
|
|
29
|
-
brokers: goptions.kafka_brokers.split(','),
|
|
30
|
-
keepAlive: goptions.kafka_keep_alive,
|
|
31
|
-
adminTimeout: goptions.kafka_admin_timeout,
|
|
32
|
-
connectionTimeout: goptions.kafka_connection_timeout,
|
|
33
|
-
logLevel: goptions.kafka_log_level,
|
|
34
|
-
useSSL: goptions.kafka_use_ssl,
|
|
35
|
-
requestTimeout: goptions.kafka_request_timeout,
|
|
36
|
-
logger: this.options?.logger
|
|
37
|
-
};
|
|
38
|
-
if (goptions.kafka_use_ssl) {
|
|
39
|
-
kmc.ssl = {
|
|
40
|
-
rejectUnauthorized: goptions.kafka_ssl_rejectUnauthorized,
|
|
41
|
-
cafile: goptions.kafka_ssl_cafile,
|
|
42
|
-
certfileFile: goptions.kafka_ssl_certfile,
|
|
43
|
-
keyfile: goptions.kafka_ssl_keyfile
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
this.#km = new kafkamanager_1.KafkaManager(kmc);
|
|
47
|
-
}
|
|
48
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
49
|
-
#LogDebugMessage(message) {
|
|
50
|
-
this.options?.logger.debug(message);
|
|
51
|
-
}
|
|
52
|
-
async #UnSubscribeKafka(subscription) {
|
|
53
|
-
const unsubscribeKafka = async () => {
|
|
54
|
-
const kafka = subscription.kafka;
|
|
55
|
-
if (kafka) {
|
|
56
|
-
// Un-Subscribe from this kafka topic
|
|
57
|
-
this.#LogDebugMessage(chalk_1.default.yellow(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Starting`));
|
|
58
|
-
await kafka.kafkaConsumer?.Stop((error) => {
|
|
59
|
-
this.#LogDebugMessage(chalk_1.default.red(`InstrumentationSubscriber:#UnSubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.Stop Error: [${error}]`));
|
|
60
|
-
});
|
|
61
|
-
await kafka.kafkaConsumer?.Disconnect((error) => {
|
|
62
|
-
this.#LogDebugMessage(chalk_1.default.red(`InstrumentationSubscriber:#UnSubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.Disconnect Error: [${error}]`));
|
|
63
|
-
});
|
|
64
|
-
this.#LogDebugMessage(chalk_1.default.yellow(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Completed.`));
|
|
65
|
-
}
|
|
66
|
-
else {
|
|
67
|
-
this.#LogDebugMessage(chalk_1.default.magenta(`InstrumentationSubscriber:#UnSubscribeKafka(): Kafka details do not exist for this topc: [${JSON.stringify(subscription.subscriptionKey)}].`));
|
|
68
|
-
}
|
|
69
|
-
};
|
|
70
|
-
return unsubscribeKafka();
|
|
71
|
-
}
|
|
72
|
-
async #SubscribeKafka(subscription) {
|
|
73
|
-
const kafkaSubscribe = async () => {
|
|
74
|
-
if (subscription.subscriptionKey.id) {
|
|
75
|
-
const kafkaSafeKey = `${subscription.subscriptionKey.key}`.replace(/@/g, '_').replace(/\|/g, '_');
|
|
76
|
-
const kafkaGroupId = (0, uuid_1.v4)();
|
|
77
|
-
subscription.kafka = {
|
|
78
|
-
kafkaTopic: kafkaSafeKey,
|
|
79
|
-
kafkaGroupId: kafkaGroupId,
|
|
80
|
-
kafkaConsumer: this.#km.CreateConsumer(kafkaGroupId),
|
|
81
|
-
kafkaFromBeginning: goptions.kafka_consume_from_beginning
|
|
82
|
-
};
|
|
83
|
-
this.#LogDebugMessage(chalk_1.default.yellow(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${subscription.kafka.kafkaTopic}] Starting`));
|
|
84
|
-
const kafka = subscription.kafka;
|
|
85
|
-
let connectError = false;
|
|
86
|
-
await kafka.kafkaConsumer.Connect((error) => {
|
|
87
|
-
this.#LogDebugMessage(chalk_1.default.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.Connect: Error: [${error}]`));
|
|
88
|
-
connectError = true;
|
|
89
|
-
});
|
|
90
|
-
if (!connectError) {
|
|
91
|
-
let subscribeError = false;
|
|
92
|
-
await kafka.kafkaConsumer.Subscribe([kafka.kafkaTopic], subscription.kafka.kafkaFromBeginning, (error) => {
|
|
93
|
-
this.#LogDebugMessage(chalk_1.default.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.Subscribe: Error: [${error}]`));
|
|
94
|
-
subscribeError = true;
|
|
95
|
-
});
|
|
96
|
-
if (!subscribeError) {
|
|
97
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
98
|
-
await kafka.kafkaConsumer.StartConsumingMessages((topic, key, partition, value, headers) => {
|
|
99
|
-
try {
|
|
100
|
-
if (key != "" && value != "") {
|
|
101
|
-
if (topic.localeCompare(key) === 0) {
|
|
102
|
-
subscription.cb({
|
|
103
|
-
subscriptionKey: subscription.subscriptionKey, // Original requested topic (may be kafka un-safe)
|
|
104
|
-
data: {
|
|
105
|
-
kafkaTopic: kafka.kafkaTopic, // Safe kafka topic key with prefix. kafkaTopic, topic and key should be same value
|
|
106
|
-
kafkaGroupId: kafka.kafkaGroupId,
|
|
107
|
-
partition,
|
|
108
|
-
topic, // kafkaTopic, topic and key should be same value
|
|
109
|
-
key, // kafkaTopic, topic and key should be same value
|
|
110
|
-
value
|
|
111
|
-
}
|
|
112
|
-
});
|
|
113
|
-
}
|
|
114
|
-
else {
|
|
115
|
-
this.#LogDebugMessage(chalk_1.default.red(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}] and key: [${key}] do not match`));
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
else {
|
|
119
|
-
this.#LogDebugMessage(chalk_1.default.red(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}], key: [${key}], value: [${value}]`));
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
catch (error) {
|
|
123
|
-
this.#LogDebugMessage(chalk_1.default.red(`InstrumentationSubscriber:#SubscribeKafka(): Error: [${error}]`));
|
|
124
|
-
}
|
|
125
|
-
}, (error) => {
|
|
126
|
-
this.#LogDebugMessage(chalk_1.default.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.StartConsumingMessages: Error: [${error}]`));
|
|
127
|
-
});
|
|
128
|
-
this.#LogDebugMessage(chalk_1.default.yellow(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${kafka.kafkaTopic}] Completed`));
|
|
129
|
-
}
|
|
130
|
-
else {
|
|
131
|
-
this.#LogDebugMessage(chalk_1.default.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], Could not subscribe to topic, StartConsumingMessages aborted.`));
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
else {
|
|
135
|
-
this.#LogDebugMessage(chalk_1.default.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], Could not connect to topic, subscribe aborted.`));
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
else {
|
|
139
|
-
this.#LogDebugMessage(chalk_1.default.magenta(`InstrumentationSubscriber:#SubscribeKafka(): Could not subscribe to Kafka topc, missing combinedKey for topic: [${JSON.stringify(subscription.subscriptionKey)}]`));
|
|
140
|
-
}
|
|
141
|
-
};
|
|
142
|
-
return kafkaSubscribe();
|
|
143
|
-
}
|
|
144
|
-
async #OutputSubscription(subscription) {
|
|
145
|
-
// Service subscriptions
|
|
146
|
-
const { subscriptionKey } = subscription;
|
|
147
|
-
switch (subscriptionKey.topic) {
|
|
148
|
-
// Services -------------------------------------------------------------------------------------------
|
|
149
|
-
case stssocketio_client_1.SubscriptionTopic.AllServicesCombined:
|
|
150
|
-
return this.#influxDBManager.serviceManager.GetInfluxDBResultsRootService(subscriptionKey);
|
|
151
|
-
case stssocketio_client_1.SubscriptionTopic.Services:
|
|
152
|
-
return this.#influxDBManager.serviceManager.GetInfluxDBResultsService(subscriptionKey);
|
|
153
|
-
case stssocketio_client_1.SubscriptionTopic.ServiceInstances:
|
|
154
|
-
if (subscriptionKey.key) { // key = service_id
|
|
155
|
-
return this.#influxDBManager.serviceManager.GetInfluxDBResultsServiceInstances(subscriptionKey);
|
|
156
|
-
}
|
|
157
|
-
else {
|
|
158
|
-
throw new Error(`#OutputSubscription: key not provided for subscription: [${JSON.stringify(subscriptionKey)}]`);
|
|
159
|
-
}
|
|
160
|
-
case stssocketio_client_1.SubscriptionTopic.ServiceInstance:
|
|
161
|
-
if (subscriptionKey.key && subscriptionKey.subkey) { // key = service instance id
|
|
162
|
-
return this.#influxDBManager.serviceManager.GetInfluxDBResultsServiceInstance(subscriptionKey);
|
|
163
|
-
}
|
|
164
|
-
else {
|
|
165
|
-
throw new Error(`#OutputSubscription: key not provided for subscription: [${JSON.stringify(subscriptionKey)}]`);
|
|
166
|
-
}
|
|
167
|
-
// User Agents -------------------------------------------------------------------------------------------
|
|
168
|
-
case stssocketio_client_1.SubscriptionTopic.AllAgentsCombined:
|
|
169
|
-
return this.#influxDBManager.agentManager.GetInfluxDBResultsRootAgent(subscriptionKey);
|
|
170
|
-
case stssocketio_client_1.SubscriptionTopic.Agents:
|
|
171
|
-
return this.#influxDBManager.agentManager.GetInfluxDBResultsAgent(subscriptionKey);
|
|
172
|
-
case stssocketio_client_1.SubscriptionTopic.AgentWorkers:
|
|
173
|
-
if (subscriptionKey.key) { // key = agent instance id
|
|
174
|
-
return this.#influxDBManager.agentManager.GetInfluxDBResultsAgentThreads(subscriptionKey);
|
|
175
|
-
}
|
|
176
|
-
else {
|
|
177
|
-
throw new Error(`#OutputSubscription: key not provided for subscription: [${JSON.stringify(subscriptionKey)}]`);
|
|
178
|
-
}
|
|
179
|
-
case stssocketio_client_1.SubscriptionTopic.AgentWorker:
|
|
180
|
-
if (subscriptionKey.key && subscriptionKey.subkey) { // key = agent instance id, subkey = agent instance worker id
|
|
181
|
-
return this.#influxDBManager.agentManager.GetInfluxDBResultsAgentThread(subscriptionKey);
|
|
182
|
-
}
|
|
183
|
-
else {
|
|
184
|
-
throw new Error(`#OutputSubscription: key and/or subkey not provided for subscription: [${JSON.stringify(subscriptionKey.topic)}]`);
|
|
185
|
-
}
|
|
186
|
-
// Logging (both service and agent) -----------------------------------------------------------------------
|
|
187
|
-
case stssocketio_client_1.SubscriptionTopic.LogProcessing:
|
|
188
|
-
// Kafka processing will be handled by a dedicated kafka run consumer
|
|
189
|
-
return { subscriptionKey, data: null };
|
|
190
|
-
default:
|
|
191
|
-
return { subscriptionKey, data: null };
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
#ExecuteSubscriptions = async () => {
|
|
195
|
-
const promArray = [];
|
|
196
|
-
const indexArray = [];
|
|
197
|
-
for (const [, subscription] of Object.entries(this.#subscriptions)) {
|
|
198
|
-
// If NOT Kafka, go get the data from influxdb
|
|
199
|
-
if (subscription.subscriptionKey.topic.localeCompare(stssocketio_client_1.SubscriptionTopic.LogProcessing.toString()) !== 0) {
|
|
200
|
-
promArray.push(this.#OutputSubscription(subscription));
|
|
201
|
-
indexArray.push(subscription);
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
const retVal = await Promise.all(promArray);
|
|
205
|
-
try {
|
|
206
|
-
for (let i = 0; i < retVal.length; i++) {
|
|
207
|
-
// Provided we have data, invoke the callback
|
|
208
|
-
if (retVal[i].data) {
|
|
209
|
-
indexArray[i].cb(retVal[i]);
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
catch (error) {
|
|
214
|
-
this.#LogDebugMessage(chalk_1.default.red(`InstrumentationSubscriber:#ExecuteSubscriptions(): Error: [${error}], [${JSON.stringify(retVal)}]`));
|
|
215
|
-
}
|
|
216
|
-
};
|
|
217
|
-
#SetupTimeout = (timeout) => {
|
|
218
|
-
if (!this.#timeout && (Object.keys(this.#subscriptions).length > 0)) {
|
|
219
|
-
this.#timeout = setTimeout(async () => {
|
|
220
|
-
const start = performance.now();
|
|
221
|
-
await this.#ExecuteSubscriptions();
|
|
222
|
-
this.#timeout = null;
|
|
223
|
-
if (!this.#stop) {
|
|
224
|
-
let requiredTimeout = 1000; //@@ default
|
|
225
|
-
if (this.options) {
|
|
226
|
-
requiredTimeout = this.options.SubscriptionInterval;
|
|
227
|
-
}
|
|
228
|
-
const end = performance.now();
|
|
229
|
-
const diff = end - start;
|
|
230
|
-
let newTimeout = requiredTimeout - diff;
|
|
231
|
-
if (newTimeout < 500) {
|
|
232
|
-
newTimeout = 500;
|
|
233
|
-
}
|
|
234
|
-
this.#LogDebugMessage(chalk_1.default.grey(`InstrumentationSubscriber.#SetupTimeout(): ExecuteSubscriptions time: [${diff}], next timeout: [${newTimeout}]`));
|
|
235
|
-
this.#SetupTimeout(newTimeout);
|
|
236
|
-
}
|
|
237
|
-
}, timeout);
|
|
238
|
-
}
|
|
239
|
-
};
|
|
240
|
-
Subscribe(subscriptionKey, cb) {
|
|
241
|
-
if (this.#timeout) {
|
|
242
|
-
clearTimeout(this.#timeout);
|
|
243
|
-
this.#timeout = null;
|
|
244
|
-
}
|
|
245
|
-
if (this.#subscriptions[subscriptionKey.id]) {
|
|
246
|
-
// Subscription already exists - ignoring
|
|
247
|
-
this.#LogDebugMessage(chalk_1.default.magenta(`InstrumentationSubscriber.Subscribe(): Subscription already exists: [${subscriptionKey.id}] - ignoring.`));
|
|
248
|
-
}
|
|
249
|
-
else {
|
|
250
|
-
this.#subscriptions[subscriptionKey.id] = {
|
|
251
|
-
subscriptionKey,
|
|
252
|
-
cb
|
|
253
|
-
};
|
|
254
|
-
if (subscriptionKey.topic.localeCompare(stssocketio_client_1.SubscriptionTopic.LogProcessing) === 0) {
|
|
255
|
-
this.#SubscribeKafka(this.#subscriptions[subscriptionKey.id]);
|
|
256
|
-
}
|
|
257
|
-
this.#LogDebugMessage(chalk_1.default.grey(`InstrumentationSubscriber.Subscribe(): subscriptionKey.id: [${subscriptionKey.id}] successfully subscribed.`));
|
|
258
|
-
if (!this.#timeout) {
|
|
259
|
-
(async () => {
|
|
260
|
-
// Output subscription immediately and then iterate ...
|
|
261
|
-
await this.#ExecuteSubscriptions();
|
|
262
|
-
let timeout = 1000; //@@ default
|
|
263
|
-
if (this.options) {
|
|
264
|
-
timeout = this.options.SubscriptionInterval;
|
|
265
|
-
}
|
|
266
|
-
this.#SetupTimeout(timeout);
|
|
267
|
-
})();
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
UnSubscribe(subscriptionKey) {
|
|
272
|
-
if (!this.#subscriptions[subscriptionKey.id]) {
|
|
273
|
-
this.#LogDebugMessage(chalk_1.default.magenta(`UnSubscribe(): Warning, subscriptionKey.id: [${subscriptionKey.id}] does not exist within subscription records`));
|
|
274
|
-
}
|
|
275
|
-
else {
|
|
276
|
-
if (subscriptionKey.topic.localeCompare(stssocketio_client_1.SubscriptionTopic.LogProcessing) === 0) {
|
|
277
|
-
this.#UnSubscribeKafka(this.#subscriptions[subscriptionKey.id]);
|
|
278
|
-
}
|
|
279
|
-
delete this.#subscriptions[subscriptionKey.id];
|
|
280
|
-
if (Object.keys(this.#subscriptions).length === 0) {
|
|
281
|
-
if (this.#timeout) {
|
|
282
|
-
clearTimeout(this.#timeout);
|
|
283
|
-
this.#timeout = null;
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
#UnSubscribeAll() {
|
|
289
|
-
for (const [, subscription] of Object.entries(this.#subscriptions)) {
|
|
290
|
-
this.UnSubscribe(subscription.subscriptionKey);
|
|
291
|
-
}
|
|
292
|
-
}
|
|
293
|
-
async Start() {
|
|
294
|
-
this.#stop = false;
|
|
295
|
-
return this.#influxDBManager.Start();
|
|
296
|
-
}
|
|
297
|
-
async Stop() {
|
|
298
|
-
this.#stop = true;
|
|
299
|
-
this.#UnSubscribeAll();
|
|
300
|
-
return this.#influxDBManager.Terminate();
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
|
-
exports.InstrumentationSubscriber = InstrumentationSubscriber;
|
|
304
|
-
//# sourceMappingURL=instrumentationsubscriber.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"instrumentationsubscriber.js","sourceRoot":"","sources":["../src/instrumentationsubscriber.ts"],"names":[],"mappings":";;;;;;AACA,oEAAmH;AAEnH,uDAAwE;AACxE,gEAA4D;AAC5D,+BAAoC;AAEpC,kDAA6C;AAC7C,gDAA+D;AAE/D,kDAA0B;AAE1B,MAAM,QAAQ,GAAG,IAAA,oBAAQ,GAAE,CAAA;AAsB3B;;GAEG;AACH,MAAa,yBAA0B,SAAQ,yBAAc;IACzD,cAAc,GAAmB,EAAG,CAAC;IACrC,gBAAgB,CAAkB;IAClC,QAAQ,GAAwB,IAAI,CAAC;IACrC,GAAG,CAAe;IAClB,KAAK,GAAY,IAAI,CAAC;IAEtB,YAAY,OAA0C;QAClD,KAAK,CAAC,OAAO,CAAC,CAAC;QAEf,IAAI,CAAC,gBAAgB,GAAG,IAAI,iCAAe,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAElE,MAAM,GAAG,GAAwB;YAC7B,QAAQ,EAAE,GAAG,QAAQ,CAAC,cAAc,IAAI,IAAA,SAAM,GAAE,EAAE;YAClD,OAAO,EAAE,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC;YAC1C,SAAS,EAAE,QAAQ,CAAC,gBAAgB;YACpC,YAAY,EAAE,QAAQ,CAAC,mBAAmB;YAC1C,iBAAiB,EAAE,QAAQ,CAAC,wBAAwB;YACpD,QAAQ,EAAE,QAAQ,CAAC,eAAe;YAClC,MAAM,EAAE,QAAQ,CAAC,aAAa;YAC9B,cAAc,EAAE,QAAQ,CAAC,qBAAqB;YAC9C,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM;SAC/B,CAAA;QACD,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;YACzB,GAAG,CAAC,GAAG,GAAG;gBACN,kBAAkB,EAAE,QAAQ,CAAC,4BAA4B;gBACzD,MAAM,EAAE,QAAQ,CAAC,gBAAgB;gBACjC,YAAY,EAAE,QAAQ,CAAC,kBAAkB;gBACzC,OAAO,EAAE,QAAQ,CAAC,iBAAiB;aACtC,CAAA;QACL,CAAC;QACD,IAAI,CAAC,GAAG,GAAG,IAAI,2BAAY,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,+DAA+D;IAC/D,gBAAgB,CAAC,OAAY;QACzB,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,YAA2B;QAC/C,MAAM,gBAAgB,GAAG,KAAK,IAAI,EAAE;YAChC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;YACjC,IAAI,KAAK,EAAE,CAAC;gBACR,qCAAqC;gBACrC,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,MAAM,CAAC,gFAAgF,KAAK,CAAC,UAAU,YAAY,CAAC,CAAC,CAAC;gBAElJ,MAAM,KAAK,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;oBACtC,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,GAAG,CAAC,gEAAgE,KAAK,CAAC,UAAU,iCAAiC,KAAK,GAAG,CAAC,CAAC,CAAC;gBAChK,CAAC,CAAC,CAAC;gBAEH,MAAM,KAAK,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE;oBAC5C,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,GAAG,CAAC,gEAAgE,KAAK,CAAC,UAAU,uCAAuC,KAAK,GAAG,CAAC,CAAC,CAAC;gBACtK,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,MAAM,CAAC,gFAAgF,KAAK,CAAC,UAAU,cAAc,CAAC,CAAC,CAAC;YACxJ,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,OAAO,CAAC,6FAA6F,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;YACxL,CAAC;QACL,CAAC,CAAC;QACF,OAAO,gBAAgB,EAAE,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,YAA2B;QAC7C,MAAM,cAAc,GAAG,KAAK,IAAI,EAAE;YAC9B,IAAI,YAAY,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC;gBAElC,MAAM,YAAY,GAAG,GAAG,YAAY,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBAElG,MAAM,YAAY,GAAG,IAAA,SAAM,GAAE,CAAC;gBAC9B,YAAY,CAAC,KAAK,GAAG;oBACjB,UAAU,EAAE,YAAY;oBACxB,YAAY,EAAE,YAAY;oBAC1B,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,YAAY,CAAC;oBACpD,kBAAkB,EAAE,QAAQ,CAAC,4BAA4B;iBAC5D,CAAC;gBACF,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,MAAM,CAAC,0EAA0E,YAAY,CAAC,KAAK,CAAC,UAAU,YAAY,CAAC,CAAC,CAAC;gBAEzJ,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;gBAEjC,IAAI,YAAY,GAAG,KAAK,CAAC;gBACzB,MAAM,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;oBACxC,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,GAAG,CAAC,8DAA8D,KAAK,CAAC,UAAU,qCAAqC,KAAK,GAAG,CAAC,CAAC,CAAC;oBAC9J,YAAY,GAAG,IAAI,CAAC;gBACxB,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,YAAY,EAAE,CAAC;oBAChB,IAAI,cAAc,GAAG,KAAK,CAAC;oBAC3B,MAAM,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,YAAY,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,EAAE;wBACrG,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,GAAG,CAAC,8DAA8D,KAAK,CAAC,UAAU,uCAAuC,KAAK,GAAG,CAAC,CAAC,CAAC;wBAChK,cAAc,GAAG,IAAI,CAAC;oBAC1B,CAAC,CAAC,CAAC;oBAEH,IAAI,CAAC,cAAc,EAAE,CAAC;wBAClB,8DAA8D;wBAC9D,MAAM,KAAK,CAAC,aAAa,CAAC,sBAAsB,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;4BACvF,IAAI,CAAC;gCACD,IAAI,GAAG,IAAI,EAAE,IAAI,KAAK,IAAI,EAAE,EAAE,CAAC;oCAC3B,IAAI,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;wCACjC,YAAY,CAAC,EAAE,CAAC;4CACZ,eAAe,EAAE,YAAY,CAAC,eAAe,EAAC,kDAAkD;4CAChG,IAAI,EAAE;gDACF,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,mFAAmF;gDACjH,YAAY,EAAE,KAAK,CAAC,YAAY;gDAChC,SAAS;gDACT,KAAK,EAAE,iDAAiD;gDACxD,GAAG,EAAE,iDAAiD;gDACtD,KAAK;6CACM;yCAClB,CAAC,CAAC;oCACP,CAAC;yCAAM,CAAC;wCACJ,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,GAAG,CAAC,gGAAgG,KAAK,eAAe,GAAG,gBAAgB,CAAC,CAAC,CAAC;oCAC9K,CAAC;gCACL,CAAC;qCAAM,CAAC;oCACJ,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,GAAG,CAAC,gGAAgG,KAAK,YAAY,GAAG,cAAc,KAAK,GAAG,CAAC,CAAC,CAAC;gCACjL,CAAC;4BACL,CAAC;4BAAC,OAAO,KAAK,EAAE,CAAC;gCACb,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,GAAG,CAAC,wDAAwD,KAAK,GAAG,CAAC,CAAC,CAAC;4BACvG,CAAC;wBACL,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE;4BACT,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,GAAG,CAAC,8DAA8D,KAAK,CAAC,UAAU,oDAAoD,KAAK,GAAG,CAAC,CAAC,CAAC;wBACjL,CAAC,CAAC,CAAC;wBACH,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,MAAM,CAAC,0EAA0E,KAAK,CAAC,UAAU,aAAa,CAAC,CAAC,CAAC;oBACjJ,CAAC;yBAAM,CAAC;wBACJ,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,GAAG,CAAC,8DAA8D,KAAK,CAAC,UAAU,kEAAkE,CAAC,CAAC,CAAC;oBACvL,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACJ,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,GAAG,CAAC,8DAA8D,KAAK,CAAC,UAAU,mDAAmD,CAAC,CAAC,CAAC;gBACxK,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,OAAO,CAAC,mHAAmH,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7M,CAAC;QACL,CAAC,CAAA;QACD,OAAO,cAAc,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,YAA2B;QACjD,wBAAwB;QACxB,MAAM,EAAE,eAAe,EAAE,GAAI,YAAY,CAAC;QAC1C,QAAQ,eAAe,CAAC,KAAK,EAAE,CAAC;YAChC,uGAAuG;YACvG,KAAK,sCAAiB,CAAC,mBAAmB;gBACtC,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,6BAA6B,CAAC,eAAe,CAAC,CAAC;YAC/F,KAAK,sCAAiB,CAAC,QAAQ;gBAC3B,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,yBAAyB,CAAC,eAAe,CAAC,CAAC;YAC3F,KAAK,sCAAiB,CAAC,gBAAgB;gBACnC,IAAI,eAAe,CAAC,GAAG,EAAE,CAAC,CAAC,mBAAmB;oBAC1C,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,kCAAkC,CAAC,eAAe,CAAC,CAAC;gBACpG,CAAC;qBAAM,CAAC;oBACJ,MAAM,IAAI,KAAK,CAAC,4DAA4D,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;gBACpH,CAAC;YACL,KAAK,sCAAiB,CAAC,eAAe;gBAClC,IAAI,eAAe,CAAC,GAAG,IAAI,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,4BAA4B;oBAC7E,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,iCAAiC,CAAC,eAAe,CAAC,CAAC;gBACnG,CAAC;qBAAM,CAAC;oBACJ,MAAM,IAAI,KAAK,CAAC,4DAA4D,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;gBACpH,CAAC;YAED,0GAA0G;YAC9G,KAAK,sCAAiB,CAAC,iBAAiB;gBACpC,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,2BAA2B,CAAC,eAAe,CAAC,CAAC;YAC3F,KAAM,sCAAiB,CAAC,MAAM;gBAC1B,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,uBAAuB,CAAC,eAAe,CAAC,CAAC;YACvF,KAAM,sCAAiB,CAAC,YAAY;gBAChC,IAAI,eAAe,CAAC,GAAG,EAAE,CAAC,CAAC,0BAA0B;oBACjD,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,8BAA8B,CAAC,eAAe,CAAC,CAAC;gBAC9F,CAAC;qBAAM,CAAC;oBACJ,MAAM,IAAI,KAAK,CAAC,4DAA4D,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;gBACpH,CAAC;YACL,KAAK,sCAAiB,CAAC,WAAW;gBAC9B,IAAI,eAAe,CAAC,GAAG,IAAI,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,6DAA6D;oBAC9G,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,6BAA6B,CAAC,eAAe,CAAC,CAAC;gBAC7F,CAAC;qBAAM,CAAC;oBACJ,MAAM,IAAI,KAAK,CAAC,0EAA0E,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACxI,CAAC;YAED,2GAA2G;YAC/G,KAAK,sCAAiB,CAAC,aAAa;gBAChC,qEAAqE;gBACrE,OAAO,EAAE,eAAe,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC3C;gBACI,OAAO,EAAE,eAAe,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC3C,CAAC;IACL,CAAC;IAED,qBAAqB,GAAG,KAAK,IAAI,EAAE;QAC/B,MAAM,SAAS,GAAoC,EAAG,CAAC;QACvD,MAAM,UAAU,GAAoB,EAAG,CAAC;QAExC,KAAK,MAAM,CAAC,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YACjE,8CAA8C;YAC9C,IAAI,YAAY,CAAC,eAAe,CAAC,KAAK,CAAC,aAAa,CAAC,sCAAiB,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;gBACrG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC;gBACvD,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAClC,CAAC;QACL,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAI,CAAC;YACD,KAAK,IAAI,CAAC,GAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,6CAA6C;gBAC7C,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;oBACjB,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChC,CAAC;YACL,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,GAAG,CAAC,8DAA8D,KAAK,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1I,CAAC;IACL,CAAC,CAAA;IAED,aAAa,GAAG,CAAC,OAAe,EAAE,EAAE;QAChC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;YAClE,IAAI,CAAC,QAAQ,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE;gBAClC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;gBAChC,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;gBACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;gBACrB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;oBACd,IAAI,eAAe,GAAG,IAAI,CAAC,CAAC,YAAY;oBACxC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;wBACf,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAA;oBACvD,CAAC;oBACD,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;oBAC9B,MAAM,IAAI,GAAG,GAAG,GAAG,KAAK,CAAC;oBACzB,IAAI,UAAU,GAAG,eAAe,GAAG,IAAI,CAAC;oBACxC,IAAI,UAAU,GAAG,GAAG,EAAE,CAAC;wBACnB,UAAU,GAAG,GAAG,CAAC;oBACrB,CAAC;oBACD,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,IAAI,CAAC,0EAA0E,IAAI,qBAAqB,UAAU,GAAG,CAAC,CAAC,CAAC;oBACpJ,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;gBACnC,CAAC;YACL,CAAC,EAAE,OAAO,CAAC,CAAC;QAChB,CAAC;IACL,CAAC,CAAA;IAED,SAAS,CAAC,eAAiC,EAAE,EAA2C;QACpF,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACzB,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,EAAE,CAAC,EAAE,CAAC;YAC1C,yCAAyC;YACzC,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,OAAO,CAAC,wEAAwE,eAAe,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC;QACpJ,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,EAAE,CAAC,GAAG;gBACtC,eAAe;gBACf,EAAE;aACL,CAAA;YACD,IAAI,eAAe,CAAC,KAAK,CAAC,aAAa,CAAC,sCAAiB,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC7E,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC;YAClE,CAAC;YACD,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,IAAI,CAAC,+DAA+D,eAAe,CAAC,EAAE,4BAA4B,CAAC,CAAC,CAAC;YAEjJ,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACjB,CAAC,KAAK,IAAI,EAAE;oBACR,uDAAuD;oBACvD,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;oBACnC,IAAI,OAAO,GAAG,IAAI,CAAC,CAAC,YAAY;oBAChC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;wBACf,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAA;oBAC/C,CAAC;oBACD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;gBAChC,CAAC,CAAC,EAAE,CAAC;YACT,CAAC;QACL,CAAC;IACL,CAAC;IAED,WAAW,CAAC,eAAiC;QACzC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3C,IAAI,CAAC,gBAAgB,CAAC,eAAK,CAAC,OAAO,CAAC,gDAAgD,eAAe,CAAC,EAAE,8CAA8C,CAAC,CAAC,CAAC;QAC3J,CAAC;aAAM,CAAC;YACJ,IAAI,eAAe,CAAC,KAAK,CAAC,aAAa,CAAC,sCAAiB,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC7E,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC;YACpE,CAAC;YACD,OAAO,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;YAE/C,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAChD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAChB,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;gBACzB,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,eAAe;QACX,KAAK,MAAM,CAAC,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YACjE,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,IAAI;QACN,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,eAAe,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC;IAC7C,CAAC;CACJ;AA1SD,8DA0SC"}
|
|
@@ -1,336 +0,0 @@
|
|
|
1
|
-
import { IKafkaConsumer } from './commonTypes'
|
|
2
|
-
import { ISubscriptionPayload, ISubscriptionKey, SubscriptionTopic, IKafkaData } from '@nsshunt/stssocketio-client'
|
|
3
|
-
|
|
4
|
-
import { KafkaManager, IKafkaManagerConfig } from './kafka/kafkamanager'
|
|
5
|
-
import { InfluxDBManager } from './influxdb/influxDBManager'
|
|
6
|
-
import { v4 as uuidv4 } from 'uuid';
|
|
7
|
-
|
|
8
|
-
import { $Options } from '@nsshunt/stsconfig'
|
|
9
|
-
import { STSOptionsBase, ISTSLogger } from '@nsshunt/stsutils';
|
|
10
|
-
|
|
11
|
-
import chalk from 'chalk';
|
|
12
|
-
|
|
13
|
-
const goptions = $Options()
|
|
14
|
-
|
|
15
|
-
declare interface IKafkaSubscription {
|
|
16
|
-
kafkaTopic: string
|
|
17
|
-
kafkaGroupId: string
|
|
18
|
-
kafkaConsumer: IKafkaConsumer
|
|
19
|
-
kafkaFromBeginning: boolean
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
declare interface ISubscription {
|
|
23
|
-
subscriptionKey: ISubscriptionKey
|
|
24
|
-
kafka?: IKafkaSubscription
|
|
25
|
-
cb: (payload: ISubscriptionPayload) => void
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
declare type ISubscriptions = Record<string, ISubscription>;
|
|
29
|
-
|
|
30
|
-
export interface IInstrumentationSubscriberOptions {
|
|
31
|
-
SubscriptionInterval: number
|
|
32
|
-
logger: ISTSLogger
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
/**
|
|
36
|
-
* Maintain list of instrumentation subscriptions.
|
|
37
|
-
*/
|
|
38
|
-
export class InstrumentationSubscriber extends STSOptionsBase {
|
|
39
|
-
#subscriptions: ISubscriptions = { };
|
|
40
|
-
#influxDBManager: InfluxDBManager;
|
|
41
|
-
#timeout: NodeJS.Timer | null = null;
|
|
42
|
-
#km: KafkaManager;
|
|
43
|
-
#stop: boolean = true;
|
|
44
|
-
|
|
45
|
-
constructor(options: IInstrumentationSubscriberOptions) {
|
|
46
|
-
super(options);
|
|
47
|
-
|
|
48
|
-
this.#influxDBManager = new InfluxDBManager(this.options?.logger);
|
|
49
|
-
|
|
50
|
-
const kmc: IKafkaManagerConfig = {
|
|
51
|
-
clientId: `${goptions.kafka_clientId}_${uuidv4()}`,
|
|
52
|
-
brokers: goptions.kafka_brokers.split(','),
|
|
53
|
-
keepAlive: goptions.kafka_keep_alive,
|
|
54
|
-
adminTimeout: goptions.kafka_admin_timeout,
|
|
55
|
-
connectionTimeout: goptions.kafka_connection_timeout,
|
|
56
|
-
logLevel: goptions.kafka_log_level,
|
|
57
|
-
useSSL: goptions.kafka_use_ssl,
|
|
58
|
-
requestTimeout: goptions.kafka_request_timeout,
|
|
59
|
-
logger: this.options?.logger
|
|
60
|
-
}
|
|
61
|
-
if (goptions.kafka_use_ssl) {
|
|
62
|
-
kmc.ssl = {
|
|
63
|
-
rejectUnauthorized: goptions.kafka_ssl_rejectUnauthorized,
|
|
64
|
-
cafile: goptions.kafka_ssl_cafile,
|
|
65
|
-
certfileFile: goptions.kafka_ssl_certfile,
|
|
66
|
-
keyfile: goptions.kafka_ssl_keyfile
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
this.#km = new KafkaManager(kmc);
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
73
|
-
#LogDebugMessage(message: any) {
|
|
74
|
-
this.options?.logger.debug(message);
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
async #UnSubscribeKafka(subscription: ISubscription): Promise<void> {
|
|
78
|
-
const unsubscribeKafka = async () => {
|
|
79
|
-
const kafka = subscription.kafka;
|
|
80
|
-
if (kafka) {
|
|
81
|
-
// Un-Subscribe from this kafka topic
|
|
82
|
-
this.#LogDebugMessage(chalk.yellow(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Starting`));
|
|
83
|
-
|
|
84
|
-
await kafka.kafkaConsumer?.Stop((error) => {
|
|
85
|
-
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#UnSubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.Stop Error: [${error}]`));
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
await kafka.kafkaConsumer?.Disconnect((error) => {
|
|
89
|
-
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#UnSubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.Disconnect Error: [${error}]`));
|
|
90
|
-
});
|
|
91
|
-
|
|
92
|
-
this.#LogDebugMessage(chalk.yellow(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Completed.`));
|
|
93
|
-
} else {
|
|
94
|
-
this.#LogDebugMessage(chalk.magenta(`InstrumentationSubscriber:#UnSubscribeKafka(): Kafka details do not exist for this topc: [${JSON.stringify(subscription.subscriptionKey)}].`));
|
|
95
|
-
}
|
|
96
|
-
};
|
|
97
|
-
return unsubscribeKafka();
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
async #SubscribeKafka(subscription: ISubscription): Promise<void> {
|
|
101
|
-
const kafkaSubscribe = async () => {
|
|
102
|
-
if (subscription.subscriptionKey.id) {
|
|
103
|
-
|
|
104
|
-
const kafkaSafeKey = `${subscription.subscriptionKey.key}`.replace(/@/g, '_').replace(/\|/g, '_');
|
|
105
|
-
|
|
106
|
-
const kafkaGroupId = uuidv4();
|
|
107
|
-
subscription.kafka = {
|
|
108
|
-
kafkaTopic: kafkaSafeKey,
|
|
109
|
-
kafkaGroupId: kafkaGroupId,
|
|
110
|
-
kafkaConsumer: this.#km.CreateConsumer(kafkaGroupId),
|
|
111
|
-
kafkaFromBeginning: goptions.kafka_consume_from_beginning
|
|
112
|
-
};
|
|
113
|
-
this.#LogDebugMessage(chalk.yellow(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${subscription.kafka.kafkaTopic}] Starting`));
|
|
114
|
-
|
|
115
|
-
const kafka = subscription.kafka;
|
|
116
|
-
|
|
117
|
-
let connectError = false;
|
|
118
|
-
await kafka.kafkaConsumer.Connect((error) => {
|
|
119
|
-
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.Connect: Error: [${error}]`));
|
|
120
|
-
connectError = true;
|
|
121
|
-
});
|
|
122
|
-
|
|
123
|
-
if (!connectError) {
|
|
124
|
-
let subscribeError = false;
|
|
125
|
-
await kafka.kafkaConsumer.Subscribe([kafka.kafkaTopic], subscription.kafka.kafkaFromBeginning, (error) => {
|
|
126
|
-
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.Subscribe: Error: [${error}]`));
|
|
127
|
-
subscribeError = true;
|
|
128
|
-
});
|
|
129
|
-
|
|
130
|
-
if (!subscribeError) {
|
|
131
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
132
|
-
await kafka.kafkaConsumer.StartConsumingMessages((topic, key, partition, value, headers) => {
|
|
133
|
-
try {
|
|
134
|
-
if (key != "" && value != "") {
|
|
135
|
-
if (topic.localeCompare(key) === 0) {
|
|
136
|
-
subscription.cb({
|
|
137
|
-
subscriptionKey: subscription.subscriptionKey,// Original requested topic (may be kafka un-safe)
|
|
138
|
-
data: {
|
|
139
|
-
kafkaTopic: kafka.kafkaTopic, // Safe kafka topic key with prefix. kafkaTopic, topic and key should be same value
|
|
140
|
-
kafkaGroupId: kafka.kafkaGroupId,
|
|
141
|
-
partition,
|
|
142
|
-
topic, // kafkaTopic, topic and key should be same value
|
|
143
|
-
key, // kafkaTopic, topic and key should be same value
|
|
144
|
-
value
|
|
145
|
-
} as IKafkaData
|
|
146
|
-
});
|
|
147
|
-
} else {
|
|
148
|
-
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}] and key: [${key}] do not match`));
|
|
149
|
-
}
|
|
150
|
-
} else {
|
|
151
|
-
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}], key: [${key}], value: [${value}]`));
|
|
152
|
-
}
|
|
153
|
-
} catch (error) {
|
|
154
|
-
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Error: [${error}]`));
|
|
155
|
-
}
|
|
156
|
-
}, (error) => {
|
|
157
|
-
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.StartConsumingMessages: Error: [${error}]`));
|
|
158
|
-
});
|
|
159
|
-
this.#LogDebugMessage(chalk.yellow(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${kafka.kafkaTopic}] Completed`));
|
|
160
|
-
} else {
|
|
161
|
-
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], Could not subscribe to topic, StartConsumingMessages aborted.`));
|
|
162
|
-
}
|
|
163
|
-
} else {
|
|
164
|
-
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], Could not connect to topic, subscribe aborted.`));
|
|
165
|
-
}
|
|
166
|
-
} else {
|
|
167
|
-
this.#LogDebugMessage(chalk.magenta(`InstrumentationSubscriber:#SubscribeKafka(): Could not subscribe to Kafka topc, missing combinedKey for topic: [${JSON.stringify(subscription.subscriptionKey)}]`));
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
return kafkaSubscribe();
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
async #OutputSubscription(subscription: ISubscription): Promise<ISubscriptionPayload> {
|
|
174
|
-
// Service subscriptions
|
|
175
|
-
const { subscriptionKey } = subscription;
|
|
176
|
-
switch (subscriptionKey.topic) {
|
|
177
|
-
// Services -------------------------------------------------------------------------------------------
|
|
178
|
-
case SubscriptionTopic.AllServicesCombined :
|
|
179
|
-
return this.#influxDBManager.serviceManager.GetInfluxDBResultsRootService(subscriptionKey);
|
|
180
|
-
case SubscriptionTopic.Services :
|
|
181
|
-
return this.#influxDBManager.serviceManager.GetInfluxDBResultsService(subscriptionKey);
|
|
182
|
-
case SubscriptionTopic.ServiceInstances :
|
|
183
|
-
if (subscriptionKey.key) { // key = service_id
|
|
184
|
-
return this.#influxDBManager.serviceManager.GetInfluxDBResultsServiceInstances(subscriptionKey);
|
|
185
|
-
} else {
|
|
186
|
-
throw new Error(`#OutputSubscription: key not provided for subscription: [${JSON.stringify(subscriptionKey)}]`);
|
|
187
|
-
}
|
|
188
|
-
case SubscriptionTopic.ServiceInstance :
|
|
189
|
-
if (subscriptionKey.key && subscriptionKey.subkey) { // key = service instance id
|
|
190
|
-
return this.#influxDBManager.serviceManager.GetInfluxDBResultsServiceInstance(subscriptionKey);
|
|
191
|
-
} else {
|
|
192
|
-
throw new Error(`#OutputSubscription: key not provided for subscription: [${JSON.stringify(subscriptionKey)}]`);
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
// User Agents -------------------------------------------------------------------------------------------
|
|
196
|
-
case SubscriptionTopic.AllAgentsCombined :
|
|
197
|
-
return this.#influxDBManager.agentManager.GetInfluxDBResultsRootAgent(subscriptionKey);
|
|
198
|
-
case SubscriptionTopic.Agents :
|
|
199
|
-
return this.#influxDBManager.agentManager.GetInfluxDBResultsAgent(subscriptionKey);
|
|
200
|
-
case SubscriptionTopic.AgentWorkers :
|
|
201
|
-
if (subscriptionKey.key) { // key = agent instance id
|
|
202
|
-
return this.#influxDBManager.agentManager.GetInfluxDBResultsAgentThreads(subscriptionKey);
|
|
203
|
-
} else {
|
|
204
|
-
throw new Error(`#OutputSubscription: key not provided for subscription: [${JSON.stringify(subscriptionKey)}]`);
|
|
205
|
-
}
|
|
206
|
-
case SubscriptionTopic.AgentWorker :
|
|
207
|
-
if (subscriptionKey.key && subscriptionKey.subkey) { // key = agent instance id, subkey = agent instance worker id
|
|
208
|
-
return this.#influxDBManager.agentManager.GetInfluxDBResultsAgentThread(subscriptionKey);
|
|
209
|
-
} else {
|
|
210
|
-
throw new Error(`#OutputSubscription: key and/or subkey not provided for subscription: [${JSON.stringify(subscriptionKey.topic)}]`);
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
// Logging (both service and agent) -----------------------------------------------------------------------
|
|
214
|
-
case SubscriptionTopic.LogProcessing :
|
|
215
|
-
// Kafka processing will be handled by a dedicated kafka run consumer
|
|
216
|
-
return { subscriptionKey, data: null };
|
|
217
|
-
default :
|
|
218
|
-
return { subscriptionKey, data: null };
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
#ExecuteSubscriptions = async () => {
|
|
223
|
-
const promArray: Promise<ISubscriptionPayload>[] = [ ];
|
|
224
|
-
const indexArray: ISubscription[] = [ ];
|
|
225
|
-
|
|
226
|
-
for (const [, subscription] of Object.entries(this.#subscriptions)) {
|
|
227
|
-
// If NOT Kafka, go get the data from influxdb
|
|
228
|
-
if (subscription.subscriptionKey.topic.localeCompare(SubscriptionTopic.LogProcessing.toString()) !== 0) {
|
|
229
|
-
promArray.push(this.#OutputSubscription(subscription));
|
|
230
|
-
indexArray.push(subscription);
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
const retVal = await Promise.all(promArray);
|
|
234
|
-
try {
|
|
235
|
-
for (let i=0; i < retVal.length; i++) {
|
|
236
|
-
// Provided we have data, invoke the callback
|
|
237
|
-
if (retVal[i].data) {
|
|
238
|
-
indexArray[i].cb(retVal[i]);
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
} catch (error) {
|
|
242
|
-
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#ExecuteSubscriptions(): Error: [${error}], [${JSON.stringify(retVal)}]`));
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
#SetupTimeout = (timeout: number) => {
|
|
247
|
-
if (!this.#timeout && (Object.keys(this.#subscriptions).length > 0)) {
|
|
248
|
-
this.#timeout = setTimeout(async () => {
|
|
249
|
-
const start = performance.now();
|
|
250
|
-
await this.#ExecuteSubscriptions();
|
|
251
|
-
this.#timeout = null;
|
|
252
|
-
if (!this.#stop) {
|
|
253
|
-
let requiredTimeout = 1000; //@@ default
|
|
254
|
-
if (this.options) {
|
|
255
|
-
requiredTimeout = this.options.SubscriptionInterval
|
|
256
|
-
}
|
|
257
|
-
const end = performance.now();
|
|
258
|
-
const diff = end - start;
|
|
259
|
-
let newTimeout = requiredTimeout - diff;
|
|
260
|
-
if (newTimeout < 500) {
|
|
261
|
-
newTimeout = 500;
|
|
262
|
-
}
|
|
263
|
-
this.#LogDebugMessage(chalk.grey(`InstrumentationSubscriber.#SetupTimeout(): ExecuteSubscriptions time: [${diff}], next timeout: [${newTimeout}]`));
|
|
264
|
-
this.#SetupTimeout(newTimeout);
|
|
265
|
-
}
|
|
266
|
-
}, timeout);
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
Subscribe(subscriptionKey: ISubscriptionKey, cb: (payload: ISubscriptionPayload) => void) {
|
|
271
|
-
if (this.#timeout) {
|
|
272
|
-
clearTimeout(this.#timeout);
|
|
273
|
-
this.#timeout = null;
|
|
274
|
-
}
|
|
275
|
-
if (this.#subscriptions[subscriptionKey.id]) {
|
|
276
|
-
// Subscription already exists - ignoring
|
|
277
|
-
this.#LogDebugMessage(chalk.magenta(`InstrumentationSubscriber.Subscribe(): Subscription already exists: [${subscriptionKey.id}] - ignoring.`));
|
|
278
|
-
} else {
|
|
279
|
-
this.#subscriptions[subscriptionKey.id] = {
|
|
280
|
-
subscriptionKey,
|
|
281
|
-
cb
|
|
282
|
-
}
|
|
283
|
-
if (subscriptionKey.topic.localeCompare(SubscriptionTopic.LogProcessing) === 0) {
|
|
284
|
-
this.#SubscribeKafka(this.#subscriptions[subscriptionKey.id]);
|
|
285
|
-
}
|
|
286
|
-
this.#LogDebugMessage(chalk.grey(`InstrumentationSubscriber.Subscribe(): subscriptionKey.id: [${subscriptionKey.id}] successfully subscribed.`));
|
|
287
|
-
|
|
288
|
-
if (!this.#timeout) {
|
|
289
|
-
(async () => {
|
|
290
|
-
// Output subscription immediately and then iterate ...
|
|
291
|
-
await this.#ExecuteSubscriptions();
|
|
292
|
-
let timeout = 1000; //@@ default
|
|
293
|
-
if (this.options) {
|
|
294
|
-
timeout = this.options.SubscriptionInterval
|
|
295
|
-
}
|
|
296
|
-
this.#SetupTimeout(timeout);
|
|
297
|
-
})();
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
UnSubscribe(subscriptionKey: ISubscriptionKey) {
|
|
303
|
-
if (!this.#subscriptions[subscriptionKey.id]) {
|
|
304
|
-
this.#LogDebugMessage(chalk.magenta(`UnSubscribe(): Warning, subscriptionKey.id: [${subscriptionKey.id}] does not exist within subscription records`));
|
|
305
|
-
} else {
|
|
306
|
-
if (subscriptionKey.topic.localeCompare(SubscriptionTopic.LogProcessing) === 0) {
|
|
307
|
-
this.#UnSubscribeKafka(this.#subscriptions[subscriptionKey.id]);
|
|
308
|
-
}
|
|
309
|
-
delete this.#subscriptions[subscriptionKey.id];
|
|
310
|
-
|
|
311
|
-
if (Object.keys(this.#subscriptions).length === 0) {
|
|
312
|
-
if (this.#timeout) {
|
|
313
|
-
clearTimeout(this.#timeout);
|
|
314
|
-
this.#timeout = null;
|
|
315
|
-
}
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
#UnSubscribeAll() {
|
|
321
|
-
for (const [, subscription] of Object.entries(this.#subscriptions)) {
|
|
322
|
-
this.UnSubscribe(subscription.subscriptionKey);
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
async Start() {
|
|
327
|
-
this.#stop = false;
|
|
328
|
-
return this.#influxDBManager.Start();
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
async Stop() {
|
|
332
|
-
this.#stop = true;
|
|
333
|
-
this.#UnSubscribeAll();
|
|
334
|
-
return this.#influxDBManager.Terminate();
|
|
335
|
-
}
|
|
336
|
-
}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import { ISubscriptionPayload, ISubscriptionKey } from '@nsshunt/stssocketio-client';
|
|
2
|
-
import { STSOptionsBase, ISTSLogger } from '@nsshunt/stsutils';
|
|
3
|
-
export interface IInstrumentationSubscriberOptions {
|
|
4
|
-
SubscriptionInterval: number;
|
|
5
|
-
logger: ISTSLogger;
|
|
6
|
-
}
|
|
7
|
-
/**
|
|
8
|
-
* Maintain list of instrumentation subscriptions.
|
|
9
|
-
*/
|
|
10
|
-
export declare class InstrumentationSubscriber extends STSOptionsBase {
|
|
11
|
-
#private;
|
|
12
|
-
constructor(options: IInstrumentationSubscriberOptions);
|
|
13
|
-
Subscribe(subscriptionKey: ISubscriptionKey, cb: (payload: ISubscriptionPayload) => void): void;
|
|
14
|
-
UnSubscribe(subscriptionKey: ISubscriptionKey): void;
|
|
15
|
-
Start(): Promise<void>;
|
|
16
|
-
Stop(): Promise<void>;
|
|
17
|
-
}
|
|
18
|
-
//# sourceMappingURL=instrumentationsubscriber.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"instrumentationsubscriber.d.ts","sourceRoot":"","sources":["../src/instrumentationsubscriber.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAiC,MAAM,6BAA6B,CAAA;AAOnH,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAqB/D,MAAM,WAAW,iCAAiC;IAC9C,oBAAoB,EAAE,MAAM,CAAA;IAC5B,MAAM,EAAE,UAAU,CAAA;CACrB;AAED;;GAEG;AACH,qBAAa,yBAA0B,SAAQ,cAAc;;gBAO7C,OAAO,EAAE,iCAAiC;IAiOtD,SAAS,CAAC,eAAe,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAAC,OAAO,EAAE,oBAAoB,KAAK,IAAI;IAgCxF,WAAW,CAAC,eAAe,EAAE,gBAAgB;IAwBvC,KAAK;IAKL,IAAI;CAKb"}
|