@cdp-forge/plugin-pipeline-sdk 1.1.18 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ConfigListener.d.ts +3 -3
- package/dist/ConfigListener.d.ts.map +1 -1
- package/dist/ConfigListener.js +23 -20
- package/dist/ConfigListener.js.map +1 -1
- package/dist/PipelineStage.d.ts +6 -5
- package/dist/PipelineStage.d.ts.map +1 -1
- package/dist/PipelineStage.js +34 -25
- package/dist/PipelineStage.js.map +1 -1
- package/dist/config/default.d.ts.map +1 -1
- package/dist/config/default.js +4 -2
- package/dist/config/default.js.map +1 -1
- package/dist/startServer.d.ts.map +1 -1
- package/dist/startServer.js +2 -1
- package/dist/startServer.js.map +1 -1
- package/package.json +6 -4
- package/src/config/default.ts +4 -2
- package/dist/config/default.ts +0 -43
package/dist/ConfigListener.d.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import PipelineStage from "./PipelineStage";
|
|
2
2
|
import { Config } from "@cdp-forge/types";
|
|
3
3
|
export default class ConfigListener {
|
|
4
|
-
private kafka;
|
|
5
|
-
private consumer;
|
|
6
4
|
private stage;
|
|
7
5
|
private config;
|
|
8
|
-
private
|
|
6
|
+
private pulsar;
|
|
7
|
+
private consumer;
|
|
9
8
|
constructor(stage: PipelineStage, config: Config);
|
|
10
9
|
start(): Promise<void>;
|
|
10
|
+
stop(): Promise<void>;
|
|
11
11
|
}
|
|
12
12
|
//# sourceMappingURL=ConfigListener.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ConfigListener.d.ts","sourceRoot":"","sources":["../src/ConfigListener.ts"],"names":[],"mappings":"AACA,OAAO,aAAa,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAiB,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEzD,MAAM,CAAC,OAAO,OAAO,cAAc;IAC/B,OAAO,CAAC,KAAK,
|
|
1
|
+
{"version":3,"file":"ConfigListener.d.ts","sourceRoot":"","sources":["../src/ConfigListener.ts"],"names":[],"mappings":"AACA,OAAO,aAAa,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAiB,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEzD,MAAM,CAAC,OAAO,OAAO,cAAc;IAC/B,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,QAAQ,CAA8B;gBAElC,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM;IAS1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAgBtB,IAAI;CAIb"}
|
package/dist/ConfigListener.js
CHANGED
|
@@ -1,33 +1,36 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const
|
|
6
|
+
const pulsar_client_1 = __importDefault(require("pulsar-client"));
|
|
4
7
|
class ConfigListener {
|
|
5
8
|
constructor(stage, config) {
|
|
6
|
-
this.
|
|
7
|
-
this.
|
|
8
|
-
|
|
9
|
-
brokers: config.kafka.brokers,
|
|
9
|
+
this.consumer = null;
|
|
10
|
+
this.pulsar = new pulsar_client_1.default.Client({
|
|
11
|
+
serviceUrl: config.pulsar.proxy
|
|
10
12
|
});
|
|
11
|
-
this.
|
|
13
|
+
this.config = config;
|
|
12
14
|
this.stage = stage;
|
|
13
|
-
this.consumerReadyP = new Promise((resolve) => {
|
|
14
|
-
this.consumer.on("consumer.fetch_start", () => resolve());
|
|
15
|
-
});
|
|
16
15
|
}
|
|
17
16
|
async start() {
|
|
18
|
-
await this.
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
17
|
+
this.consumer = await this.pulsar.subscribe({
|
|
18
|
+
topic: this.config.pipelinemanager.config_topic,
|
|
19
|
+
subscription: this.config.plugin.name + `plugin-${this.config.pod.name}`,
|
|
20
|
+
subscriptionType: 'Exclusive',
|
|
21
|
+
listener: async (msg, msgConsumer) => {
|
|
22
|
+
console.log(msg.getData().toString());
|
|
23
|
+
const cfgMessage = JSON.parse(msg.getData().toString());
|
|
24
|
+
if (cfgMessage && cfgMessage.plugin === this.config.plugin.name) {
|
|
25
|
+
await this.stage.restart(cfgMessage.inputTopic, cfgMessage.outputTopic);
|
|
26
26
|
}
|
|
27
|
-
await
|
|
28
|
-
}
|
|
27
|
+
await msgConsumer.acknowledge(msg);
|
|
28
|
+
},
|
|
29
29
|
});
|
|
30
|
-
|
|
30
|
+
}
|
|
31
|
+
async stop() {
|
|
32
|
+
await this.consumer?.close();
|
|
33
|
+
await this.pulsar.close();
|
|
31
34
|
}
|
|
32
35
|
}
|
|
33
36
|
exports.default = ConfigListener;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ConfigListener.js","sourceRoot":"","sources":["../src/ConfigListener.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"ConfigListener.js","sourceRoot":"","sources":["../src/ConfigListener.ts"],"names":[],"mappings":";;;;;AAAA,kEAAmC;AAInC,MAAqB,cAAc;IAM/B,YAAY,KAAoB,EAAE,MAAc;QAFxC,aAAQ,GAAyB,IAAI,CAAC;QAG1C,IAAI,CAAC,MAAM,GAAG,IAAI,uBAAM,CAAC,MAAM,CAAC;YAC5B,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;SAClC,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACxC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,eAAgB,CAAC,YAAY;YAChD,YAAY,EAAG,IAAI,CAAC,MAAM,CAAC,MAAO,CAAC,IAAI,GAAG,UAAU,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE;YAC1E,gBAAgB,EAAE,WAAW;YAC7B,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,EAAE;gBACjC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACtC,MAAM,UAAU,GAAkB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACvE,IAAI,UAAU,IAAI,UAAU,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC;oBAC/D,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,WAAW,CAAC,CAAC;gBAC5E,CAAC;gBACD,MAAM,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACvC,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC9B,CAAC;CACJ;AAnCD,iCAmCC"}
|
package/dist/PipelineStage.d.ts
CHANGED
|
@@ -1,20 +1,21 @@
|
|
|
1
|
-
import
|
|
1
|
+
import Pulsar from 'pulsar-client';
|
|
2
2
|
import { PipelinePluginI } from "./plugin/PipelinePluginI";
|
|
3
3
|
import { Config } from "@cdp-forge/types";
|
|
4
4
|
export default class PipelineSTage {
|
|
5
5
|
plugin: PipelinePluginI;
|
|
6
|
-
consumer: Consumer;
|
|
7
|
-
producer: Producer;
|
|
8
|
-
kafka: Kafka;
|
|
6
|
+
consumer: Pulsar.Consumer | null;
|
|
7
|
+
producer: Pulsar.Producer | null;
|
|
9
8
|
input: string | null;
|
|
10
9
|
output: string | null;
|
|
11
10
|
currentOperation: Promise<void>;
|
|
12
|
-
|
|
11
|
+
private pulsar;
|
|
12
|
+
private config;
|
|
13
13
|
constructor(plugin: PipelinePluginI, config: Config);
|
|
14
14
|
start(inputTopic: string, outputTopic?: string | null): Promise<void>;
|
|
15
15
|
private _start;
|
|
16
16
|
stop(): Promise<void>;
|
|
17
17
|
private _stop;
|
|
18
18
|
restart(inputTopic: string, outputTopic: string | undefined): Promise<void>;
|
|
19
|
+
close(): Promise<void>;
|
|
19
20
|
}
|
|
20
21
|
//# sourceMappingURL=PipelineStage.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PipelineStage.d.ts","sourceRoot":"","sources":["../src/PipelineStage.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"PipelineStage.d.ts","sourceRoot":"","sources":["../src/PipelineStage.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,eAAe,CAAC;AACnC,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAM,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,CAAC,OAAO,OAAO,aAAa;IAC9B,MAAM,EAAE,eAAe,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAQ;IACxC,QAAQ,EAAE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAQ;IACxC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM;IAW7C,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,GAAE,MAAM,GAAG,IAAW,GAAG,OAAO,CAAC,IAAI,CAAC;YASnE,MAAM;IAiCd,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAMb,KAAK;IAOb,OAAO,CAAC,UAAU,EAAE,MAAM,EAAC,WAAW,EAAE,MAAM,GAAC,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAMjE,KAAK;CAIrB"}
|
package/dist/PipelineStage.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const
|
|
6
|
+
const pulsar_client_1 = __importDefault(require("pulsar-client"));
|
|
4
7
|
class PipelineSTage {
|
|
5
8
|
constructor(plugin, config) {
|
|
9
|
+
this.consumer = null;
|
|
10
|
+
this.producer = null;
|
|
6
11
|
this.plugin = plugin;
|
|
7
|
-
this.
|
|
8
|
-
|
|
9
|
-
brokers: config.kafka.brokers,
|
|
10
|
-
});
|
|
11
|
-
this.consumer = this.kafka.consumer({ allowAutoTopicCreation: true, groupId: config.plugin.name + `plugin` });
|
|
12
|
-
this.producer = this.kafka.producer({ allowAutoTopicCreation: true });
|
|
13
|
-
this.consumerReadyP = new Promise((resolve) => {
|
|
14
|
-
this.consumer.on("consumer.fetch_start", () => resolve());
|
|
12
|
+
this.pulsar = new pulsar_client_1.default.Client({
|
|
13
|
+
serviceUrl: config.pulsar.proxy
|
|
15
14
|
});
|
|
15
|
+
this.config = config;
|
|
16
16
|
this.input = null;
|
|
17
17
|
this.output = null;
|
|
18
18
|
this.currentOperation = Promise.resolve();
|
|
@@ -26,32 +26,35 @@ class PipelineSTage {
|
|
|
26
26
|
return this.currentOperation;
|
|
27
27
|
}
|
|
28
28
|
async _start() {
|
|
29
|
-
if (this.output)
|
|
30
|
-
await this.
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
29
|
+
if (this.output) {
|
|
30
|
+
this.producer = await this.pulsar.createProducer({
|
|
31
|
+
topic: this.config.pipelinemanager.first_topic,
|
|
32
|
+
producerName: this.config.plugin.name + "-" + this.config.pod.name
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
this.consumer = await this.pulsar.subscribe({
|
|
36
|
+
topic: this.input,
|
|
37
|
+
subscription: this.config.plugin.name,
|
|
38
|
+
subscriptionType: 'Shared',
|
|
39
|
+
listener: async (msg, msgConsumer) => {
|
|
40
|
+
console.log(msg.getData().toString());
|
|
41
|
+
const log = JSON.parse(msg.getData().toString());
|
|
37
42
|
if (!log)
|
|
38
43
|
return;
|
|
39
44
|
const elaboratedLog = await this.plugin.elaborate(log);
|
|
40
45
|
if (!elaboratedLog) {
|
|
41
46
|
console.warn('Messaggio non elaborato');
|
|
42
|
-
await
|
|
47
|
+
await msgConsumer.acknowledge(msg);
|
|
43
48
|
return;
|
|
44
49
|
}
|
|
45
|
-
if (this.output) {
|
|
50
|
+
if (this.output && this.producer) {
|
|
46
51
|
await this.producer.send({
|
|
47
|
-
|
|
48
|
-
messages: [{ value: JSON.stringify(elaboratedLog) }],
|
|
52
|
+
data: Buffer.from(JSON.stringify(elaboratedLog)),
|
|
49
53
|
});
|
|
50
|
-
await
|
|
54
|
+
await msgConsumer.acknowledge(msg);
|
|
51
55
|
}
|
|
52
56
|
},
|
|
53
57
|
});
|
|
54
|
-
await this.consumerReadyP;
|
|
55
58
|
}
|
|
56
59
|
async stop() {
|
|
57
60
|
await this.currentOperation;
|
|
@@ -59,8 +62,10 @@ class PipelineSTage {
|
|
|
59
62
|
return this.currentOperation;
|
|
60
63
|
}
|
|
61
64
|
async _stop() {
|
|
62
|
-
await this.consumer
|
|
63
|
-
await this.producer
|
|
65
|
+
await this.consumer?.close();
|
|
66
|
+
await this.producer?.close();
|
|
67
|
+
this.consumer = null;
|
|
68
|
+
this.producer = null;
|
|
64
69
|
}
|
|
65
70
|
async restart(inputTopic, outputTopic) {
|
|
66
71
|
if (this.input === inputTopic && this.output === outputTopic)
|
|
@@ -68,6 +73,10 @@ class PipelineSTage {
|
|
|
68
73
|
await this.stop();
|
|
69
74
|
await this.start(inputTopic, outputTopic);
|
|
70
75
|
}
|
|
76
|
+
async close() {
|
|
77
|
+
await this.stop();
|
|
78
|
+
await this.pulsar.close();
|
|
79
|
+
}
|
|
71
80
|
}
|
|
72
81
|
exports.default = PipelineSTage;
|
|
73
82
|
//# sourceMappingURL=PipelineStage.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PipelineStage.js","sourceRoot":"","sources":["../src/PipelineStage.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"PipelineStage.js","sourceRoot":"","sources":["../src/PipelineStage.ts"],"names":[],"mappings":";;;;;AAAA,kEAAmC;AAInC,MAAqB,aAAa;IAU9B,YAAY,MAAuB,EAAE,MAAc;QARnD,aAAQ,GAA2B,IAAI,CAAC;QACxC,aAAQ,GAA2B,IAAI,CAAC;QAQpC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,uBAAM,CAAC,MAAM,CAAC;YAC9B,UAAU,EAAE,MAAM,CAAC,MAAO,CAAC,KAAK;SACjC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAkB,EAAE,cAA6B,IAAI;QAC7D,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC;QAC1B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7E,MAAM,IAAI,CAAC,gBAAgB,CAAC;QAC5B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,MAAM;QAChB,IAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,IAAI,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;gBAC7C,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,eAAgB,CAAC,WAAW;gBAC/C,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,MAAO,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI;aACtE,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACxC,KAAK,EAAE,IAAI,CAAC,KAAM;YAClB,YAAY,EAAG,IAAI,CAAC,MAAM,CAAC,MAAO,CAAC,IAAI;YACvC,gBAAgB,EAAE,QAAQ;YAC1B,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,EAAE;gBACjC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACtC,MAAM,GAAG,GAAQ,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACtD,IAAG,CAAC,GAAG;oBAAE,OAAO;gBAChB,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBACvD,IAAG,CAAC,aAAa,EAAE,CAAC;oBAChB,OAAO,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;oBACxC,MAAM,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;oBACnC,OAAO;gBACX,CAAC;gBAED,IAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAC9B,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;wBACrB,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;qBACnD,CAAC,CAAC;oBACH,MAAM,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACvC,CAAC;YACL,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,gBAAgB,CAAC;QAC5B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACrC,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,KAAK;QACf,MAAM,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,UAAkB,EAAC,WAA6B;QAC1D,IAAG,IAAI,CAAC,KAAK,KAAK,UAAU,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW;YAAE,OAAO;QACpE,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,EAAC,WAAW,CAAC,CAAC;IAC7C,CAAC;IAEM,KAAK,CAAC,KAAK;QACd,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC9B,CAAC;CACJ;AAtFD,gCAsFC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"default.d.ts","sourceRoot":"","sources":["../../src/config/default.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAoB1C,QAAA,MAAM,MAAM,EAAE,
|
|
1
|
+
{"version":3,"file":"default.d.ts","sourceRoot":"","sources":["../../src/config/default.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAoB1C,QAAA,MAAM,MAAM,EAAE,MAsBb,CAAC;AAEF,eAAe,MAAM,CAAC"}
|
package/dist/config/default.js
CHANGED
|
@@ -18,8 +18,10 @@ function env(key, fallback, type = 'string') {
|
|
|
18
18
|
}
|
|
19
19
|
}
|
|
20
20
|
const config = {
|
|
21
|
-
|
|
22
|
-
|
|
21
|
+
pulsar: {
|
|
22
|
+
proxy: env('PULSAR_PROXY', 'pulsar://cdp-forge-pulsar-proxy:6650'),
|
|
23
|
+
uiPassword: env('PULSAR_UI_PASSWORD', ""),
|
|
24
|
+
dbPassword: env('PULSAR_DB_PASSWORD', ""),
|
|
23
25
|
},
|
|
24
26
|
mysql: {
|
|
25
27
|
uri: env("MYSQL_URI", "mysql://root:cdp-forge-root-2024@cdp-forge-mysql:3306/cdpforge")
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"default.js","sourceRoot":"","sources":["../../src/config/default.ts"],"names":[],"mappings":";;AAGA,SAAS,GAAG,CAAmB,GAAW,EAAE,QAAW,EAAE,OAAmE,QAAQ;IAChI,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,QAAQ,IAAI,EAAE,CAAC;QACX,KAAK,UAAU;YACX,OAAO,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAM,CAAC;QAClD,KAAK,UAAU;YACX,OAAO,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAM,CAAC;QAC1D,KAAK,SAAS;YACV,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,CAAM,CAAC;QAC/C,KAAK,QAAQ;YACT,OAAO,MAAM,CAAC,GAAG,CAAM,CAAC;QAC5B,KAAK,QAAQ;YACT,OAAO,GAAQ,CAAC;IACxB,CAAC;AACL,CAAC;AAED,MAAM,MAAM,GAAY;IACpB,
|
|
1
|
+
{"version":3,"file":"default.js","sourceRoot":"","sources":["../../src/config/default.ts"],"names":[],"mappings":";;AAGA,SAAS,GAAG,CAAmB,GAAW,EAAE,QAAW,EAAE,OAAmE,QAAQ;IAChI,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,QAAQ,IAAI,EAAE,CAAC;QACX,KAAK,UAAU;YACX,OAAO,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAM,CAAC;QAClD,KAAK,UAAU;YACX,OAAO,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAM,CAAC;QAC1D,KAAK,SAAS;YACV,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,CAAM,CAAC;QAC/C,KAAK,QAAQ;YACT,OAAO,MAAM,CAAC,GAAG,CAAM,CAAC;QAC5B,KAAK,QAAQ;YACT,OAAO,GAAQ,CAAC;IACxB,CAAC;AACL,CAAC;AAED,MAAM,MAAM,GAAY;IACpB,MAAM,EAAE;QACJ,KAAK,EAAE,GAAG,CAAC,cAAc,EAAE,sCAAsC,CAAC;QAClE,UAAU,EAAE,GAAG,CAAC,oBAAoB,EAAC,EAAE,CAAC;QACxC,UAAU,EAAE,GAAG,CAAC,oBAAoB,EAAC,EAAE,CAAC;KAC3C;IACD,KAAK,EAAE;QACH,GAAG,EAAE,GAAG,CAAC,WAAW,EAAC,gEAAgE,CAAC;KACzF;IACD,UAAU,EAAC;QACP,GAAG,EAAE,GAAG,CAAC,gBAAgB,EAAC,wCAAwC,CAAC;QACnE,QAAQ,EAAE,GAAG,CAAC,qBAAqB,EAAC,OAAO,CAAC;QAC5C,QAAQ,EAAE,GAAG,CAAC,qBAAqB,EAAC,gBAAgB,CAAC;KACxD;IACD,eAAe,EAAE;QACb,YAAY,EAAE,GAAG,CAAC,8BAA8B,EAAC,QAAQ,CAAC;QAC1D,GAAG,EAAE,GAAG,CAAC,qBAAqB,EAAC,wCAAwC,CAAC;QACxE,WAAW,EAAE,GAAG,CAAC,6BAA6B,EAAE,MAAM,CAAC;KAC1D;IACD,GAAG,EAAE;QACD,IAAI,EAAE,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;KACtE;CACJ,CAAC;AAEF,kBAAe,MAAM,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"startServer.d.ts","sourceRoot":"","sources":["../src/startServer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,aAAa,MAAM,iBAAiB,CAAC;AAC5C,OAAO,cAAc,MAAM,kBAAkB,CAAC;AAG9C,wBAAsB,KAAK,CAAC,MAAM,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC;;;
|
|
1
|
+
{"version":3,"file":"startServer.d.ts","sourceRoot":"","sources":["../src/startServer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,aAAa,MAAM,iBAAiB,CAAC;AAC5C,OAAO,cAAc,MAAM,kBAAkB,CAAC;AAG9C,wBAAsB,KAAK,CAAC,MAAM,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC;;;GA2BhF"}
|
package/dist/startServer.js
CHANGED
|
@@ -24,7 +24,8 @@ async function start(plugin, pluginConfig) {
|
|
|
24
24
|
}
|
|
25
25
|
const handleExit = async () => {
|
|
26
26
|
console.log('Arresto del server in corso...');
|
|
27
|
-
await
|
|
27
|
+
await configListener.stop();
|
|
28
|
+
await stage.close();
|
|
28
29
|
process.exit(0);
|
|
29
30
|
};
|
|
30
31
|
process.on('SIGINT', handleExit);
|
package/dist/startServer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"startServer.js","sourceRoot":"","sources":["../src/startServer.ts"],"names":[],"mappings":";;;;;AAMA,
|
|
1
|
+
{"version":3,"file":"startServer.js","sourceRoot":"","sources":["../src/startServer.ts"],"names":[],"mappings":";;;;;AAMA,sBA2BG;AA/BH,oEAA4C;AAC5C,sEAA8C;AAC9C,+DAA6C;AAEtC,KAAK,UAAU,KAAK,CAAC,MAAuB,EAAE,YAA8B;IAC/E,MAAM,MAAM,GAAW,MAAM,CAAC,MAAM,CAAC,iBAAa,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;IAC9E,MAAM,KAAK,GAAG,IAAI,uBAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,MAAM,cAAc,GAAG,IAAI,wBAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACzD,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;IAC7B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,MAAM,CAAC,eAAgB,CAAC,GAAG,WAAW,EAAE;QAC/D,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;SACnC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;KACtC,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,oDAAoD,GAAG,CAAC,UAAU,cAAc,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrI,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,IAAI,EAAE;QAC5B,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC9C,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IAElC,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;IACrE,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;AACnC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cdp-forge/plugin-pipeline-sdk",
|
|
3
|
-
"version": "1.1
|
|
3
|
+
"version": "1.2.1",
|
|
4
4
|
"description": "SDK for pipeline plugins for CDP Forge platform",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
"test": "npx jest",
|
|
15
15
|
"test:build": "tsc -p tsconfig.test.json",
|
|
16
16
|
"build": "tsc",
|
|
17
|
+
"prebuild": "npm run clean",
|
|
17
18
|
"prepublishOnly": "npm run build",
|
|
18
19
|
"clean": "rm -rf dist"
|
|
19
20
|
},
|
|
@@ -33,14 +34,15 @@
|
|
|
33
34
|
},
|
|
34
35
|
"homepage": "https://github.com/CDPForge/plugin-pipeline-template#readme",
|
|
35
36
|
"dependencies": {
|
|
36
|
-
"@cdp-forge/types": "^1.
|
|
37
|
+
"@cdp-forge/types": "^1.2.0",
|
|
37
38
|
"dotenv": "^17.2.0",
|
|
38
|
-
"
|
|
39
|
-
"
|
|
39
|
+
"node-fetch": "^3.3.2",
|
|
40
|
+
"pulsar-client": "^1.14.0"
|
|
40
41
|
},
|
|
41
42
|
"devDependencies": {
|
|
42
43
|
"@types/jest": "^29.5.14",
|
|
43
44
|
"@types/node": "^20.12.12",
|
|
45
|
+
"@types/pulsar-client": "^1.3.0",
|
|
44
46
|
"jest": "^29.7.0",
|
|
45
47
|
"nodemon": "^3.1.0",
|
|
46
48
|
"ts-jest": "^29.2.5",
|
package/src/config/default.ts
CHANGED
|
@@ -19,8 +19,10 @@ function env<T extends Parsed>(key: string, fallback: T, type: 'string' | 'numbe
|
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
const config: Config = {
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
pulsar: {
|
|
23
|
+
proxy: env('PULSAR_PROXY', 'pulsar://cdp-forge-pulsar-proxy:6650'),
|
|
24
|
+
uiPassword: env('PULSAR_UI_PASSWORD',""),
|
|
25
|
+
dbPassword: env('PULSAR_DB_PASSWORD',""),
|
|
24
26
|
},
|
|
25
27
|
mysql: {
|
|
26
28
|
uri: env("MYSQL_URI","mysql://root:cdp-forge-root-2024@cdp-forge-mysql:3306/cdpforge")
|
package/dist/config/default.ts
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
import { Config } from "@cdp-forge/types";
|
|
2
|
-
type Parsed = string | number | boolean | string[] | number[];
|
|
3
|
-
|
|
4
|
-
function env<T extends Parsed>(key: string, fallback: T, type: 'string' | 'number' | 'boolean' | 'string[]' | 'number[]' = 'string'): T {
|
|
5
|
-
const val = process.env[key];
|
|
6
|
-
if (val === undefined) return fallback;
|
|
7
|
-
switch (type) {
|
|
8
|
-
case "string[]":
|
|
9
|
-
return val.split(',').map(s => s.trim()) as T;
|
|
10
|
-
case "number[]":
|
|
11
|
-
return val.split(',').map(s => Number(s.trim())) as T;
|
|
12
|
-
case "boolean":
|
|
13
|
-
return (val.toLowerCase() === 'true') as T;
|
|
14
|
-
case "number":
|
|
15
|
-
return Number(val) as T;
|
|
16
|
-
case "string":
|
|
17
|
-
return val as T;
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
const config: Config = {
|
|
22
|
-
kafka: {
|
|
23
|
-
brokers: env<string[]>('KAFKA_BROKERS', ['cdp-forge-kafka-kafka-bootstrap:9092'], 'string[]')
|
|
24
|
-
},
|
|
25
|
-
mysql: {
|
|
26
|
-
uri: env("MYSQL_URI","mysql://root:cdp-forge-root-2024@cdp-forge-mysql:3306/cdpforge")
|
|
27
|
-
},
|
|
28
|
-
opensearch:{
|
|
29
|
-
url: env("OPENSEARCH_URL","https://opensearch-cluster-master:9200"),
|
|
30
|
-
username: env("OPENSEARCH_USERNAME","admin"),
|
|
31
|
-
password: env("OPENSEARCH_PASSWORD","CdpForge@2024!")
|
|
32
|
-
},
|
|
33
|
-
pipelinemanager: {
|
|
34
|
-
config_topic: env("PIPELINEMANAGER_CONFIG_TOPIC","config"),
|
|
35
|
-
url: env("PIPELINEMANAGER_URL","http://cdp-forge-core-pipeline-manager"),
|
|
36
|
-
first_topic: env("PIPELINEMANAGER_FIRST_TOPIC", "logs")
|
|
37
|
-
},
|
|
38
|
-
pod: {
|
|
39
|
-
name: env("CLIENT_ID", Math.random().toString(36).substring(2, 10))
|
|
40
|
-
}
|
|
41
|
-
};
|
|
42
|
-
|
|
43
|
-
export default config;
|