@dbos-inc/dbos-sdk 1.9.7-preview → 1.9.10-preview.g0dc217a304
Sign up to get free protection for your applications and to get access to all the features.
|
@@ -4,11 +4,11 @@ import { ClassRegistration, MethodRegistration, RegistrationDefaults } from "../
|
|
|
4
4
|
import { DBOSExecutor } from "../dbos-executor";
|
|
5
5
|
type KafkaArgs = [string, number, KafkaMessage];
|
|
6
6
|
export declare class KafkaRegistration<This, Args extends unknown[], Return> extends MethodRegistration<This, Args, Return> {
|
|
7
|
-
|
|
7
|
+
kafkaTopics?: string | RegExp | Array<string | RegExp>;
|
|
8
8
|
consumerConfig?: ConsumerConfig;
|
|
9
9
|
constructor(origFunc: (this: This, ...args: Args) => Promise<Return>);
|
|
10
10
|
}
|
|
11
|
-
export declare function KafkaConsume(
|
|
11
|
+
export declare function KafkaConsume(topics: string | RegExp | Array<string | RegExp>, consumerConfig?: ConsumerConfig): <This, Ctx extends DBOSContext, Return>(target: object, propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ctx: Ctx, args_0: string, args_1: number, args_2: KafkaMessage) => Promise<Return>>) => TypedPropertyDescriptor<(this: This, ctx: Ctx, args_1: string, args_2: number, args_3: KafkaMessage) => Promise<Return>>;
|
|
12
12
|
export interface KafkaDefaults extends RegistrationDefaults {
|
|
13
13
|
kafkaConfig?: KafkaConfig;
|
|
14
14
|
}
|
|
@@ -25,6 +25,7 @@ export declare class DBOSKafka {
|
|
|
25
25
|
constructor(dbosExec: DBOSExecutor);
|
|
26
26
|
initKafka(): Promise<void>;
|
|
27
27
|
destroyKafka(): Promise<void>;
|
|
28
|
+
safeGroupName(topics: Array<string | RegExp>): string;
|
|
28
29
|
logRegisteredKafkaEndpoints(): void;
|
|
29
30
|
}
|
|
30
31
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"kafka.d.ts","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,QAAQ,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAwB,MAAM,SAAS,CAAC;AACtH,OAAO,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,oBAAoB,EAAyD,MAAM,eAAe,CAAC;AACnJ,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,KAAK,SAAS,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,CAAA;AAM/C,qBAAa,iBAAiB,CAAC,IAAI,EAAE,IAAI,SAAS,OAAO,EAAE,EAAE,MAAM,CAAE,SAAQ,kBAAkB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC;IACjH,
|
|
1
|
+
{"version":3,"file":"kafka.d.ts","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,QAAQ,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAwB,MAAM,SAAS,CAAC;AACtH,OAAO,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,oBAAoB,EAAyD,MAAM,eAAe,CAAC;AACnJ,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,KAAK,SAAS,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,CAAA;AAM/C,qBAAa,iBAAiB,CAAC,IAAI,EAAE,IAAI,SAAS,OAAO,EAAE,EAAE,MAAM,CAAE,SAAQ,kBAAkB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC;IACjH,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,cAAc,CAAC;gBAEpB,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC,MAAM,CAAC;CAGrE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,cAAc,CAAC,EAAE,cAAc,mDAElG,MAAM,eACD,MAAM,sQAWtB;AAMD,MAAM,WAAW,aAAc,SAAQ,oBAAoB;IACzD,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AAED,qBAAa,sBAAsB,CAAC,EAAE,SAAS;IAAE,KAAI,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,CAAA;CAAE,CAAE,SAAQ,iBAAiB,CAAC,EAAE,CAAE,YAAW,aAAa;IACxI,WAAW,CAAC,EAAE,WAAW,CAAC;gBAEd,IAAI,EAAE,EAAE;CAGrB;AAED,wBAAgB,KAAK,CAAC,WAAW,EAAE,WAAW,4BACH,OAAO,EAAE,KAAG,MAAM,mBAK5D;AAMD,qBAAa,SAAS;IAGR,QAAQ,CAAC,QAAQ,EAAE,YAAY;IAF3C,QAAQ,CAAC,SAAS,EAAE,QAAQ,EAAE,CAAM;gBAEf,QAAQ,EAAE,YAAY;IAErC,SAAS;IAiET,YAAY;IAMlB,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IAQ5C,2BAA2B;CAiB5B"}
|
package/dist/src/kafka/kafka.js
CHANGED
|
@@ -9,18 +9,18 @@ const utils_1 = require("../utils");
|
|
|
9
9
|
/* Kafka Method Decorators */
|
|
10
10
|
/////////////////////////////
|
|
11
11
|
class KafkaRegistration extends decorators_1.MethodRegistration {
|
|
12
|
-
|
|
12
|
+
kafkaTopics;
|
|
13
13
|
consumerConfig;
|
|
14
14
|
constructor(origFunc) {
|
|
15
15
|
super(origFunc);
|
|
16
16
|
}
|
|
17
17
|
}
|
|
18
18
|
exports.KafkaRegistration = KafkaRegistration;
|
|
19
|
-
function KafkaConsume(
|
|
19
|
+
function KafkaConsume(topics, consumerConfig) {
|
|
20
20
|
function kafkadec(target, propertyKey, inDescriptor) {
|
|
21
21
|
const { descriptor, registration } = (0, decorators_1.registerAndWrapFunction)(target, propertyKey, inDescriptor);
|
|
22
22
|
const kafkaRegistration = registration;
|
|
23
|
-
kafkaRegistration.
|
|
23
|
+
kafkaRegistration.kafkaTopics = topics;
|
|
24
24
|
kafkaRegistration.consumerConfig = consumerConfig;
|
|
25
25
|
return descriptor;
|
|
26
26
|
}
|
|
@@ -54,7 +54,7 @@ class DBOSKafka {
|
|
|
54
54
|
async initKafka() {
|
|
55
55
|
for (const registeredOperation of this.dbosExec.registeredOperations) {
|
|
56
56
|
const ro = registeredOperation;
|
|
57
|
-
if (ro.
|
|
57
|
+
if (ro.kafkaTopics) {
|
|
58
58
|
const defaults = ro.defaults;
|
|
59
59
|
if (!ro.txnConfig && !ro.workflowConfig) {
|
|
60
60
|
throw new error_1.DBOSError(`Error registering method ${defaults.name}.${ro.name}: A Kafka decorator can only be assigned to a transaction or workflow!`);
|
|
@@ -62,8 +62,15 @@ class DBOSKafka {
|
|
|
62
62
|
if (!defaults.kafkaConfig) {
|
|
63
63
|
throw new error_1.DBOSError(`Error registering method ${defaults.name}.${ro.name}: Kafka configuration not found. Does class ${defaults.name} have an @Kafka decorator?`);
|
|
64
64
|
}
|
|
65
|
+
const topics = [];
|
|
66
|
+
if (Array.isArray(ro.kafkaTopics)) {
|
|
67
|
+
topics.push(...ro.kafkaTopics);
|
|
68
|
+
}
|
|
69
|
+
else if (ro.kafkaTopics) {
|
|
70
|
+
topics.push(ro.kafkaTopics);
|
|
71
|
+
}
|
|
65
72
|
const kafka = new kafkajs_1.Kafka(defaults.kafkaConfig);
|
|
66
|
-
const consumerConfig = ro.consumerConfig ?? { groupId:
|
|
73
|
+
const consumerConfig = ro.consumerConfig ?? { groupId: `${this.safeGroupName(topics)}` };
|
|
67
74
|
const consumer = kafka.consumer(consumerConfig);
|
|
68
75
|
await consumer.connect();
|
|
69
76
|
// A temporary workaround for https://github.com/tulios/kafkajs/pull/1558 until it gets fixed
|
|
@@ -74,7 +81,7 @@ class DBOSKafka {
|
|
|
74
81
|
const multiplier = defaults.kafkaConfig.retry ? defaults.kafkaConfig.retry.multiplier ?? 2 : 2;
|
|
75
82
|
for (let i = 0; i < maxRetries; i++) {
|
|
76
83
|
try {
|
|
77
|
-
await consumer.subscribe({
|
|
84
|
+
await consumer.subscribe({ topics: topics, fromBeginning: true });
|
|
78
85
|
break;
|
|
79
86
|
}
|
|
80
87
|
catch (error) {
|
|
@@ -116,14 +123,28 @@ class DBOSKafka {
|
|
|
116
123
|
await consumer.disconnect();
|
|
117
124
|
}
|
|
118
125
|
}
|
|
126
|
+
safeGroupName(topics) {
|
|
127
|
+
const safeGroupIdPart = topics
|
|
128
|
+
.map(r => r.toString())
|
|
129
|
+
.map(r => r.replaceAll(/[^a-zA-Z0-9\\-]/g, ''))
|
|
130
|
+
.join('-');
|
|
131
|
+
return `dbos-kafka-group-${safeGroupIdPart}`.slice(0, 255);
|
|
132
|
+
}
|
|
119
133
|
logRegisteredKafkaEndpoints() {
|
|
120
134
|
const logger = this.dbosExec.logger;
|
|
121
135
|
logger.info("Kafka endpoints supported:");
|
|
122
136
|
this.dbosExec.registeredOperations.forEach((registeredOperation) => {
|
|
123
137
|
const ro = registeredOperation;
|
|
124
|
-
if (ro.
|
|
138
|
+
if (ro.kafkaTopics) {
|
|
125
139
|
const defaults = ro.defaults;
|
|
126
|
-
|
|
140
|
+
if (Array.isArray(ro.kafkaTopics)) {
|
|
141
|
+
ro.kafkaTopics.forEach(kafkaTopic => {
|
|
142
|
+
logger.info(` ${kafkaTopic} -> ${defaults.name}.${ro.name}`);
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
logger.info(` ${ro.kafkaTopics} -> ${defaults.name}.${ro.name}`);
|
|
147
|
+
}
|
|
127
148
|
}
|
|
128
149
|
});
|
|
129
150
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"kafka.js","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":";;;AAAA,qCAAsH;AAEtH,8CAAmJ;AAInJ,oCAAqC;AACrC,oCAAiC;AAIjC,6BAA6B;AAC7B,6BAA6B;AAC7B,6BAA6B;AAE7B,MAAa,iBAAwD,SAAQ,+BAAsC;IACjH,
|
|
1
|
+
{"version":3,"file":"kafka.js","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":";;;AAAA,qCAAsH;AAEtH,8CAAmJ;AAInJ,oCAAqC;AACrC,oCAAiC;AAIjC,6BAA6B;AAC7B,6BAA6B;AAC7B,6BAA6B;AAE7B,MAAa,iBAAwD,SAAQ,+BAAsC;IACjH,WAAW,CAA4C;IACvD,cAAc,CAAkB;IAEhC,YAAY,QAAwD;QAClE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAClB,CAAC;CACF;AAPD,8CAOC;AAED,SAAgB,YAAY,CAAC,MAAgD,EAAE,cAA+B;IAC5G,SAAS,QAAQ,CACf,MAAc,EACd,WAAmB,EACnB,YAAoG;QAEpG,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,IAAA,oCAAuB,EAAC,MAAM,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;QAChG,MAAM,iBAAiB,GAAG,YAAqE,CAAC;QAChG,iBAAiB,CAAC,WAAW,GAAG,MAAM,CAAC;QACvC,iBAAiB,CAAC,cAAc,GAAG,cAAc,CAAC;QAElD,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAdD,oCAcC;AAUD,MAAa,sBAAuE,SAAQ,8BAAqB;IAC/G,WAAW,CAAe;IAE1B,YAAY,IAAQ;QAClB,KAAK,CAAC,IAAI,CAAC,CAAC;IACd,CAAC;CACF;AAND,wDAMC;AAED,SAAgB,KAAK,CAAC,WAAwB;IAC5C,SAAS,MAAM,CAAgD,IAAO;QACpE,MAAM,MAAM,GAAG,IAAA,yCAA4B,EAAC,IAAI,CAA8B,CAAC;QAC/E,MAAM,CAAC,WAAW,GAAG,WAAW,CAAC;IACnC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAND,sBAMC;AAED,wBAAwB;AACxB,uBAAuB;AACvB,uBAAuB;AAEvB,MAAa,SAAS;IAGC;IAFZ,SAAS,GAAe,EAAE,CAAC;IAEpC,YAAqB,QAAsB;QAAtB,aAAQ,GAAR,QAAQ,CAAc;IAAI,CAAC;IAEhD,KAAK,CAAC,SAAS;QACb,KAAK,MAAM,mBAAmB,IAAI,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE,CAAC;YACrE,MAAM,EAAE,GAAG,mBAAqE,CAAC;YACjF,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBACnB,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAyB,CAAC;gBAC9C,IAAI,CAAC,EAAE,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC;oBACxC,MAAM,IAAI,iBAAS,CAAC,4BAA4B,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,wEAAwE,CAAC,CAAA;gBACnJ,CAAC;gBACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;oBAC1B,MAAM,IAAI,iBAAS,CAAC,4BAA4B,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,+CAA+C,QAAQ,CAAC,IAAI,4BAA4B,CAAC,CAAA;gBACnK,CAAC;gBACD,MAAM,MAAM,GAA2B,EAAE,CAAC;gBAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,EAAG,CAAC;oBACnC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,CAAA;gBAChC,CAAC;qBACD,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;oBACnB,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAA;gBAC7B,CAAC;gBACD,MAAM,KAAK,GAAG,IAAI,eAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAChD,MAAM,cAAc,GAAG,EAAE,CAAC,cAAc,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;gBACzF,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;gBAChD,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACzB,6FAA6F;gBAC7F,gIAAgI;gBAChI,gEAAgE;gBAChE,MAAM,UAAU,GAAG,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC5F,IAAI,SAAS,GAAG,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,YAAY,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;gBAClG,MAAM,UAAU,GAAG,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/F,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;oBACpC,IAAI,CAAC;wBACH,MAAM,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;wBAClE,MAAM;oBACR,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,MAAM,CAAC,GAAG,KAA6B,CAAC;wBACxC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,EAAE,CAAC,CAAC,6BAA6B;4BACrE,MAAM,IAAA,aAAK,EAAC,SAAS,CAAC,CAAC;4BACvB,SAAS,IAAI,UAAU,CAAC;4BACxB,SAAS;wBACX,CAAC;6BAAM,CAAC;4BACN,MAAM,CAAC,CAAA;wBACT,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,MAAM,QAAQ,CAAC,GAAG,CAAC;oBACjB,WAAW,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,EAAE;wBACnD,2EAA2E;wBAC3E,MAAM,YAAY,GAAG,mBAAmB,KAAK,IAAI,SAAS,IAAI,OAAO,CAAC,MAAM,EAAE,CAAA;wBAC9E,MAAM,QAAQ,GAAG,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC;wBAChD,oFAAoF;wBACpF,MAAM,IAAI,GAAc,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;wBACpD,0FAA0F;wBAC1F,IAAI,EAAE,CAAC,SAAS,EAAE,CAAC;4BACjB,0BAA0B;4BAC1B,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,kBAAqD,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,CAAC;wBAC/G,CAAC;6BAAM,IAAI,EAAE,CAAC,cAAc,EAAE,CAAC;4BAC7B,4BAA4B;4BAC5B,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,kBAAkD,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,CAAC;wBACzG,CAAC;oBACH,CAAC;iBACF,CAAC,CAAA;gBACF,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,MAAM,QAAQ,CAAC,UAAU,EAAE,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,aAAa,CAAC,MAA8B;QAC1C,MAAM,eAAe,GAAI,MAAM;aAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;aACtB,GAAG,CAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;aAC/C,IAAI,CAAC,GAAG,CAAC,CAAC;QACb,OAAO,oBAAoB,eAAe,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC7D,CAAC;IAED,2BAA2B;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,mBAAmB,EAAE,EAAE;YACjE,MAAM,EAAE,GAAG,mBAAqE,CAAC;YACjF,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBACnB,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAyB,CAAC;gBAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC;oBAClC,EAAE,CAAC,WAAW,CAAC,OAAO,CAAE,UAAU,CAAC,EAAE;wBACnC,MAAM,CAAC,IAAI,CAAC,OAAO,UAAU,OAAO,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;oBAClE,CAAC,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,WAAW,OAAO,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;gBACtE,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AArGD,8BAqGC"}
|