@dbos-inc/dbos-sdk 1.9.7-preview → 1.9.10-preview.g0dc217a304
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,11 +4,11 @@ import { ClassRegistration, MethodRegistration, RegistrationDefaults } from "../
|
|
4
4
|
import { DBOSExecutor } from "../dbos-executor";
|
5
5
|
type KafkaArgs = [string, number, KafkaMessage];
|
6
6
|
export declare class KafkaRegistration<This, Args extends unknown[], Return> extends MethodRegistration<This, Args, Return> {
|
7
|
-
|
7
|
+
kafkaTopics?: string | RegExp | Array<string | RegExp>;
|
8
8
|
consumerConfig?: ConsumerConfig;
|
9
9
|
constructor(origFunc: (this: This, ...args: Args) => Promise<Return>);
|
10
10
|
}
|
11
|
-
export declare function KafkaConsume(
|
11
|
+
export declare function KafkaConsume(topics: string | RegExp | Array<string | RegExp>, consumerConfig?: ConsumerConfig): <This, Ctx extends DBOSContext, Return>(target: object, propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ctx: Ctx, args_0: string, args_1: number, args_2: KafkaMessage) => Promise<Return>>) => TypedPropertyDescriptor<(this: This, ctx: Ctx, args_1: string, args_2: number, args_3: KafkaMessage) => Promise<Return>>;
|
12
12
|
export interface KafkaDefaults extends RegistrationDefaults {
|
13
13
|
kafkaConfig?: KafkaConfig;
|
14
14
|
}
|
@@ -25,6 +25,7 @@ export declare class DBOSKafka {
|
|
25
25
|
constructor(dbosExec: DBOSExecutor);
|
26
26
|
initKafka(): Promise<void>;
|
27
27
|
destroyKafka(): Promise<void>;
|
28
|
+
safeGroupName(topics: Array<string | RegExp>): string;
|
28
29
|
logRegisteredKafkaEndpoints(): void;
|
29
30
|
}
|
30
31
|
export {};
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"kafka.d.ts","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,QAAQ,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAwB,MAAM,SAAS,CAAC;AACtH,OAAO,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,oBAAoB,EAAyD,MAAM,eAAe,CAAC;AACnJ,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,KAAK,SAAS,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,CAAA;AAM/C,qBAAa,iBAAiB,CAAC,IAAI,EAAE,IAAI,SAAS,OAAO,EAAE,EAAE,MAAM,CAAE,SAAQ,kBAAkB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC;IACjH,
|
1
|
+
{"version":3,"file":"kafka.d.ts","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,QAAQ,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAwB,MAAM,SAAS,CAAC;AACtH,OAAO,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,oBAAoB,EAAyD,MAAM,eAAe,CAAC;AACnJ,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,KAAK,SAAS,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,CAAA;AAM/C,qBAAa,iBAAiB,CAAC,IAAI,EAAE,IAAI,SAAS,OAAO,EAAE,EAAE,MAAM,CAAE,SAAQ,kBAAkB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC;IACjH,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,cAAc,CAAC;gBAEpB,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC,MAAM,CAAC;CAGrE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,cAAc,CAAC,EAAE,cAAc,mDAElG,MAAM,eACD,MAAM,sQAWtB;AAMD,MAAM,WAAW,aAAc,SAAQ,oBAAoB;IACzD,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AAED,qBAAa,sBAAsB,CAAC,EAAE,SAAS;IAAE,KAAI,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,CAAA;CAAE,CAAE,SAAQ,iBAAiB,CAAC,EAAE,CAAE,YAAW,aAAa;IACxI,WAAW,CAAC,EAAE,WAAW,CAAC;gBAEd,IAAI,EAAE,EAAE;CAGrB;AAED,wBAAgB,KAAK,CAAC,WAAW,EAAE,WAAW,4BACH,OAAO,EAAE,KAAG,MAAM,mBAK5D;AAMD,qBAAa,SAAS;IAGR,QAAQ,CAAC,QAAQ,EAAE,YAAY;IAF3C,QAAQ,CAAC,SAAS,EAAE,QAAQ,EAAE,CAAM;gBAEf,QAAQ,EAAE,YAAY;IAErC,SAAS;IAiET,YAAY;IAMlB,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IAQ5C,2BAA2B;CAiB5B"}
|
package/dist/src/kafka/kafka.js
CHANGED
@@ -9,18 +9,18 @@ const utils_1 = require("../utils");
|
|
9
9
|
/* Kafka Method Decorators */
|
10
10
|
/////////////////////////////
|
11
11
|
class KafkaRegistration extends decorators_1.MethodRegistration {
|
12
|
-
|
12
|
+
kafkaTopics;
|
13
13
|
consumerConfig;
|
14
14
|
constructor(origFunc) {
|
15
15
|
super(origFunc);
|
16
16
|
}
|
17
17
|
}
|
18
18
|
exports.KafkaRegistration = KafkaRegistration;
|
19
|
-
function KafkaConsume(
|
19
|
+
function KafkaConsume(topics, consumerConfig) {
|
20
20
|
function kafkadec(target, propertyKey, inDescriptor) {
|
21
21
|
const { descriptor, registration } = (0, decorators_1.registerAndWrapFunction)(target, propertyKey, inDescriptor);
|
22
22
|
const kafkaRegistration = registration;
|
23
|
-
kafkaRegistration.
|
23
|
+
kafkaRegistration.kafkaTopics = topics;
|
24
24
|
kafkaRegistration.consumerConfig = consumerConfig;
|
25
25
|
return descriptor;
|
26
26
|
}
|
@@ -54,7 +54,7 @@ class DBOSKafka {
|
|
54
54
|
async initKafka() {
|
55
55
|
for (const registeredOperation of this.dbosExec.registeredOperations) {
|
56
56
|
const ro = registeredOperation;
|
57
|
-
if (ro.
|
57
|
+
if (ro.kafkaTopics) {
|
58
58
|
const defaults = ro.defaults;
|
59
59
|
if (!ro.txnConfig && !ro.workflowConfig) {
|
60
60
|
throw new error_1.DBOSError(`Error registering method ${defaults.name}.${ro.name}: A Kafka decorator can only be assigned to a transaction or workflow!`);
|
@@ -62,8 +62,15 @@ class DBOSKafka {
|
|
62
62
|
if (!defaults.kafkaConfig) {
|
63
63
|
throw new error_1.DBOSError(`Error registering method ${defaults.name}.${ro.name}: Kafka configuration not found. Does class ${defaults.name} have an @Kafka decorator?`);
|
64
64
|
}
|
65
|
+
const topics = [];
|
66
|
+
if (Array.isArray(ro.kafkaTopics)) {
|
67
|
+
topics.push(...ro.kafkaTopics);
|
68
|
+
}
|
69
|
+
else if (ro.kafkaTopics) {
|
70
|
+
topics.push(ro.kafkaTopics);
|
71
|
+
}
|
65
72
|
const kafka = new kafkajs_1.Kafka(defaults.kafkaConfig);
|
66
|
-
const consumerConfig = ro.consumerConfig ?? { groupId:
|
73
|
+
const consumerConfig = ro.consumerConfig ?? { groupId: `${this.safeGroupName(topics)}` };
|
67
74
|
const consumer = kafka.consumer(consumerConfig);
|
68
75
|
await consumer.connect();
|
69
76
|
// A temporary workaround for https://github.com/tulios/kafkajs/pull/1558 until it gets fixed
|
@@ -74,7 +81,7 @@ class DBOSKafka {
|
|
74
81
|
const multiplier = defaults.kafkaConfig.retry ? defaults.kafkaConfig.retry.multiplier ?? 2 : 2;
|
75
82
|
for (let i = 0; i < maxRetries; i++) {
|
76
83
|
try {
|
77
|
-
await consumer.subscribe({
|
84
|
+
await consumer.subscribe({ topics: topics, fromBeginning: true });
|
78
85
|
break;
|
79
86
|
}
|
80
87
|
catch (error) {
|
@@ -116,14 +123,28 @@ class DBOSKafka {
|
|
116
123
|
await consumer.disconnect();
|
117
124
|
}
|
118
125
|
}
|
126
|
+
safeGroupName(topics) {
|
127
|
+
const safeGroupIdPart = topics
|
128
|
+
.map(r => r.toString())
|
129
|
+
.map(r => r.replaceAll(/[^a-zA-Z0-9\\-]/g, ''))
|
130
|
+
.join('-');
|
131
|
+
return `dbos-kafka-group-${safeGroupIdPart}`.slice(0, 255);
|
132
|
+
}
|
119
133
|
logRegisteredKafkaEndpoints() {
|
120
134
|
const logger = this.dbosExec.logger;
|
121
135
|
logger.info("Kafka endpoints supported:");
|
122
136
|
this.dbosExec.registeredOperations.forEach((registeredOperation) => {
|
123
137
|
const ro = registeredOperation;
|
124
|
-
if (ro.
|
138
|
+
if (ro.kafkaTopics) {
|
125
139
|
const defaults = ro.defaults;
|
126
|
-
|
140
|
+
if (Array.isArray(ro.kafkaTopics)) {
|
141
|
+
ro.kafkaTopics.forEach(kafkaTopic => {
|
142
|
+
logger.info(` ${kafkaTopic} -> ${defaults.name}.${ro.name}`);
|
143
|
+
});
|
144
|
+
}
|
145
|
+
else {
|
146
|
+
logger.info(` ${ro.kafkaTopics} -> ${defaults.name}.${ro.name}`);
|
147
|
+
}
|
127
148
|
}
|
128
149
|
});
|
129
150
|
}
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"kafka.js","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":";;;AAAA,qCAAsH;AAEtH,8CAAmJ;AAInJ,oCAAqC;AACrC,oCAAiC;AAIjC,6BAA6B;AAC7B,6BAA6B;AAC7B,6BAA6B;AAE7B,MAAa,iBAAwD,SAAQ,+BAAsC;IACjH,
|
1
|
+
{"version":3,"file":"kafka.js","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":";;;AAAA,qCAAsH;AAEtH,8CAAmJ;AAInJ,oCAAqC;AACrC,oCAAiC;AAIjC,6BAA6B;AAC7B,6BAA6B;AAC7B,6BAA6B;AAE7B,MAAa,iBAAwD,SAAQ,+BAAsC;IACjH,WAAW,CAA4C;IACvD,cAAc,CAAkB;IAEhC,YAAY,QAAwD;QAClE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAClB,CAAC;CACF;AAPD,8CAOC;AAED,SAAgB,YAAY,CAAC,MAAgD,EAAE,cAA+B;IAC5G,SAAS,QAAQ,CACf,MAAc,EACd,WAAmB,EACnB,YAAoG;QAEpG,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,IAAA,oCAAuB,EAAC,MAAM,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;QAChG,MAAM,iBAAiB,GAAG,YAAqE,CAAC;QAChG,iBAAiB,CAAC,WAAW,GAAG,MAAM,CAAC;QACvC,iBAAiB,CAAC,cAAc,GAAG,cAAc,CAAC;QAElD,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAdD,oCAcC;AAUD,MAAa,sBAAuE,SAAQ,8BAAqB;IAC/G,WAAW,CAAe;IAE1B,YAAY,IAAQ;QAClB,KAAK,CAAC,IAAI,CAAC,CAAC;IACd,CAAC;CACF;AAND,wDAMC;AAED,SAAgB,KAAK,CAAC,WAAwB;IAC5C,SAAS,MAAM,CAAgD,IAAO;QACpE,MAAM,MAAM,GAAG,IAAA,yCAA4B,EAAC,IAAI,CAA8B,CAAC;QAC/E,MAAM,CAAC,WAAW,GAAG,WAAW,CAAC;IACnC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAND,sBAMC;AAED,wBAAwB;AACxB,uBAAuB;AACvB,uBAAuB;AAEvB,MAAa,SAAS;IAGC;IAFZ,SAAS,GAAe,EAAE,CAAC;IAEpC,YAAqB,QAAsB;QAAtB,aAAQ,GAAR,QAAQ,CAAc;IAAI,CAAC;IAEhD,KAAK,CAAC,SAAS;QACb,KAAK,MAAM,mBAAmB,IAAI,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE,CAAC;YACrE,MAAM,EAAE,GAAG,mBAAqE,CAAC;YACjF,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBACnB,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAyB,CAAC;gBAC9C,IAAI,CAAC,EAAE,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC;oBACxC,MAAM,IAAI,iBAAS,CAAC,4BAA4B,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,wEAAwE,CAAC,CAAA;gBACnJ,CAAC;gBACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;oBAC1B,MAAM,IAAI,iBAAS,CAAC,4BAA4B,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,+CAA+C,QAAQ,CAAC,IAAI,4BAA4B,CAAC,CAAA;gBACnK,CAAC;gBACD,MAAM,MAAM,GAA2B,EAAE,CAAC;gBAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,EAAG,CAAC;oBACnC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,CAAA;gBAChC,CAAC;qBACD,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;oBACnB,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAA;gBAC7B,CAAC;gBACD,MAAM,KAAK,GAAG,IAAI,eAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAChD,MAAM,cAAc,GAAG,EAAE,CAAC,cAAc,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;gBACzF,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;gBAChD,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACzB,6FAA6F;gBAC7F,gIAAgI;gBAChI,gEAAgE;gBAChE,MAAM,UAAU,GAAG,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC5F,IAAI,SAAS,GAAG,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,YAAY,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;gBAClG,MAAM,UAAU,GAAG,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/F,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;oBACpC,IAAI,CAAC;wBACH,MAAM,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;wBAClE,MAAM;oBACR,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,MAAM,CAAC,GAAG,KAA6B,CAAC;wBACxC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,EAAE,CAAC,CAAC,6BAA6B;4BACrE,MAAM,IAAA,aAAK,EAAC,SAAS,CAAC,CAAC;4BACvB,SAAS,IAAI,UAAU,CAAC;4BACxB,SAAS;wBACX,CAAC;6BAAM,CAAC;4BACN,MAAM,CAAC,CAAA;wBACT,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,MAAM,QAAQ,CAAC,GAAG,CAAC;oBACjB,WAAW,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,EAAE;wBACnD,2EAA2E;wBAC3E,MAAM,YAAY,GAAG,mBAAmB,KAAK,IAAI,SAAS,IAAI,OAAO,CAAC,MAAM,EAAE,CAAA;wBAC9E,MAAM,QAAQ,GAAG,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC;wBAChD,oFAAoF;wBACpF,MAAM,IAAI,GAAc,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;wBACpD,0FAA0F;wBAC1F,IAAI,EAAE,CAAC,SAAS,EAAE,CAAC;4BACjB,0BAA0B;4BAC1B,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,kBAAqD,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,CAAC;wBAC/G,CAAC;6BAAM,IAAI,EAAE,CAAC,cAAc,EAAE,CAAC;4BAC7B,4BAA4B;4BAC5B,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,kBAAkD,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,CAAC;wBACzG,CAAC;oBACH,CAAC;iBACF,CAAC,CAAA;gBACF,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,MAAM,QAAQ,CAAC,UAAU,EAAE,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,aAAa,CAAC,MAA8B;QAC1C,MAAM,eAAe,GAAI,MAAM;aAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;aACtB,GAAG,CAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;aAC/C,IAAI,CAAC,GAAG,CAAC,CAAC;QACb,OAAO,oBAAoB,eAAe,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC7D,CAAC;IAED,2BAA2B;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,mBAAmB,EAAE,EAAE;YACjE,MAAM,EAAE,GAAG,mBAAqE,CAAC;YACjF,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBACnB,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAyB,CAAC;gBAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC;oBAClC,EAAE,CAAC,WAAW,CAAC,OAAO,CAAE,UAAU,CAAC,EAAE;wBACnC,MAAM,CAAC,IAAI,CAAC,OAAO,UAAU,OAAO,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;oBAClE,CAAC,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,WAAW,OAAO,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;gBACtE,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AArGD,8BAqGC"}
|