@dbos-inc/dbos-sdk 1.31.23-preview → 1.31.24-preview
Sign up to get free protection for your applications and to get access to all the features.
|
@@ -1,25 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
export declare
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
safeGroupName(topics: Array<string | RegExp>): string;
|
|
12
|
-
logRegisteredEndpoints(): void;
|
|
13
|
-
}
|
|
14
|
-
export interface KafkaRegistrationInfo {
|
|
15
|
-
kafkaTopics?: string | RegExp | Array<string | RegExp>;
|
|
16
|
-
consumerConfig?: ConsumerConfig;
|
|
17
|
-
queueName?: string;
|
|
18
|
-
}
|
|
19
|
-
export declare function KafkaConsume(topics: string | RegExp | Array<string | RegExp>, consumerConfig?: ConsumerConfig, queueName?: string): <This, Ctx extends DBOSContext, Args extends KafkaArgs | [Ctx, string, number, KafkaMessage], Return>(target: object, propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise<Return>>) => TypedPropertyDescriptor<(this: This, ...args: Args) => Promise<Return>>;
|
|
20
|
-
export interface KafkaDefaults {
|
|
21
|
-
kafkaConfig?: KafkaConfig;
|
|
22
|
-
}
|
|
23
|
-
export declare function Kafka(kafkaConfig: KafkaConfig): <T extends new (...args: unknown[]) => object>(ctor: T) => void;
|
|
24
|
-
export {};
|
|
1
|
+
/**
|
|
2
|
+
* @deprecated The `@KafkaConsume` decorator function has moved to an extension package.
|
|
3
|
+
* Please install @dbos-inc/dbos-kafkajs, and change your import.
|
|
4
|
+
*/
|
|
5
|
+
export declare function KafkaConsume(_topics: unknown, _consumerConfig?: unknown, _queueName?: unknown): never;
|
|
6
|
+
/**
|
|
7
|
+
* @deprecated The `@Kafka` decorator function has moved to an extension package.
|
|
8
|
+
* Please install @dbos-inc/dbos-kafkajs, and change your import.
|
|
9
|
+
*/
|
|
10
|
+
export declare function Kafka(_kafkaConfig: unknown): never;
|
|
25
11
|
//# sourceMappingURL=kafka.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"kafka.d.ts","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"kafka.d.ts","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":"AAIA;;;GAGG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,eAAe,CAAC,EAAE,OAAO,EAAE,UAAW,CAAC,EAAE,OAAO,GAAI,KAAK,CAEvG;AAED;;;GAGG;AACH,wBAAgB,KAAK,CAAC,YAAY,EAAE,OAAO,GAAI,KAAK,CAEnD"}
|
package/dist/src/kafka/kafka.js
CHANGED
|
@@ -1,151 +1,23 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Kafka = exports.KafkaConsume = exports.DBOSKafka = void 0;
|
|
4
|
-
const kafkajs_1 = require("kafkajs");
|
|
5
|
-
const __1 = require("..");
|
|
6
|
-
const __2 = require("..");
|
|
7
|
-
const utils_1 = require("../utils");
|
|
8
|
-
////////////////////////
|
|
9
|
-
/* Kafka Management */
|
|
10
|
-
///////////////////////
|
|
11
|
-
class DBOSKafka {
|
|
12
|
-
consumers = [];
|
|
13
|
-
executor = undefined;
|
|
14
|
-
constructor() { }
|
|
15
|
-
async initialize(dbosExecI) {
|
|
16
|
-
this.executor = dbosExecI;
|
|
17
|
-
const regops = this.executor.getRegistrationsFor(this);
|
|
18
|
-
for (const registeredOperation of regops) {
|
|
19
|
-
const ro = registeredOperation.methodConfig;
|
|
20
|
-
if (ro.kafkaTopics) {
|
|
21
|
-
const defaults = registeredOperation.classConfig;
|
|
22
|
-
const method = registeredOperation.methodReg;
|
|
23
|
-
const cname = method.className;
|
|
24
|
-
const mname = method.name;
|
|
25
|
-
if (!method.txnConfig && !method.workflowConfig) {
|
|
26
|
-
throw new __2.Error.DBOSError(`Error registering method ${cname}.${mname}: A Kafka decorator can only be assigned to a transaction or workflow!`);
|
|
27
|
-
}
|
|
28
|
-
if (!defaults.kafkaConfig) {
|
|
29
|
-
throw new __2.Error.DBOSError(`Error registering method ${cname}.${mname}: Kafka configuration not found. Does class ${cname} have an @Kafka decorator?`);
|
|
30
|
-
}
|
|
31
|
-
const topics = [];
|
|
32
|
-
if (Array.isArray(ro.kafkaTopics)) {
|
|
33
|
-
topics.push(...ro.kafkaTopics);
|
|
34
|
-
}
|
|
35
|
-
else if (ro.kafkaTopics) {
|
|
36
|
-
topics.push(ro.kafkaTopics);
|
|
37
|
-
}
|
|
38
|
-
const kafka = new kafkajs_1.Kafka(defaults.kafkaConfig);
|
|
39
|
-
const consumerConfig = ro.consumerConfig ?? { groupId: `${this.safeGroupName(topics)}` };
|
|
40
|
-
const consumer = kafka.consumer(consumerConfig);
|
|
41
|
-
await consumer.connect();
|
|
42
|
-
// A temporary workaround for https://github.com/tulios/kafkajs/pull/1558 until it gets fixed
|
|
43
|
-
// If topic autocreation is on and you try to subscribe to a nonexistent topic, KafkaJS should retry until the topic is created.
|
|
44
|
-
// However, it has a bug where it won't. Thus, we retry instead.
|
|
45
|
-
const maxRetries = defaults.kafkaConfig.retry ? defaults.kafkaConfig.retry.retries ?? 5 : 5;
|
|
46
|
-
let retryTime = defaults.kafkaConfig.retry ? defaults.kafkaConfig.retry.maxRetryTime ?? 300 : 300;
|
|
47
|
-
const multiplier = defaults.kafkaConfig.retry ? defaults.kafkaConfig.retry.multiplier ?? 2 : 2;
|
|
48
|
-
for (let i = 0; i < maxRetries; i++) {
|
|
49
|
-
try {
|
|
50
|
-
await consumer.subscribe({ topics: topics, fromBeginning: true });
|
|
51
|
-
break;
|
|
52
|
-
}
|
|
53
|
-
catch (error) {
|
|
54
|
-
const e = error;
|
|
55
|
-
if (e.code === 3 && i + 1 < maxRetries) { // UNKNOWN_TOPIC_OR_PARTITION
|
|
56
|
-
await (0, utils_1.sleepms)(retryTime);
|
|
57
|
-
retryTime *= multiplier;
|
|
58
|
-
continue;
|
|
59
|
-
}
|
|
60
|
-
else {
|
|
61
|
-
throw e;
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
await consumer.run({
|
|
66
|
-
eachMessage: async ({ topic, partition, message }) => {
|
|
67
|
-
// This combination uniquely identifies a message for a given Kafka cluster
|
|
68
|
-
const workflowUUID = `kafka-unique-id-${topic}-${partition}-${message.offset}`;
|
|
69
|
-
const wfParams = { workflowUUID: workflowUUID, configuredInstance: null, queueName: ro.queueName };
|
|
70
|
-
// All operations annotated with Kafka decorators must take in these three arguments
|
|
71
|
-
const args = [topic, partition, message];
|
|
72
|
-
// We can only guarantee exactly-once-per-message execution of transactions and workflows.
|
|
73
|
-
if (method.txnConfig) {
|
|
74
|
-
// Execute the transaction
|
|
75
|
-
await this.executor.transaction(method.registeredFunction, wfParams, ...args);
|
|
76
|
-
}
|
|
77
|
-
else if (method.workflowConfig) {
|
|
78
|
-
// Safely start the workflow
|
|
79
|
-
await this.executor.workflow(method.registeredFunction, wfParams, ...args);
|
|
80
|
-
}
|
|
81
|
-
},
|
|
82
|
-
});
|
|
83
|
-
this.consumers.push(consumer);
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
async destroy() {
|
|
88
|
-
for (const consumer of this.consumers) {
|
|
89
|
-
await consumer.disconnect();
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
safeGroupName(topics) {
|
|
93
|
-
const safeGroupIdPart = topics
|
|
94
|
-
.map(r => r.toString())
|
|
95
|
-
.map(r => r.replaceAll(/[^a-zA-Z0-9\\-]/g, ''))
|
|
96
|
-
.join('-');
|
|
97
|
-
return `dbos-kafka-group-${safeGroupIdPart}`.slice(0, 255);
|
|
98
|
-
}
|
|
99
|
-
logRegisteredEndpoints() {
|
|
100
|
-
if (!this.executor)
|
|
101
|
-
return;
|
|
102
|
-
const logger = this.executor.logger;
|
|
103
|
-
logger.info("Kafka endpoints supported:");
|
|
104
|
-
const regops = this.executor.getRegistrationsFor(this);
|
|
105
|
-
regops.forEach((registeredOperation) => {
|
|
106
|
-
const ro = registeredOperation.methodConfig;
|
|
107
|
-
if (ro.kafkaTopics) {
|
|
108
|
-
const cname = registeredOperation.methodReg.className;
|
|
109
|
-
const mname = registeredOperation.methodReg.name;
|
|
110
|
-
if (Array.isArray(ro.kafkaTopics)) {
|
|
111
|
-
ro.kafkaTopics.forEach(kafkaTopic => {
|
|
112
|
-
logger.info(` ${kafkaTopic} -> ${cname}.${mname}`);
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
else {
|
|
116
|
-
logger.info(` ${ro.kafkaTopics} -> ${cname}.${mname}`);
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
exports.DBOSKafka = DBOSKafka;
|
|
123
2
|
/////////////////////////////
|
|
124
|
-
/* Kafka
|
|
3
|
+
/* Kafka Decorators */
|
|
125
4
|
/////////////////////////////
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
kafkaRegistration.consumerConfig = consumerConfig;
|
|
135
|
-
kafkaRegistration.queueName = queueName;
|
|
136
|
-
return descriptor;
|
|
137
|
-
}
|
|
138
|
-
return kafkadec;
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.Kafka = exports.KafkaConsume = void 0;
|
|
7
|
+
/**
|
|
8
|
+
* @deprecated The `@KafkaConsume` decorator function has moved to an extension package.
|
|
9
|
+
* Please install @dbos-inc/dbos-kafkajs, and change your import.
|
|
10
|
+
*/
|
|
11
|
+
function KafkaConsume(_topics, _consumerConfig, _queueName) {
|
|
12
|
+
throw new Error("@KafkaConsume decorator has moved to the `@dbos-inc/dbos-kafkajs` package, please install and change the imports");
|
|
139
13
|
}
|
|
140
14
|
exports.KafkaConsume = KafkaConsume;
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
}
|
|
148
|
-
return clsdec;
|
|
15
|
+
/**
|
|
16
|
+
* @deprecated The `@Kafka` decorator function has moved to an extension package.
|
|
17
|
+
* Please install @dbos-inc/dbos-kafkajs, and change your import.
|
|
18
|
+
*/
|
|
19
|
+
function Kafka(_kafkaConfig) {
|
|
20
|
+
throw new Error("@Kafka decorator has moved to the `@dbos-inc/dbos-kafkajs` package, please install and change the imports");
|
|
149
21
|
}
|
|
150
22
|
exports.Kafka = Kafka;
|
|
151
23
|
//# sourceMappingURL=kafka.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"kafka.js","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"kafka.js","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":";AAAA,6BAA6B;AAC7B,uBAAuB;AACvB,6BAA6B;;;AAE7B;;;GAGG;AACH,SAAgB,YAAY,CAAC,OAAgB,EAAE,eAAyB,EAAE,UAAqB;IAC7F,MAAM,IAAI,KAAK,CAAC,kHAAkH,CAAC,CAAC;AACtI,CAAC;AAFD,oCAEC;AAED;;;GAGG;AACH,SAAgB,KAAK,CAAC,YAAqB;IACzC,MAAM,IAAI,KAAK,CAAC,2GAA2G,CAAC,CAAC;AAC/H,CAAC;AAFD,sBAEC"}
|