@dbos-inc/dbos-sdk 1.31.23-preview → 1.31.24-preview
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,25 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
export declare
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
safeGroupName(topics: Array<string | RegExp>): string;
|
|
12
|
-
logRegisteredEndpoints(): void;
|
|
13
|
-
}
|
|
14
|
-
export interface KafkaRegistrationInfo {
|
|
15
|
-
kafkaTopics?: string | RegExp | Array<string | RegExp>;
|
|
16
|
-
consumerConfig?: ConsumerConfig;
|
|
17
|
-
queueName?: string;
|
|
18
|
-
}
|
|
19
|
-
export declare function KafkaConsume(topics: string | RegExp | Array<string | RegExp>, consumerConfig?: ConsumerConfig, queueName?: string): <This, Ctx extends DBOSContext, Args extends KafkaArgs | [Ctx, string, number, KafkaMessage], Return>(target: object, propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise<Return>>) => TypedPropertyDescriptor<(this: This, ...args: Args) => Promise<Return>>;
|
|
20
|
-
export interface KafkaDefaults {
|
|
21
|
-
kafkaConfig?: KafkaConfig;
|
|
22
|
-
}
|
|
23
|
-
export declare function Kafka(kafkaConfig: KafkaConfig): <T extends new (...args: unknown[]) => object>(ctor: T) => void;
|
|
24
|
-
export {};
|
|
1
|
+
/**
|
|
2
|
+
* @deprecated The `@KafkaConsume` decorator function has moved to an extension package.
|
|
3
|
+
* Please install @dbos-inc/dbos-kafkajs, and change your import.
|
|
4
|
+
*/
|
|
5
|
+
export declare function KafkaConsume(_topics: unknown, _consumerConfig?: unknown, _queueName?: unknown): never;
|
|
6
|
+
/**
|
|
7
|
+
* @deprecated The `@Kafka` decorator function has moved to an extension package.
|
|
8
|
+
* Please install @dbos-inc/dbos-kafkajs, and change your import.
|
|
9
|
+
*/
|
|
10
|
+
export declare function Kafka(_kafkaConfig: unknown): never;
|
|
25
11
|
//# sourceMappingURL=kafka.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"kafka.d.ts","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"kafka.d.ts","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":"AAIA;;;GAGG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,eAAe,CAAC,EAAE,OAAO,EAAE,UAAW,CAAC,EAAE,OAAO,GAAI,KAAK,CAEvG;AAED;;;GAGG;AACH,wBAAgB,KAAK,CAAC,YAAY,EAAE,OAAO,GAAI,KAAK,CAEnD"}
|
package/dist/src/kafka/kafka.js
CHANGED
|
@@ -1,151 +1,23 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Kafka = exports.KafkaConsume = exports.DBOSKafka = void 0;
|
|
4
|
-
const kafkajs_1 = require("kafkajs");
|
|
5
|
-
const __1 = require("..");
|
|
6
|
-
const __2 = require("..");
|
|
7
|
-
const utils_1 = require("../utils");
|
|
8
|
-
////////////////////////
|
|
9
|
-
/* Kafka Management */
|
|
10
|
-
///////////////////////
|
|
11
|
-
class DBOSKafka {
|
|
12
|
-
consumers = [];
|
|
13
|
-
executor = undefined;
|
|
14
|
-
constructor() { }
|
|
15
|
-
async initialize(dbosExecI) {
|
|
16
|
-
this.executor = dbosExecI;
|
|
17
|
-
const regops = this.executor.getRegistrationsFor(this);
|
|
18
|
-
for (const registeredOperation of regops) {
|
|
19
|
-
const ro = registeredOperation.methodConfig;
|
|
20
|
-
if (ro.kafkaTopics) {
|
|
21
|
-
const defaults = registeredOperation.classConfig;
|
|
22
|
-
const method = registeredOperation.methodReg;
|
|
23
|
-
const cname = method.className;
|
|
24
|
-
const mname = method.name;
|
|
25
|
-
if (!method.txnConfig && !method.workflowConfig) {
|
|
26
|
-
throw new __2.Error.DBOSError(`Error registering method ${cname}.${mname}: A Kafka decorator can only be assigned to a transaction or workflow!`);
|
|
27
|
-
}
|
|
28
|
-
if (!defaults.kafkaConfig) {
|
|
29
|
-
throw new __2.Error.DBOSError(`Error registering method ${cname}.${mname}: Kafka configuration not found. Does class ${cname} have an @Kafka decorator?`);
|
|
30
|
-
}
|
|
31
|
-
const topics = [];
|
|
32
|
-
if (Array.isArray(ro.kafkaTopics)) {
|
|
33
|
-
topics.push(...ro.kafkaTopics);
|
|
34
|
-
}
|
|
35
|
-
else if (ro.kafkaTopics) {
|
|
36
|
-
topics.push(ro.kafkaTopics);
|
|
37
|
-
}
|
|
38
|
-
const kafka = new kafkajs_1.Kafka(defaults.kafkaConfig);
|
|
39
|
-
const consumerConfig = ro.consumerConfig ?? { groupId: `${this.safeGroupName(topics)}` };
|
|
40
|
-
const consumer = kafka.consumer(consumerConfig);
|
|
41
|
-
await consumer.connect();
|
|
42
|
-
// A temporary workaround for https://github.com/tulios/kafkajs/pull/1558 until it gets fixed
|
|
43
|
-
// If topic autocreation is on and you try to subscribe to a nonexistent topic, KafkaJS should retry until the topic is created.
|
|
44
|
-
// However, it has a bug where it won't. Thus, we retry instead.
|
|
45
|
-
const maxRetries = defaults.kafkaConfig.retry ? defaults.kafkaConfig.retry.retries ?? 5 : 5;
|
|
46
|
-
let retryTime = defaults.kafkaConfig.retry ? defaults.kafkaConfig.retry.maxRetryTime ?? 300 : 300;
|
|
47
|
-
const multiplier = defaults.kafkaConfig.retry ? defaults.kafkaConfig.retry.multiplier ?? 2 : 2;
|
|
48
|
-
for (let i = 0; i < maxRetries; i++) {
|
|
49
|
-
try {
|
|
50
|
-
await consumer.subscribe({ topics: topics, fromBeginning: true });
|
|
51
|
-
break;
|
|
52
|
-
}
|
|
53
|
-
catch (error) {
|
|
54
|
-
const e = error;
|
|
55
|
-
if (e.code === 3 && i + 1 < maxRetries) { // UNKNOWN_TOPIC_OR_PARTITION
|
|
56
|
-
await (0, utils_1.sleepms)(retryTime);
|
|
57
|
-
retryTime *= multiplier;
|
|
58
|
-
continue;
|
|
59
|
-
}
|
|
60
|
-
else {
|
|
61
|
-
throw e;
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
await consumer.run({
|
|
66
|
-
eachMessage: async ({ topic, partition, message }) => {
|
|
67
|
-
// This combination uniquely identifies a message for a given Kafka cluster
|
|
68
|
-
const workflowUUID = `kafka-unique-id-${topic}-${partition}-${message.offset}`;
|
|
69
|
-
const wfParams = { workflowUUID: workflowUUID, configuredInstance: null, queueName: ro.queueName };
|
|
70
|
-
// All operations annotated with Kafka decorators must take in these three arguments
|
|
71
|
-
const args = [topic, partition, message];
|
|
72
|
-
// We can only guarantee exactly-once-per-message execution of transactions and workflows.
|
|
73
|
-
if (method.txnConfig) {
|
|
74
|
-
// Execute the transaction
|
|
75
|
-
await this.executor.transaction(method.registeredFunction, wfParams, ...args);
|
|
76
|
-
}
|
|
77
|
-
else if (method.workflowConfig) {
|
|
78
|
-
// Safely start the workflow
|
|
79
|
-
await this.executor.workflow(method.registeredFunction, wfParams, ...args);
|
|
80
|
-
}
|
|
81
|
-
},
|
|
82
|
-
});
|
|
83
|
-
this.consumers.push(consumer);
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
async destroy() {
|
|
88
|
-
for (const consumer of this.consumers) {
|
|
89
|
-
await consumer.disconnect();
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
safeGroupName(topics) {
|
|
93
|
-
const safeGroupIdPart = topics
|
|
94
|
-
.map(r => r.toString())
|
|
95
|
-
.map(r => r.replaceAll(/[^a-zA-Z0-9\\-]/g, ''))
|
|
96
|
-
.join('-');
|
|
97
|
-
return `dbos-kafka-group-${safeGroupIdPart}`.slice(0, 255);
|
|
98
|
-
}
|
|
99
|
-
logRegisteredEndpoints() {
|
|
100
|
-
if (!this.executor)
|
|
101
|
-
return;
|
|
102
|
-
const logger = this.executor.logger;
|
|
103
|
-
logger.info("Kafka endpoints supported:");
|
|
104
|
-
const regops = this.executor.getRegistrationsFor(this);
|
|
105
|
-
regops.forEach((registeredOperation) => {
|
|
106
|
-
const ro = registeredOperation.methodConfig;
|
|
107
|
-
if (ro.kafkaTopics) {
|
|
108
|
-
const cname = registeredOperation.methodReg.className;
|
|
109
|
-
const mname = registeredOperation.methodReg.name;
|
|
110
|
-
if (Array.isArray(ro.kafkaTopics)) {
|
|
111
|
-
ro.kafkaTopics.forEach(kafkaTopic => {
|
|
112
|
-
logger.info(` ${kafkaTopic} -> ${cname}.${mname}`);
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
else {
|
|
116
|
-
logger.info(` ${ro.kafkaTopics} -> ${cname}.${mname}`);
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
exports.DBOSKafka = DBOSKafka;
|
|
123
2
|
/////////////////////////////
|
|
124
|
-
/* Kafka
|
|
3
|
+
/* Kafka Decorators */
|
|
125
4
|
/////////////////////////////
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
kafkaRegistration.consumerConfig = consumerConfig;
|
|
135
|
-
kafkaRegistration.queueName = queueName;
|
|
136
|
-
return descriptor;
|
|
137
|
-
}
|
|
138
|
-
return kafkadec;
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.Kafka = exports.KafkaConsume = void 0;
|
|
7
|
+
/**
|
|
8
|
+
* @deprecated The `@KafkaConsume` decorator function has moved to an extension package.
|
|
9
|
+
* Please install @dbos-inc/dbos-kafkajs, and change your import.
|
|
10
|
+
*/
|
|
11
|
+
function KafkaConsume(_topics, _consumerConfig, _queueName) {
|
|
12
|
+
throw new Error("@KafkaConsume decorator has moved to the `@dbos-inc/dbos-kafkajs` package, please install and change the imports");
|
|
139
13
|
}
|
|
140
14
|
exports.KafkaConsume = KafkaConsume;
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
}
|
|
148
|
-
return clsdec;
|
|
15
|
+
/**
|
|
16
|
+
* @deprecated The `@Kafka` decorator function has moved to an extension package.
|
|
17
|
+
* Please install @dbos-inc/dbos-kafkajs, and change your import.
|
|
18
|
+
*/
|
|
19
|
+
function Kafka(_kafkaConfig) {
|
|
20
|
+
throw new Error("@Kafka decorator has moved to the `@dbos-inc/dbos-kafkajs` package, please install and change the imports");
|
|
149
21
|
}
|
|
150
22
|
exports.Kafka = Kafka;
|
|
151
23
|
//# sourceMappingURL=kafka.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"kafka.js","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"kafka.js","sourceRoot":"","sources":["../../../src/kafka/kafka.ts"],"names":[],"mappings":";AAAA,6BAA6B;AAC7B,uBAAuB;AACvB,6BAA6B;;;AAE7B;;;GAGG;AACH,SAAgB,YAAY,CAAC,OAAgB,EAAE,eAAyB,EAAE,UAAqB;IAC7F,MAAM,IAAI,KAAK,CAAC,kHAAkH,CAAC,CAAC;AACtI,CAAC;AAFD,oCAEC;AAED;;;GAGG;AACH,SAAgB,KAAK,CAAC,YAAqB;IACzC,MAAM,IAAI,KAAK,CAAC,2GAA2G,CAAC,CAAC;AAC/H,CAAC;AAFD,sBAEC"}
|